Debian Codegraph @

storage/2018-11-09/file/files/range_get_file.go

raw
1 package files
2
3 import (
4 "context"
5 "fmt"
6 "log"
7 "math"
8 "runtime"
9 "sync"
10
11 "github.com/Azure/go-autorest/autorest"
12 )
13
14 // GetFile is a helper method to download a file by chunking it automatically
15 func (client Client) GetFile(ctx context.Context, accountName, shareName, path, fileName string, parallelism int) (result autorest.Response, outputBytes []byte, err error) {
16
17 // first look up the file and check out how many bytes it is
18 file, e := client.GetProperties(ctx, accountName, shareName, path, fileName)
19 if err != nil {
20 result = file.Response
21 err = e
22 return
23 }
24
25 if file.ContentLength == nil {
26 err = fmt.Errorf("Content-Length was nil!")
27 return
28 }
29
30 length := int64(*file.ContentLength)
31 chunkSize := int64(4 * 1024 * 1024) // 4MB
32
33 if chunkSize > length {
34 chunkSize = length
35 }
36
37 // then split that up into chunks and retrieve it retrieve it into the 'results' set
38 chunks := int(math.Ceil(float64(length) / float64(chunkSize)))
39 workerCount := parallelism * runtime.NumCPU()
40 if workerCount > chunks {
41 workerCount = chunks
42 }
43
44 var waitGroup sync.WaitGroup
45 waitGroup.Add(workerCount)
46
47 results := make([]*downloadFileChunkResult, chunks)
48 errors := make(chan error, chunkSize)
49
50 for i := 0; i < chunks; i++ {
51 go func(i int) {
52 log.Printf("[DEBUG] Downloading Chunk %d of %d", i+1, chunks)
53
54 dfci := downloadFileChunkInput{
55 thisChunk: i,
56 chunkSize: chunkSize,
57 fileSize: length,
58 }
59
60 result, err := client.downloadFileChunk(ctx, accountName, shareName, path, fileName, dfci)
61 if err != nil {
62 errors <- err
63 waitGroup.Done()
64 return
65 }
66
67 // if there's no error, we should have bytes, so this is safe
68 results[i] = result
69
70 waitGroup.Done()
71 }(i)
72 }
73 waitGroup.Wait()
74
75 // TODO: we should switch to hashicorp/multi-error here
76 if len(errors) > 0 {
77 err = fmt.Errorf("Error downloading file: %s", <-errors)
78 return
79 }
80
81 // then finally put it all together, in order and return it
82 output := make([]byte, length)
83 for _, v := range results {
84 copy(output[v.startBytes:v.endBytes], v.bytes)
85 }
86
87 outputBytes = output
88 return
89 }
90
91 type downloadFileChunkInput struct {
92 thisChunk int
93 chunkSize int64
94 fileSize int64
95 }
96
97 type downloadFileChunkResult struct {
98 startBytes int64
99 endBytes int64
100 bytes []byte
101 }
102
103 func (client Client) downloadFileChunk(ctx context.Context, accountName, shareName, path, fileName string, input downloadFileChunkInput) (*downloadFileChunkResult, error) {
104 startBytes := input.chunkSize * int64(input.thisChunk)
105 endBytes := startBytes + input.chunkSize
106
107 // the last chunk may exceed the size of the file
108 remaining := input.fileSize - startBytes
109 if input.chunkSize > remaining {
110 endBytes = startBytes + remaining
111 }
112
113 getInput := GetByteRangeInput{
114 StartBytes: startBytes,
115 EndBytes: endBytes,
116 }
117 result, err := client.GetByteRange(ctx, accountName, shareName, path, fileName, getInput)
118 if err != nil {
119 return nil, fmt.Errorf("Error putting bytes: %s", err)
120 }
121
122 output := downloadFileChunkResult{
123 startBytes: startBytes,
124 endBytes: endBytes,
125 bytes: result.Contents,
126 }
127 return &output, nil
128 }