Debian Codegraph @

storage/2018-11-09/file/files/range_put_file.go

raw
1 package files
2
3 import (
4 "context"
5 "fmt"
6 "io"
7 "log"
8 "math"
9 "os"
10 "sync"
11
12 "github.com/Azure/go-autorest/autorest"
13 )
14
15 // PutFile is a helper method which takes a file, and automatically chunks it up, rather than having to do this yourself
16 func (client Client) PutFile(ctx context.Context, accountName, shareName, path, fileName string, file *os.File, parallelism int) error {
17 fileInfo, err := file.Stat()
18 if err != nil {
19 return fmt.Errorf("Error loading file info: %s", err)
20 }
21
22 fileSize := fileInfo.Size()
23 chunkSize := 4 * 1024 * 1024 // 4MB
24 if chunkSize > int(fileSize) {
25 chunkSize = int(fileSize)
26 }
27 chunks := int(math.Ceil(float64(fileSize) / float64(chunkSize*1.0)))
28
29 workerCount := parallelism
30 if workerCount > chunks {
31 workerCount = chunks
32 }
33
34 var waitGroup sync.WaitGroup
35 waitGroup.Add(workerCount)
36
37 jobs := make(chan int, workerCount)
38 errors := make(chan error, chunkSize)
39
40 for i := 0; i < workerCount; i++ {
41 go func() {
42 for i := range jobs {
43 log.Printf("[DEBUG] Chunk %d of %d", i+1, chunks)
44
45 uci := uploadChunkInput{
46 thisChunk: i,
47 chunkSize: chunkSize,
48 fileSize: fileSize,
49 }
50
51 _, err := client.uploadChunk(ctx, accountName, shareName, path, fileName, uci, file)
52 if err != nil {
53 errors <- err
54 }
55 }
56 waitGroup.Done()
57 }()
58 }
59
60 for i := 0; i < chunks; i++ {
61 jobs <- i
62 }
63 close(jobs)
64 waitGroup.Wait()
65
66 // TODO: we should switch to hashicorp/multi-error here
67 if len(errors) > 0 {
68 return fmt.Errorf("Error uploading file: %s", <-errors)
69 }
70
71 return nil
72 }
73
74 type uploadChunkInput struct {
75 thisChunk int
76 chunkSize int
77 fileSize int64
78 }
79
80 func (client Client) uploadChunk(ctx context.Context, accountName, shareName, path, fileName string, input uploadChunkInput, file *os.File) (result autorest.Response, err error) {
81 startBytes := int64(input.chunkSize * input.thisChunk)
82 endBytes := startBytes + int64(input.chunkSize)
83
84 // the last size may exceed the size of the file
85 remaining := input.fileSize - startBytes
86 if int64(input.chunkSize) > remaining {
87 endBytes = startBytes + remaining
88 }
89
90 bytesToRead := int(endBytes) - int(startBytes)
91 bytes := make([]byte, bytesToRead)
92
93 _, err = file.ReadAt(bytes, startBytes)
94 if err != nil {
95 if err != io.EOF {
96 return result, fmt.Errorf("Error reading bytes: %s", err)
97 }
98 }
99
100 putBytesInput := PutByteRangeInput{
101 StartBytes: startBytes,
102 EndBytes: endBytes,
103 Content: bytes,
104 }
105 result, err = client.PutByteRange(ctx, accountName, shareName, path, fileName, putBytesInput)
106 if err != nil {
107 return result, fmt.Errorf("Error putting bytes: %s", err)
108 }
109
110 return
111 }