Debian Codegraph @

b2.go

raw
1 // Package b2 provides an idiomatic, efficient interface to Backblaze B2 Cloud Storage.
2 //
3 // Uploading
4 //
5 // Uploads to B2 require a SHA1 header, so the hash of the file must be known
6 // before the upload starts. The (*Bucket).Upload API tries its best not to
7 // buffer the entire file in memory, and it will avoid it if passed either a
8 // bytes.Buffer or a io.ReadSeeker.
9 //
10 // If you know the SHA1 and the length of the file in advance, you can use
11 // (*Bucket).UploadWithSHA1 but you are responsible for retrying on
12 // transient errors.
13 //
14 // Downloading
15 //
16 // Downloads from B2 are simple GETs, so if you want more control than the
17 // standard functions you can build your own URL according to the API docs.
18 // All the information you need is returned by Client.LoginInfo().
19 //
20 // Hidden files and versions
21 //
22 // There is no first-class support for versions in this library, but most
23 // behaviors are transparently exposed. Upload can be used multiple times
24 // with the same name, ListFiles will only return the latest version of
25 // non-hidden files, and ListFilesVersions will return all files and versions.
26 //
27 // Unsupported APIs
28 //
29 // Large files (b2_*_large_file, b2_*_part), b2_get_download_authorization,
30 // b2_hide_file, b2_update_bucket.
31 //
32 // Debug mode
33 //
34 // If the B2_DEBUG environment variable is set to 1, all API calls will be
35 // logged. On Go 1.7 and later, it will also log when new (non-reused)
36 // connections are established.
37 package b2
38
39 import (
40 "bytes"
41 "encoding/base64"
42 "encoding/json"
43 "errors"
44 "fmt"
45 "io"
46 "io/ioutil"
47 "log"
48 "net/http"
49 "net/url"
50 "os"
51 "sync"
52 "sync/atomic"
53 )
54
55 // Error is the decoded B2 JSON error return value. It's not the only type of
56 // error returned by this package, and it is mostly returned wrapped in a
57 // url.Error. Use UnwrapError to access it.
58 type Error struct {
59 Code string
60 Message string
61 Status int
62 }
63
64 func (e *Error) Error() string {
65 return fmt.Sprintf("b2 remote error [%s]: %s", e.Code, e.Message)
66 }
67
68 // UnwrapError attempts to extract the Error that caused err. If there is no
69 // Error object to unwrap, ok is false and err is nil. That does not mean that
70 // the original error should be ignored.
71 func UnwrapError(err error) (b2Err *Error, ok bool) {
72 if e, ok := err.(*url.Error); ok {
73 err = e.Err
74 }
75 if e, ok := err.(*Error); ok {
76 return e, true
77 }
78 return nil, false
79 }
80
81 const (
82 defaultAPIURL = "https://api.backblaze.com"
83 apiPath = "/b2api/v1/"
84 )
85
86 // LoginInfo holds the information obtained upon login, which are sufficient
87 // to interact with the API directly.
88 type LoginInfo struct {
89 AccountID string
90 ApiURL string
91
92 // DownloadURL is the base URL for file downloads. It is supposed
93 // to never change for the same account.
94 DownloadURL string
95 // AuthorizationToken is the value to pass in the Authorization
96 // header of all private calls. This is valid for at most 24 hours.
97 AuthorizationToken string
98 }
99
100 // LoginInfo returns the LoginInfo object currently in use. If refresh is
101 // true, it obtains a new one before returning.
102 //
103 // Note that once you obtain this there is no guarantee on its freshness,
104 // and it will eventually expire.
105 func (c *Client) LoginInfo(refresh bool) (*LoginInfo, error) {
106 if refresh {
107 if err := c.login(nil); err != nil {
108 return nil, err
109 }
110 }
111 return c.loginInfo.Load().(*LoginInfo), nil
112 }
113
114 // A Client is an authenticated API client. It is safe for concurrent use and should
115 // be reused to take advantage of connection and URL reuse.
116 //
117 // The Client handles refreshing authorization tokens transparently.
118 type Client struct {
119 accountID, applicationKey string
120
121 loginInfo atomic.Value // *LoginInfo
122 // loginMu is held to avoid multiple logins in flight at the same time
123 loginMu sync.Mutex
124
125 hc *http.Client
126 }
127
128 // NewClient calls b2_authorize_account and returns an authenticated Client.
129 // httpClient can be nil, in which case http.DefaultClient will be used.
130 func NewClient(accountID, applicationKey string, httpClient *http.Client) (*Client, error) {
131 if httpClient == nil {
132 httpClient = http.DefaultClient
133 }
134
135 c := &Client{
136 accountID: accountID,
137 applicationKey: applicationKey,
138 hc: httpClient,
139 }
140
141 if err := c.login(nil); err != nil {
142 return nil, err
143 }
144
145 c.hc.Transport = &transport{t: c.hc.Transport, c: c}
146 return c, nil
147 }
148
149 func (c *Client) login(failedRes *http.Response) error {
150 c.loginMu.Lock()
151 defer c.loginMu.Unlock()
152
153 // check under the lock that another login didn't beat us
154 if failedRes != nil && c.loginInfo.Load() != nil {
155 current := c.loginInfo.Load().(*LoginInfo).AuthorizationToken
156 failed := failedRes.Request.Header.Get("Authorization")
157 if current != failed {
158 debugf("another login call succeeded concurrently")
159 return nil
160 }
161 }
162
163 r, err := http.NewRequest("GET", defaultAPIURL+apiPath+"b2_authorize_account", nil)
164 if err != nil {
165 return err
166 }
167 r.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString(
168 []byte(c.accountID+":"+c.applicationKey)))
169
170 res, err := c.hc.Do(r)
171 if err != nil {
172 return err
173 }
174 defer drainAndClose(res.Body)
175 debugf("login: %d", res.StatusCode)
176
177 if res.StatusCode != 200 {
178 b2Err := &Error{}
179 if err := json.NewDecoder(res.Body).Decode(b2Err); err != nil {
180 return fmt.Errorf("unknown error during b2_authorize_account: %d", res.StatusCode)
181 }
182 return b2Err
183 }
184
185 li := &LoginInfo{}
186 if err := json.NewDecoder(res.Body).Decode(li); err != nil {
187 return fmt.Errorf("failed to decode b2_authorize_account answer: %s", err)
188 }
189 c.loginInfo.Store(li)
190
191 return nil
192 }
193
194 // transport is a wrapper providing authentication, tracing and error handling.
195 type transport struct {
196 t http.RoundTripper
197 c *Client
198 }
199
200 // requestExtFunc is implemented in the go1.7 file, to add httptrace
201 var requestExtFunc func(*http.Request) *http.Request
202
203 func (t *transport) RoundTrip(req *http.Request) (res *http.Response, err error) {
204 if req.Header.Get("Authorization") == "" {
205 req.Header.Set("Authorization", t.c.loginInfo.Load().(*LoginInfo).AuthorizationToken)
206 }
207
208 if requestExtFunc != nil {
209 req = requestExtFunc(req)
210 }
211
212 if t.t == nil {
213 res, err = http.DefaultTransport.RoundTrip(req)
214 } else {
215 res, err = t.t.RoundTrip(req)
216 }
217
218 if err == nil && res.StatusCode != 200 {
219 return nil, parseB2Error(res)
220 }
221
222 return res, err
223 }
224
225 var debug = os.Getenv("B2_DEBUG") == "1"
226
227 func debugf(format string, a ...interface{}) {
228 if debug {
229 log.Printf("[b2] "+format, a...)
230 }
231 }
232
233 func (c *Client) doRequest(endpoint string, params map[string]interface{}) (*http.Response, error) {
234 body, err := json.Marshal(params)
235 if err != nil {
236 return nil, err
237 }
238 // Reduce debug log noise
239 delete(params, "accountID")
240 delete(params, "bucketID")
241
242 apiURL := c.loginInfo.Load().(*LoginInfo).ApiURL
243 res, err := c.hc.Post(apiURL+apiPath+endpoint, "application/json", bytes.NewBuffer(body))
244 if e, ok := UnwrapError(err); ok && e.Status == http.StatusUnauthorized {
245 if err = c.login(res); err == nil {
246 res, err = c.hc.Post(apiURL+apiPath+endpoint, "application/json", bytes.NewBuffer(body))
247 }
248 }
249 if err != nil {
250 debugf("%s (%v): %v", endpoint, params, err)
251 } else {
252 debugf("%s (%v)", endpoint, params)
253 }
254 return res, err
255 }
256
257 func parseB2Error(res *http.Response) error {
258 defer drainAndClose(res.Body)
259 b2Err := &Error{}
260 if err := json.NewDecoder(res.Body).Decode(b2Err); err != nil {
261 return fmt.Errorf("unknown error during b2_authorize_account: %d", res.StatusCode)
262 }
263 return b2Err
264 }
265
266 // drainAndClose will make an attempt at flushing and closing the body so that the
267 // underlying connection can be reused. It will not read more than 10KB.
268 func drainAndClose(body io.ReadCloser) {
269 io.CopyN(ioutil.Discard, body, 10*1024)
270 body.Close()
271 }
272
273 // A Bucket is bound to the Client that created it. It is safe for concurrent use and
274 // should be reused to take advantage of connection and URL reuse.
275 type Bucket struct {
276 ID string
277 c *Client
278
279 uploadURLs []*uploadURL
280 uploadURLsMu sync.Mutex
281 }
282
283 // BucketInfo is an extended Bucket object with metadata.
284 type BucketInfo struct {
285 Bucket
286
287 Name string
288 Type string
289 }
290
291 // BucketByID returns a Bucket bound to the Client. It does NOT check that the
292 // bucket actually exists, or perform any network operation.
293 func (c *Client) BucketByID(id string) *Bucket {
294 return &Bucket{ID: id, c: c}
295 }
296
297 // BucketByName returns the Bucket with the given name. If such a bucket is not
298 // found and createIfNotExists is true, CreateBucket is called with allPublic set
299 // to false. Otherwise, an error is returned.
300 func (c *Client) BucketByName(name string, createIfNotExists bool) (*BucketInfo, error) {
301 bs, err := c.Buckets()
302 if err != nil {
303 return nil, err
304 }
305 for _, b := range bs {
306 if b.Name == name {
307 return b, nil
308 }
309 }
310 if !createIfNotExists {
311 return nil, errors.New("bucket not found: " + name)
312 }
313 return c.CreateBucket(name, false)
314 }
315
316 // Buckets returns a list of buckets sorted by name.
317 func (c *Client) Buckets() ([]*BucketInfo, error) {
318 res, err := c.doRequest("b2_list_buckets", map[string]interface{}{
319 "accountId": c.accountID,
320 })
321 if err != nil {
322 return nil, err
323 }
324 defer drainAndClose(res.Body)
325 var buckets struct {
326 Buckets []struct {
327 BucketID, BucketName, BucketType string
328 }
329 }
330 if err := json.NewDecoder(res.Body).Decode(&buckets); err != nil {
331 return nil, err
332 }
333 var r []*BucketInfo
334 for _, b := range buckets.Buckets {
335 r = append(r, &BucketInfo{
336 Bucket: Bucket{
337 ID: b.BucketID,
338 c: c,
339 },
340 Name: b.BucketName,
341 Type: b.BucketType,
342 })
343 }
344 return r, nil
345 }
346
347 // CreateBucket creates a bucket with b2_create_bucket. If allPublic is true,
348 // files in this bucket can be downloaded by anybody.
349 func (c *Client) CreateBucket(name string, allPublic bool) (*BucketInfo, error) {
350 bucketType := "allPrivate"
351 if allPublic {
352 bucketType = "allPublic"
353 }
354 res, err := c.doRequest("b2_create_bucket", map[string]interface{}{
355 "accountId": c.accountID,
356 "bucketName": name,
357 "bucketType": bucketType,
358 })
359 if err != nil {
360 return nil, err
361 }
362 defer drainAndClose(res.Body)
363 var bucket struct {
364 BucketID string
365 }
366 if err := json.NewDecoder(res.Body).Decode(&bucket); err != nil {
367 return nil, err
368 }
369 return &BucketInfo{
370 Bucket: Bucket{
371 c: c, ID: bucket.BucketID,
372 },
373 Name: name,
374 Type: bucketType,
375 }, nil
376 }
377
378 // Delete calls b2_delete_bucket. After this call succeeds the Bucket object
379 // becomes invalid and any other calls will fail.
380 func (b *Bucket) Delete() error {
381 res, err := b.c.doRequest("b2_delete_bucket", map[string]interface{}{
382 "accountId": b.c.accountID,
383 "bucketId": b.ID,
384 })
385 if err != nil {
386 return err
387 }
388 drainAndClose(res.Body)
389 return nil
390 }