| 1 |
// Package b2 provides an idiomatic, efficient interface to Backblaze B2 Cloud Storage. |
| 2 |
// |
| 3 |
// Uploading |
| 4 |
// |
| 5 |
// Uploads to B2 require a SHA1 header, so the hash of the file must be known |
| 6 |
// before the upload starts. The (*Bucket).Upload API tries its best not to |
| 7 |
// buffer the entire file in memory, and it will avoid it if passed either a |
| 8 |
// bytes.Buffer or a io.ReadSeeker. |
| 9 |
// |
| 10 |
// If you know the SHA1 and the length of the file in advance, you can use |
| 11 |
// (*Bucket).UploadWithSHA1 but you are responsible for retrying on |
| 12 |
// transient errors. |
| 13 |
// |
| 14 |
// Downloading |
| 15 |
// |
| 16 |
// Downloads from B2 are simple GETs, so if you want more control than the |
| 17 |
// standard functions you can build your own URL according to the API docs. |
| 18 |
// All the information you need is returned by Client.LoginInfo(). |
| 19 |
// |
| 20 |
// Hidden files and versions |
| 21 |
// |
| 22 |
// There is no first-class support for versions in this library, but most |
| 23 |
// behaviors are transparently exposed. Upload can be used multiple times |
| 24 |
// with the same name, ListFiles will only return the latest version of |
| 25 |
// non-hidden files, and ListFilesVersions will return all files and versions. |
| 26 |
// |
| 27 |
// Unsupported APIs |
| 28 |
// |
| 29 |
// Large files (b2_*_large_file, b2_*_part), b2_get_download_authorization, |
| 30 |
// b2_hide_file, b2_update_bucket. |
| 31 |
// |
| 32 |
// Debug mode |
| 33 |
// |
| 34 |
// If the B2_DEBUG environment variable is set to 1, all API calls will be |
| 35 |
// logged. On Go 1.7 and later, it will also log when new (non-reused) |
| 36 |
// connections are established. |
| 37 |
package b2 |
| 38 |
|
| 39 |
import ( |
| 40 |
"bytes" |
| 41 |
"encoding/base64" |
| 42 |
"encoding/json" |
| 43 |
"errors" |
| 44 |
"fmt" |
| 45 |
"io" |
| 46 |
"io/ioutil" |
| 47 |
"log" |
| 48 |
"net/http" |
| 49 |
"net/url" |
| 50 |
"os" |
| 51 |
"sync" |
| 52 |
"sync/atomic" |
| 53 |
) |
| 54 |
|
| 55 |
// Error is the decoded B2 JSON error return value. It's not the only type of |
| 56 |
// error returned by this package, and it is mostly returned wrapped in a |
| 57 |
// url.Error. Use UnwrapError to access it. |
| 58 |
type Error struct { |
| 59 |
Code string |
| 60 |
Message string |
| 61 |
Status int |
| 62 |
} |
| 63 |
|
| 64 |
func (e *Error) Error() string { |
| 65 |
return fmt.Sprintf("b2 remote error [%s]: %s", e.Code, e.Message) |
| 66 |
} |
| 67 |
|
| 68 |
// UnwrapError attempts to extract the Error that caused err. If there is no |
| 69 |
// Error object to unwrap, ok is false and err is nil. That does not mean that |
| 70 |
// the original error should be ignored. |
| 71 |
func UnwrapError(err error) (b2Err *Error, ok bool) { |
| 72 |
if e, ok := err.(*url.Error); ok { |
| 73 |
err = e.Err |
| 74 |
} |
| 75 |
if e, ok := err.(*Error); ok { |
| 76 |
return e, true |
| 77 |
} |
| 78 |
return nil, false |
| 79 |
} |
| 80 |
|
| 81 |
const ( |
| 82 |
defaultAPIURL = "https://api.backblaze.com" |
| 83 |
apiPath = "/b2api/v1/" |
| 84 |
) |
| 85 |
|
| 86 |
// LoginInfo holds the information obtained upon login, which are sufficient |
| 87 |
// to interact with the API directly. |
| 88 |
type LoginInfo struct { |
| 89 |
AccountID string |
| 90 |
ApiURL string |
| 91 |
|
| 92 |
// DownloadURL is the base URL for file downloads. It is supposed |
| 93 |
// to never change for the same account. |
| 94 |
DownloadURL string |
| 95 |
// AuthorizationToken is the value to pass in the Authorization |
| 96 |
// header of all private calls. This is valid for at most 24 hours. |
| 97 |
AuthorizationToken string |
| 98 |
} |
| 99 |
|
| 100 |
// LoginInfo returns the LoginInfo object currently in use. If refresh is |
| 101 |
// true, it obtains a new one before returning. |
| 102 |
// |
| 103 |
// Note that once you obtain this there is no guarantee on its freshness, |
| 104 |
// and it will eventually expire. |
| 105 |
func (c *Client) LoginInfo(refresh bool) (*LoginInfo, error) { |
| 106 |
if refresh { |
| 107 |
if err := c.login(nil); err != nil { |
| 108 |
return nil, err |
| 109 |
} |
| 110 |
} |
| 111 |
return c.loginInfo.Load().(*LoginInfo), nil |
| 112 |
} |
| 113 |
|
| 114 |
// A Client is an authenticated API client. It is safe for concurrent use and should |
| 115 |
// be reused to take advantage of connection and URL reuse. |
| 116 |
// |
| 117 |
// The Client handles refreshing authorization tokens transparently. |
| 118 |
type Client struct { |
| 119 |
accountID, applicationKey string |
| 120 |
|
| 121 |
loginInfo atomic.Value // *LoginInfo |
| 122 |
// loginMu is held to avoid multiple logins in flight at the same time |
| 123 |
loginMu sync.Mutex |
| 124 |
|
| 125 |
hc *http.Client |
| 126 |
} |
| 127 |
|
| 128 |
// NewClient calls b2_authorize_account and returns an authenticated Client. |
| 129 |
// httpClient can be nil, in which case http.DefaultClient will be used. |
| 130 |
func NewClient(accountID, applicationKey string, httpClient *http.Client) (*Client, error) { |
| 131 |
if httpClient == nil { |
| 132 |
httpClient = http.DefaultClient |
| 133 |
} |
| 134 |
|
| 135 |
c := &Client{ |
| 136 |
accountID: accountID, |
| 137 |
applicationKey: applicationKey, |
| 138 |
hc: httpClient, |
| 139 |
} |
| 140 |
|
| 141 |
if err := c.login(nil); err != nil { |
| 142 |
return nil, err |
| 143 |
} |
| 144 |
|
| 145 |
c.hc.Transport = &transport{t: c.hc.Transport, c: c} |
| 146 |
return c, nil |
| 147 |
} |
| 148 |
|
| 149 |
func (c *Client) login(failedRes *http.Response) error { |
| 150 |
c.loginMu.Lock() |
| 151 |
defer c.loginMu.Unlock() |
| 152 |
|
| 153 |
// check under the lock that another login didn't beat us |
| 154 |
if failedRes != nil && c.loginInfo.Load() != nil { |
| 155 |
current := c.loginInfo.Load().(*LoginInfo).AuthorizationToken |
| 156 |
failed := failedRes.Request.Header.Get("Authorization") |
| 157 |
if current != failed { |
| 158 |
debugf("another login call succeeded concurrently") |
| 159 |
return nil |
| 160 |
} |
| 161 |
} |
| 162 |
|
| 163 |
r, err := http.NewRequest("GET", defaultAPIURL+apiPath+"b2_authorize_account", nil) |
| 164 |
if err != nil { |
| 165 |
return err |
| 166 |
} |
| 167 |
r.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString( |
| 168 |
[]byte(c.accountID+":"+c.applicationKey))) |
| 169 |
|
| 170 |
res, err := c.hc.Do(r) |
| 171 |
if err != nil { |
| 172 |
return err |
| 173 |
} |
| 174 |
defer drainAndClose(res.Body) |
| 175 |
debugf("login: %d", res.StatusCode) |
| 176 |
|
| 177 |
if res.StatusCode != 200 { |
| 178 |
b2Err := &Error{} |
| 179 |
if err := json.NewDecoder(res.Body).Decode(b2Err); err != nil { |
| 180 |
return fmt.Errorf("unknown error during b2_authorize_account: %d", res.StatusCode) |
| 181 |
} |
| 182 |
return b2Err |
| 183 |
} |
| 184 |
|
| 185 |
li := &LoginInfo{} |
| 186 |
if err := json.NewDecoder(res.Body).Decode(li); err != nil { |
| 187 |
return fmt.Errorf("failed to decode b2_authorize_account answer: %s", err) |
| 188 |
} |
| 189 |
c.loginInfo.Store(li) |
| 190 |
|
| 191 |
return nil |
| 192 |
} |
| 193 |
|
| 194 |
// transport is a wrapper providing authentication, tracing and error handling. |
| 195 |
type transport struct { |
| 196 |
t http.RoundTripper |
| 197 |
c *Client |
| 198 |
} |
| 199 |
|
| 200 |
// requestExtFunc is implemented in the go1.7 file, to add httptrace |
| 201 |
var requestExtFunc func(*http.Request) *http.Request |
| 202 |
|
| 203 |
func (t *transport) RoundTrip(req *http.Request) (res *http.Response, err error) { |
| 204 |
if req.Header.Get("Authorization") == "" { |
| 205 |
req.Header.Set("Authorization", t.c.loginInfo.Load().(*LoginInfo).AuthorizationToken) |
| 206 |
} |
| 207 |
|
| 208 |
if requestExtFunc != nil { |
| 209 |
req = requestExtFunc(req) |
| 210 |
} |
| 211 |
|
| 212 |
if t.t == nil { |
| 213 |
res, err = http.DefaultTransport.RoundTrip(req) |
| 214 |
} else { |
| 215 |
res, err = t.t.RoundTrip(req) |
| 216 |
} |
| 217 |
|
| 218 |
if err == nil && res.StatusCode != 200 { |
| 219 |
return nil, parseB2Error(res) |
| 220 |
} |
| 221 |
|
| 222 |
return res, err |
| 223 |
} |
| 224 |
|
| 225 |
var debug = os.Getenv("B2_DEBUG") == "1" |
| 226 |
|
| 227 |
func debugf(format string, a ...interface{}) { |
| 228 |
if debug { |
| 229 |
log.Printf("[b2] "+format, a...) |
| 230 |
} |
| 231 |
} |
| 232 |
|
| 233 |
func (c *Client) doRequest(endpoint string, params map[string]interface{}) (*http.Response, error) { |
| 234 |
body, err := json.Marshal(params) |
| 235 |
if err != nil { |
| 236 |
return nil, err |
| 237 |
} |
| 238 |
// Reduce debug log noise |
| 239 |
delete(params, "accountID") |
| 240 |
delete(params, "bucketID") |
| 241 |
|
| 242 |
apiURL := c.loginInfo.Load().(*LoginInfo).ApiURL |
| 243 |
res, err := c.hc.Post(apiURL+apiPath+endpoint, "application/json", bytes.NewBuffer(body)) |
| 244 |
if e, ok := UnwrapError(err); ok && e.Status == http.StatusUnauthorized { |
| 245 |
if err = c.login(res); err == nil { |
| 246 |
res, err = c.hc.Post(apiURL+apiPath+endpoint, "application/json", bytes.NewBuffer(body)) |
| 247 |
} |
| 248 |
} |
| 249 |
if err != nil { |
| 250 |
debugf("%s (%v): %v", endpoint, params, err) |
| 251 |
} else { |
| 252 |
debugf("%s (%v)", endpoint, params) |
| 253 |
} |
| 254 |
return res, err |
| 255 |
} |
| 256 |
|
| 257 |
func parseB2Error(res *http.Response) error { |
| 258 |
defer drainAndClose(res.Body) |
| 259 |
b2Err := &Error{} |
| 260 |
if err := json.NewDecoder(res.Body).Decode(b2Err); err != nil { |
| 261 |
return fmt.Errorf("unknown error during b2_authorize_account: %d", res.StatusCode) |
| 262 |
} |
| 263 |
return b2Err |
| 264 |
} |
| 265 |
|
| 266 |
// drainAndClose will make an attempt at flushing and closing the body so that the |
| 267 |
// underlying connection can be reused. It will not read more than 10KB. |
| 268 |
func drainAndClose(body io.ReadCloser) { |
| 269 |
io.CopyN(ioutil.Discard, body, 10*1024) |
| 270 |
body.Close() |
| 271 |
} |
| 272 |
|
| 273 |
// A Bucket is bound to the Client that created it. It is safe for concurrent use and |
| 274 |
// should be reused to take advantage of connection and URL reuse. |
| 275 |
type Bucket struct { |
| 276 |
ID string |
| 277 |
c *Client |
| 278 |
|
| 279 |
uploadURLs []*uploadURL |
| 280 |
uploadURLsMu sync.Mutex |
| 281 |
} |
| 282 |
|
| 283 |
// BucketInfo is an extended Bucket object with metadata. |
| 284 |
type BucketInfo struct { |
| 285 |
Bucket |
| 286 |
|
| 287 |
Name string |
| 288 |
Type string |
| 289 |
} |
| 290 |
|
| 291 |
// BucketByID returns a Bucket bound to the Client. It does NOT check that the |
| 292 |
// bucket actually exists, or perform any network operation. |
| 293 |
func (c *Client) BucketByID(id string) *Bucket { |
| 294 |
return &Bucket{ID: id, c: c} |
| 295 |
} |
| 296 |
|
| 297 |
// BucketByName returns the Bucket with the given name. If such a bucket is not |
| 298 |
// found and createIfNotExists is true, CreateBucket is called with allPublic set |
| 299 |
// to false. Otherwise, an error is returned. |
| 300 |
func (c *Client) BucketByName(name string, createIfNotExists bool) (*BucketInfo, error) { |
| 301 |
bs, err := c.Buckets() |
| 302 |
if err != nil { |
| 303 |
return nil, err |
| 304 |
} |
| 305 |
for _, b := range bs { |
| 306 |
if b.Name == name { |
| 307 |
return b, nil |
| 308 |
} |
| 309 |
} |
| 310 |
if !createIfNotExists { |
| 311 |
return nil, errors.New("bucket not found: " + name) |
| 312 |
} |
| 313 |
return c.CreateBucket(name, false) |
| 314 |
} |
| 315 |
|
| 316 |
// Buckets returns a list of buckets sorted by name. |
| 317 |
func (c *Client) Buckets() ([]*BucketInfo, error) { |
| 318 |
res, err := c.doRequest("b2_list_buckets", map[string]interface{}{ |
| 319 |
"accountId": c.accountID, |
| 320 |
}) |
| 321 |
if err != nil { |
| 322 |
return nil, err |
| 323 |
} |
| 324 |
defer drainAndClose(res.Body) |
| 325 |
var buckets struct { |
| 326 |
Buckets []struct { |
| 327 |
BucketID, BucketName, BucketType string |
| 328 |
} |
| 329 |
} |
| 330 |
if err := json.NewDecoder(res.Body).Decode(&buckets); err != nil { |
| 331 |
return nil, err |
| 332 |
} |
| 333 |
var r []*BucketInfo |
| 334 |
for _, b := range buckets.Buckets { |
| 335 |
r = append(r, &BucketInfo{ |
| 336 |
Bucket: Bucket{ |
| 337 |
ID: b.BucketID, |
| 338 |
c: c, |
| 339 |
}, |
| 340 |
Name: b.BucketName, |
| 341 |
Type: b.BucketType, |
| 342 |
}) |
| 343 |
} |
| 344 |
return r, nil |
| 345 |
} |
| 346 |
|
| 347 |
// CreateBucket creates a bucket with b2_create_bucket. If allPublic is true, |
| 348 |
// files in this bucket can be downloaded by anybody. |
| 349 |
func (c *Client) CreateBucket(name string, allPublic bool) (*BucketInfo, error) { |
| 350 |
bucketType := "allPrivate" |
| 351 |
if allPublic { |
| 352 |
bucketType = "allPublic" |
| 353 |
} |
| 354 |
res, err := c.doRequest("b2_create_bucket", map[string]interface{}{ |
| 355 |
"accountId": c.accountID, |
| 356 |
"bucketName": name, |
| 357 |
"bucketType": bucketType, |
| 358 |
}) |
| 359 |
if err != nil { |
| 360 |
return nil, err |
| 361 |
} |
| 362 |
defer drainAndClose(res.Body) |
| 363 |
var bucket struct { |
| 364 |
BucketID string |
| 365 |
} |
| 366 |
if err := json.NewDecoder(res.Body).Decode(&bucket); err != nil { |
| 367 |
return nil, err |
| 368 |
} |
| 369 |
return &BucketInfo{ |
| 370 |
Bucket: Bucket{ |
| 371 |
c: c, ID: bucket.BucketID, |
| 372 |
}, |
| 373 |
Name: name, |
| 374 |
Type: bucketType, |
| 375 |
}, nil |
| 376 |
} |
| 377 |
|
| 378 |
// Delete calls b2_delete_bucket. After this call succeeds the Bucket object |
| 379 |
// becomes invalid and any other calls will fail. |
| 380 |
func (b *Bucket) Delete() error { |
| 381 |
res, err := b.c.doRequest("b2_delete_bucket", map[string]interface{}{ |
| 382 |
"accountId": b.c.accountID, |
| 383 |
"bucketId": b.ID, |
| 384 |
}) |
| 385 |
if err != nil { |
| 386 |
return err |
| 387 |
} |
| 388 |
drainAndClose(res.Body) |
| 389 |
return nil |
| 390 |
} |