| 1 |
// Copyright 2019 The Go Cloud Development Kit Authors |
| 2 |
// |
| 3 |
// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 |
// you may not use this file except in compliance with the License. |
| 5 |
// You may obtain a copy of the License at |
| 6 |
// |
| 7 |
// https://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
// |
| 9 |
// Unless required by applicable law or agreed to in writing, software |
| 10 |
// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 |
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 |
// See the License for the specific language governing permissions and |
| 13 |
// limitations under the License. |
| 14 |
|
| 15 |
// Package mongodocstore provides a docstore implementation for MongoDB |
| 16 |
// and MongoDB-compatible services hosted on-premise or by cloud providers, |
| 17 |
// including Amazon DocumentDB and Azure Cosmos DB. |
| 18 |
// |
| 19 |
// # URLs |
| 20 |
// |
| 21 |
// For docstore.OpenCollection, mongodocstore registers for the scheme "mongo". |
| 22 |
// The default URL opener will dial a Mongo server using the environment |
| 23 |
// variable "MONGO_SERVER_URL". |
| 24 |
// To customize the URL opener, or for more details on the URL format, |
| 25 |
// see URLOpener. |
| 26 |
// See https://gocloud.dev/concepts/urls/ for background information. |
| 27 |
// |
| 28 |
// # Action Lists |
| 29 |
// |
| 30 |
// mongodocstore uses the unordered BulkWrite call of the underlying driver for writes, and uses Find with a list of document IDs for Get. |
| 31 |
// (These implementation choices are subject to change.) |
| 32 |
// It calls the BeforeDo function once before each call to the underlying driver. The as function passed |
| 33 |
// to the BeforeDo function exposes the following types: |
| 34 |
// - Gets: *options.FindOptions |
| 35 |
// - writes: []mongo.WriteModel and *options.BulkWriteOptions |
| 36 |
// |
| 37 |
// # As |
| 38 |
// |
| 39 |
// mongodocstore exposes the following types for As: |
| 40 |
// - Collection: *mongo.Collection |
| 41 |
// - Query.BeforeQuery: *options.FindOptions or bson.D (the filter for Delete and Update queries) |
| 42 |
// - DocumentIterator: *mongo.Cursor |
| 43 |
// - Error: mongo.CommandError, mongo.BulkWriteError, mongo.BulkWriteException |
| 44 |
// |
| 45 |
// # Special Considerations |
| 46 |
// |
| 47 |
// MongoDB represents times to millisecond precision, while Go's time.Time type has |
| 48 |
// nanosecond precision. To save time.Times to MongoDB without loss of precision, |
| 49 |
// save the result of calling UnixNano on the time. |
| 50 |
// |
| 51 |
// The official Go driver for MongoDB, go.mongodb.org/mongo-driver/mongo, lowercases |
| 52 |
// struct field names; other docstore drivers do not. This means that you have to choose |
| 53 |
// between interoperating with the MongoDB driver and interoperating with other docstore drivers. |
| 54 |
// See Options.LowercaseFields for more information. |
| 55 |
package mongodocstore // import "gocloud.dev/docstore/mongodocstore" |
| 56 |
|
| 57 |
// MongoDB reference manual: https://docs.mongodb.com/manual |
| 58 |
// Client documentation: https://godoc.org/go.mongodb.org/mongo-driver/mongo |
| 59 |
// |
| 60 |
// The client methods accept a document of type interface{}, |
| 61 |
// which is marshaled by the go.mongodb.org/mongo-driver/bson package. |
| 62 |
|
| 63 |
import ( |
| 64 |
"context" |
| 65 |
"reflect" |
| 66 |
"strings" |
| 67 |
|
| 68 |
"github.com/google/wire" |
| 69 |
"go.mongodb.org/mongo-driver/bson" |
| 70 |
"go.mongodb.org/mongo-driver/bson/primitive" |
| 71 |
"go.mongodb.org/mongo-driver/mongo" |
| 72 |
"go.mongodb.org/mongo-driver/mongo/options" |
| 73 |
"gocloud.dev/docstore" |
| 74 |
"gocloud.dev/docstore/driver" |
| 75 |
"gocloud.dev/gcerrors" |
| 76 |
"gocloud.dev/internal/gcerr" |
| 77 |
) |
| 78 |
|
| 79 |
// Dial returns a new mongoDB client that is connected to the server URI. |
| 80 |
func Dial(ctx context.Context, uri string) (*mongo.Client, error) { |
| 81 |
opts := options.Client().ApplyURI(uri) |
| 82 |
if err := opts.Validate(); err != nil { |
| 83 |
return nil, err |
| 84 |
} |
| 85 |
client, err := mongo.NewClient(opts) |
| 86 |
if err != nil { |
| 87 |
return nil, err |
| 88 |
} |
| 89 |
if err := client.Connect(ctx); err != nil { |
| 90 |
return nil, err |
| 91 |
} |
| 92 |
return client, nil |
| 93 |
} |
| 94 |
|
| 95 |
// Set holds Wire providers for this package. |
| 96 |
var Set = wire.NewSet( |
| 97 |
Dial, |
| 98 |
wire.Struct(new(URLOpener), "Client"), |
| 99 |
) |
| 100 |
|
| 101 |
type collection struct { |
| 102 |
coll *mongo.Collection |
| 103 |
idField string |
| 104 |
idFunc func(docstore.Document) interface{} |
| 105 |
revisionField string |
| 106 |
opts *Options |
| 107 |
} |
| 108 |
|
| 109 |
// Options holds various options. |
| 110 |
type Options struct { |
| 111 |
// Lowercase all field names for document encoding, field selection, update |
| 112 |
// modifications and queries. |
| 113 |
// |
| 114 |
// If false (the default), then struct fields and MongoDB document fields will |
| 115 |
// have the same names. For example, a struct field F will correspond to a |
| 116 |
// MongoDB document field "F". This setting matches the behavior of other |
| 117 |
// docstore drivers, making code portable across services. |
| 118 |
// |
| 119 |
// If true, all fields correspond to lower-cased MongoDB document fields. The |
| 120 |
// field name F will correspond to the MongoDB document field "f", for |
| 121 |
// instance. Use this to make code that uses this package interoperate with |
| 122 |
// code that uses the official Go client for MongoDB, |
| 123 |
// go.mongodb.org/mongo-driver/mongo, which lowercases field names. |
| 124 |
LowercaseFields bool |
| 125 |
// The name of the field holding the document revision. |
| 126 |
// Defaults to docstore.DefaultRevisionField. |
| 127 |
RevisionField string |
| 128 |
// Whether Query.Update writes a new revision into the updated documents. |
| 129 |
// The default is false, meaning that a revision will be written to all |
| 130 |
// documents that satisfy the query's conditions. Set to true if and only if |
| 131 |
// the collection's documents do not have revision fields. |
| 132 |
NoWriteQueryUpdateRevisions bool |
| 133 |
} |
| 134 |
|
| 135 |
// OpenCollection opens a MongoDB collection for use with Docstore. |
| 136 |
// The idField argument is the name of the document field to use for the document ID |
| 137 |
// (MongoDB's _id field). If it is empty, the field "_id" will be used. |
| 138 |
func OpenCollection(mcoll *mongo.Collection, idField string, opts *Options) (*docstore.Collection, error) { |
| 139 |
dc, err := newCollection(mcoll, idField, nil, opts) |
| 140 |
if err != nil { |
| 141 |
return nil, err |
| 142 |
} |
| 143 |
return docstore.NewCollection(dc), nil |
| 144 |
} |
| 145 |
|
| 146 |
// OpenCollectionWithIDFunc opens a MongoDB collection for use with Docstore. |
| 147 |
// The idFunc argument is function that accepts a document and returns the value to |
| 148 |
// be used for the document ID (MongoDB's _id field). IDFunc should return nil if the |
| 149 |
// document is missing the information to construct an ID. This will cause all |
| 150 |
// actions, even Create, to fail. |
| 151 |
func OpenCollectionWithIDFunc(mcoll *mongo.Collection, idFunc func(docstore.Document) interface{}, opts *Options) (*docstore.Collection, error) { |
| 152 |
dc, err := newCollection(mcoll, "", idFunc, opts) |
| 153 |
if err != nil { |
| 154 |
return nil, err |
| 155 |
} |
| 156 |
return docstore.NewCollection(dc), nil |
| 157 |
} |
| 158 |
|
| 159 |
func newCollection(mcoll *mongo.Collection, idField string, idFunc func(docstore.Document) interface{}, opts *Options) (*collection, error) { |
| 160 |
if opts == nil { |
| 161 |
opts = &Options{} |
| 162 |
} |
| 163 |
if opts.RevisionField == "" { |
| 164 |
opts.RevisionField = docstore.DefaultRevisionField |
| 165 |
} |
| 166 |
c := &collection{ |
| 167 |
coll: mcoll, |
| 168 |
idField: idField, |
| 169 |
idFunc: idFunc, |
| 170 |
revisionField: opts.RevisionField, |
| 171 |
opts: opts, |
| 172 |
} |
| 173 |
if c.idField == "" && c.idFunc == nil { |
| 174 |
c.idField = mongoIDField |
| 175 |
} |
| 176 |
|
| 177 |
if opts.LowercaseFields { |
| 178 |
c.idField = strings.ToLower(c.idField) |
| 179 |
c.revisionField = strings.ToLower(c.revisionField) |
| 180 |
} |
| 181 |
return c, nil |
| 182 |
} |
| 183 |
|
| 184 |
func (c *collection) Key(doc driver.Document) (interface{}, error) { |
| 185 |
if c.idField != "" { |
| 186 |
id, _ := doc.GetField(c.idField) |
| 187 |
return id, nil // missing field is not an error |
| 188 |
} |
| 189 |
id := c.idFunc(doc.Origin) |
| 190 |
if id == nil || driver.IsEmptyValue(reflect.ValueOf(id)) { |
| 191 |
return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "missing document key") |
| 192 |
} |
| 193 |
return id, nil |
| 194 |
} |
| 195 |
|
| 196 |
func (c *collection) RevisionField() string { |
| 197 |
return c.opts.RevisionField |
| 198 |
} |
| 199 |
|
| 200 |
// From https://docs.mongodb.com/manual/core/document: "The field name _id is |
| 201 |
// reserved for use as a primary key; its value must be unique in the collection, is |
| 202 |
// immutable, and may be of any type other than an array." |
| 203 |
const mongoIDField = "_id" |
| 204 |
|
| 205 |
func (c *collection) RunActions(ctx context.Context, actions []*driver.Action, opts *driver.RunActionsOptions) driver.ActionListError { |
| 206 |
errs := make([]error, len(actions)) |
| 207 |
beforeGets, gets, writes, writesTx, afterGets := driver.GroupActions(actions) |
| 208 |
c.runGets(ctx, beforeGets, errs, opts) |
| 209 |
ch := make(chan []error) |
| 210 |
go func() { ch <- c.bulkWrite(ctx, writes, errs, opts) }() |
| 211 |
ch2 := make(chan []error) |
| 212 |
go func() { ch2 <- c.txWrite(ctx, writesTx, errs, opts) }() |
| 213 |
c.runGets(ctx, gets, errs, opts) |
| 214 |
writeErrs := <-ch |
| 215 |
<-ch2 |
| 216 |
c.runGets(ctx, afterGets, errs, opts) |
| 217 |
alerr := driver.NewActionListError(errs) |
| 218 |
for _, werr := range writeErrs { |
| 219 |
alerr = append(alerr, indexedError{-1, werr}) |
| 220 |
} |
| 221 |
return alerr |
| 222 |
} |
| 223 |
|
| 224 |
type indexedError = struct { |
| 225 |
Index int |
| 226 |
Err error |
| 227 |
} |
| 228 |
|
| 229 |
func (c *collection) runGets(ctx context.Context, gets []*driver.Action, errs []error, opts *driver.RunActionsOptions) { |
| 230 |
// TODO(shantuo): figure out a reasonable batch size, there is no hard limit on |
| 231 |
// the item number or filter string length. The limit for bulk write batch size |
| 232 |
// is 100,000. |
| 233 |
for _, group := range driver.GroupByFieldPath(gets) { |
| 234 |
c.bulkFind(ctx, group, errs, opts) |
| 235 |
} |
| 236 |
} |
| 237 |
|
| 238 |
func (c *collection) bulkFind(ctx context.Context, gets []*driver.Action, errs []error, dopts *driver.RunActionsOptions) { |
| 239 |
// errors need to be mapped to the actions' indices. |
| 240 |
setErr := func(err error) { |
| 241 |
for _, get := range gets { |
| 242 |
if errs[get.Index] == nil { |
| 243 |
errs[get.Index] = err |
| 244 |
} |
| 245 |
} |
| 246 |
} |
| 247 |
|
| 248 |
opts := options.Find() |
| 249 |
if len(gets[0].FieldPaths) > 0 { |
| 250 |
opts.Projection = c.projectionDoc(gets[0].FieldPaths) |
| 251 |
} |
| 252 |
ids := bson.A{} |
| 253 |
idToAction := map[interface{}]*driver.Action{} |
| 254 |
for _, a := range gets { |
| 255 |
id, err := encodeValue(a.Key) |
| 256 |
if err != nil { |
| 257 |
errs[a.Index] = err |
| 258 |
} else { |
| 259 |
ids = append(ids, id) |
| 260 |
idToAction[id] = a |
| 261 |
} |
| 262 |
} |
| 263 |
if dopts.BeforeDo != nil { |
| 264 |
if err := dopts.BeforeDo(driver.AsFunc(opts)); err != nil { |
| 265 |
setErr(err) |
| 266 |
return |
| 267 |
} |
| 268 |
} |
| 269 |
cursor, err := c.coll.Find(ctx, bson.D{bson.E{Key: mongoIDField, Value: bson.D{{Key: "$in", Value: ids}}}}, opts) |
| 270 |
if err != nil { |
| 271 |
setErr(err) |
| 272 |
return |
| 273 |
} |
| 274 |
defer cursor.Close(ctx) |
| 275 |
|
| 276 |
found := make(map[*driver.Action]bool) |
| 277 |
for cursor.Next(ctx) { |
| 278 |
var m map[string]interface{} |
| 279 |
if err := cursor.Decode(&m); err != nil { |
| 280 |
continue |
| 281 |
} |
| 282 |
a := idToAction[m[mongoIDField]] |
| 283 |
errs[a.Index] = decodeDoc(m, a.Doc, c.idField, c.opts.LowercaseFields) |
| 284 |
found[a] = true |
| 285 |
} |
| 286 |
for _, a := range gets { |
| 287 |
if !found[a] { |
| 288 |
errs[a.Index] = gcerr.Newf(gcerr.NotFound, nil, "item with key %v not found", a.Key) |
| 289 |
} |
| 290 |
} |
| 291 |
} |
| 292 |
|
| 293 |
// Construct a mongo "projection document" from field paths. |
| 294 |
// Always include the revision field. |
| 295 |
func (c *collection) projectionDoc(fps [][]string) bson.D { |
| 296 |
proj := bson.D{{Key: c.revisionField, Value: 1}} |
| 297 |
for _, fp := range fps { |
| 298 |
path := c.toMongoFieldPath(fp) |
| 299 |
if path != c.revisionField { |
| 300 |
proj = append(proj, bson.E{Key: path, Value: 1}) |
| 301 |
} |
| 302 |
} |
| 303 |
return proj |
| 304 |
} |
| 305 |
|
| 306 |
func (c *collection) toMongoFieldPath(fp []string) string { |
| 307 |
if c.opts.LowercaseFields { |
| 308 |
sliceToLower(fp) |
| 309 |
} |
| 310 |
return strings.Join(fp, ".") |
| 311 |
} |
| 312 |
|
| 313 |
func sliceToLower(s []string) { |
| 314 |
for i, e := range s { |
| 315 |
s[i] = strings.ToLower(e) |
| 316 |
} |
| 317 |
} |
| 318 |
|
| 319 |
func (c *collection) prepareCreate(a *driver.Action) (mdoc, createdID interface{}, rev string, err error) { |
| 320 |
id := a.Key |
| 321 |
if id == nil { |
| 322 |
// Create a unique ID here. (The MongoDB Go client does this for us when calling InsertOne, |
| 323 |
// but not for BulkWrite.) |
| 324 |
id = primitive.NewObjectID() |
| 325 |
createdID = id |
| 326 |
} else { |
| 327 |
id, err = encodeValue(id) |
| 328 |
if err != nil { |
| 329 |
return nil, nil, "", err |
| 330 |
} |
| 331 |
} |
| 332 |
mdoc, rev, err = c.encodeDoc(a.Doc, id) |
| 333 |
if err != nil { |
| 334 |
return nil, nil, "", err |
| 335 |
} |
| 336 |
return mdoc, createdID, rev, nil |
| 337 |
} |
| 338 |
|
| 339 |
func (c *collection) prepareReplace(a *driver.Action) (filter bson.D, mdoc map[string]interface{}, rev string, err error) { |
| 340 |
id, err := encodeValue(a.Key) |
| 341 |
if err != nil { |
| 342 |
return nil, nil, "", err |
| 343 |
} |
| 344 |
filter, _, err = c.makeFilter(id, a.Doc) |
| 345 |
if err != nil { |
| 346 |
return nil, nil, "", err |
| 347 |
} |
| 348 |
mdoc, rev, err = c.encodeDoc(a.Doc, id) |
| 349 |
if err != nil { |
| 350 |
return nil, nil, "", err |
| 351 |
} |
| 352 |
return filter, mdoc, rev, nil |
| 353 |
} |
| 354 |
|
| 355 |
// encodeDoc encodes doc and sets its ID to the encoded value id. It also creates a new revision and sets it. |
| 356 |
// It returns the encoded document and the new revision. |
| 357 |
func (c *collection) encodeDoc(doc driver.Document, id interface{}) (map[string]interface{}, string, error) { |
| 358 |
mdoc, err := encodeDoc(doc, c.opts.LowercaseFields) |
| 359 |
if err != nil { |
| 360 |
return nil, "", err |
| 361 |
} |
| 362 |
if id != nil { |
| 363 |
if c.idField != "" { |
| 364 |
delete(mdoc, c.idField) |
| 365 |
} |
| 366 |
mdoc[mongoIDField] = id |
| 367 |
} |
| 368 |
var rev string |
| 369 |
if c.hasField(doc, c.revisionField) { |
| 370 |
rev = driver.UniqueString() |
| 371 |
mdoc[c.revisionField] = rev |
| 372 |
} |
| 373 |
return mdoc, rev, nil |
| 374 |
} |
| 375 |
|
| 376 |
func (c *collection) prepareUpdate(a *driver.Action) (filter bson.D, updateDoc map[string]bson.D, rev string, err error) { |
| 377 |
id, err := encodeValue(a.Key) |
| 378 |
if err != nil { |
| 379 |
return nil, nil, "", err |
| 380 |
} |
| 381 |
filter, _, err = c.makeFilter(id, a.Doc) |
| 382 |
if err != nil { |
| 383 |
return nil, nil, "", err |
| 384 |
} |
| 385 |
updateDoc, rev, err = c.newUpdateDoc(a.Mods, c.hasField(a.Doc, c.revisionField)) |
| 386 |
if err != nil { |
| 387 |
return nil, nil, "", err |
| 388 |
} |
| 389 |
return filter, updateDoc, rev, nil |
| 390 |
} |
| 391 |
|
| 392 |
func (c *collection) newUpdateDoc(mods []driver.Mod, writeRevision bool) (map[string]bson.D, string, error) { |
| 393 |
var ( |
| 394 |
sets bson.D |
| 395 |
unsets bson.D |
| 396 |
incs bson.D |
| 397 |
) |
| 398 |
for _, m := range mods { |
| 399 |
key := c.toMongoFieldPath(m.FieldPath) |
| 400 |
if m.Value == nil { |
| 401 |
unsets = append(unsets, bson.E{Key: key, Value: ""}) |
| 402 |
} else if inc, ok := m.Value.(driver.IncOp); ok { |
| 403 |
val, err := encodeValue(inc.Amount) |
| 404 |
if err != nil { |
| 405 |
return nil, "", err |
| 406 |
} |
| 407 |
incs = append(incs, bson.E{Key: key, Value: val}) |
| 408 |
} else { |
| 409 |
val, err := encodeValue(m.Value) |
| 410 |
if err != nil { |
| 411 |
return nil, "", err |
| 412 |
} |
| 413 |
sets = append(sets, bson.E{Key: key, Value: val}) |
| 414 |
} |
| 415 |
} |
| 416 |
updateDoc := map[string]bson.D{} |
| 417 |
var rev string |
| 418 |
if writeRevision { |
| 419 |
rev = driver.UniqueString() |
| 420 |
sets = append(sets, bson.E{Key: c.revisionField, Value: rev}) |
| 421 |
} |
| 422 |
if len(sets) > 0 { |
| 423 |
updateDoc["$set"] = sets |
| 424 |
} |
| 425 |
if len(unsets) > 0 { |
| 426 |
updateDoc["$unset"] = unsets |
| 427 |
} |
| 428 |
if len(incs) > 0 { |
| 429 |
updateDoc["$inc"] = incs |
| 430 |
} |
| 431 |
return updateDoc, rev, nil |
| 432 |
} |
| 433 |
|
| 434 |
// makeFilter constructs a filter using the given encoded id and the document's revision field, if any. |
| 435 |
func (c *collection) makeFilter(id interface{}, doc driver.Document) (filter bson.D, rev interface{}, err error) { |
| 436 |
rev, err = doc.GetField(c.revisionField) |
| 437 |
if err != nil && gcerrors.Code(err) != gcerrors.NotFound { |
| 438 |
return nil, nil, err |
| 439 |
} |
| 440 |
// Only select the document with the given ID. |
| 441 |
filter = bson.D{bson.E{Key: "_id", Value: id}} |
| 442 |
// If the given document has a revision, it must match the stored document. |
| 443 |
if rev != nil { |
| 444 |
filter = append(filter, bson.E{Key: c.revisionField, Value: rev}) |
| 445 |
} |
| 446 |
return filter, rev, nil |
| 447 |
} |
| 448 |
|
| 449 |
// bulkWrite calls the Mongo driver's BulkWrite RPC in unordered mode with the |
| 450 |
// actions, which must be writes. |
| 451 |
// errs is the slice of errors indexed by the position of the action in the original |
| 452 |
// action list. bulkWrite populates this slice. In addition, bulkWrite returns a list |
| 453 |
// of errors that cannot be attributed to any single action. |
| 454 |
func (c *collection) bulkWrite(ctx context.Context, actions []*driver.Action, errs []error, dopts *driver.RunActionsOptions) []error { |
| 455 |
var ( |
| 456 |
models []mongo.WriteModel |
| 457 |
modelActions []*driver.Action // corresponding action for each model |
| 458 |
newIDs []interface{} // new IDs for Create actions, corresponding to models slice |
| 459 |
revs []string // new revisions, corresponding to models slice |
| 460 |
nDeletes int64 |
| 461 |
nNonCreateWrite int64 // total operations expected from Put, Replace and Update |
| 462 |
) |
| 463 |
for _, a := range actions { |
| 464 |
var m mongo.WriteModel |
| 465 |
var err error |
| 466 |
var newID interface{} |
| 467 |
var rev string |
| 468 |
switch a.Kind { |
| 469 |
case driver.Create: |
| 470 |
m, newID, rev, err = c.newCreateModel(a) |
| 471 |
case driver.Delete: |
| 472 |
m, err = c.newDeleteModel(a) |
| 473 |
if err == nil { |
| 474 |
nDeletes++ |
| 475 |
} |
| 476 |
case driver.Replace, driver.Put: |
| 477 |
m, rev, err = c.newReplaceModel(a, a.Kind == driver.Put) |
| 478 |
if err == nil { |
| 479 |
nNonCreateWrite++ |
| 480 |
} |
| 481 |
case driver.Update: |
| 482 |
m, rev, err = c.newUpdateModel(a) |
| 483 |
if err == nil && m != nil { |
| 484 |
nNonCreateWrite++ |
| 485 |
} |
| 486 |
default: |
| 487 |
err = gcerr.Newf(gcerr.Internal, nil, "bad action %+v", a) |
| 488 |
} |
| 489 |
if err != nil { |
| 490 |
errs[a.Index] = err |
| 491 |
} else if m != nil { // m can be nil for a no-op update |
| 492 |
models = append(models, m) |
| 493 |
modelActions = append(modelActions, a) |
| 494 |
newIDs = append(newIDs, newID) |
| 495 |
revs = append(revs, rev) |
| 496 |
} |
| 497 |
} |
| 498 |
if len(models) == 0 { |
| 499 |
return nil |
| 500 |
} |
| 501 |
|
| 502 |
bopts := options.BulkWrite().SetOrdered(false) |
| 503 |
if dopts.BeforeDo != nil { |
| 504 |
asFunc := func(target interface{}) bool { |
| 505 |
switch t := target.(type) { |
| 506 |
case *[]mongo.WriteModel: |
| 507 |
*t = models |
| 508 |
case **options.BulkWriteOptions: |
| 509 |
*t = bopts |
| 510 |
default: |
| 511 |
return false |
| 512 |
} |
| 513 |
return true |
| 514 |
} |
| 515 |
if err := dopts.BeforeDo(asFunc); err != nil { |
| 516 |
return []error{err} |
| 517 |
} |
| 518 |
} |
| 519 |
|
| 520 |
// TODO(jba): improve independent execution. I think that even if BulkWrite returns an error, |
| 521 |
// some of the actions may have succeeded. |
| 522 |
var reterrs []error |
| 523 |
res, err := c.coll.BulkWrite(ctx, models, bopts) |
| 524 |
if err != nil { |
| 525 |
bwe, ok := err.(mongo.BulkWriteException) |
| 526 |
if !ok { // assume everything failed with this error |
| 527 |
return []error{err} |
| 528 |
} |
| 529 |
// The returned indexes of the WriteErrors are wrong. See https://jira.mongodb.org/browse/GODRIVER-1028. |
| 530 |
// Until it's fixed, use negative values for the indexes in the errors we return. |
| 531 |
for _, w := range bwe.WriteErrors { |
| 532 |
reterrs = append(reterrs, gcerr.Newf(translateMongoCode(w.Code), w, "%s", w.Message)) |
| 533 |
} |
| 534 |
return reterrs |
| 535 |
} |
| 536 |
for i, newID := range newIDs { |
| 537 |
if newID == nil { |
| 538 |
continue |
| 539 |
} |
| 540 |
a := modelActions[i] |
| 541 |
if err := a.Doc.SetField(c.idField, newID); err != nil { |
| 542 |
errs[a.Index] = err |
| 543 |
} |
| 544 |
} |
| 545 |
for i, rev := range revs { |
| 546 |
a := modelActions[i] |
| 547 |
if rev != "" && c.hasField(a.Doc, c.revisionField) { |
| 548 |
if err := a.Doc.SetField(c.revisionField, rev); err != nil && errs[a.Index] == nil { |
| 549 |
errs[a.Index] = err |
| 550 |
} |
| 551 |
} |
| 552 |
} |
| 553 |
if res.DeletedCount != nDeletes { |
| 554 |
// Some Delete actions failed. It's not an error if a Delete failed because |
| 555 |
// the document didn't exist, but it is an error if it failed because of a |
| 556 |
// precondition mismatch. Find all the documents with revisions we tried to delete; if |
| 557 |
// any are still present, that's an error. |
| 558 |
c.determineDeleteErrors(ctx, models, modelActions, errs) |
| 559 |
} |
| 560 |
if res.MatchedCount+res.UpsertedCount != nNonCreateWrite { |
| 561 |
reterrs = append(reterrs, gcerr.Newf(gcerr.NotFound, nil, "some writes failed (replaced %d, upserted %d, out of total %d)", res.MatchedCount, res.UpsertedCount, nNonCreateWrite)) |
| 562 |
} |
| 563 |
return reterrs |
| 564 |
} |
| 565 |
|
| 566 |
func (c *collection) txWrite(ctx context.Context, actions []*driver.Action, errs []error, dopts *driver.RunActionsOptions) []error { |
| 567 |
var ( |
| 568 |
models []mongo.WriteModel |
| 569 |
modelActions []*driver.Action // corresponding action for each model |
| 570 |
newIDs []interface{} // new IDs for Create actions, corresponding to models slice |
| 571 |
revs []string // new revisions, corresponding to models slice |
| 572 |
nDeletes int64 |
| 573 |
nNonCreateWrite int64 // total operations expected from Put, Replace and Update |
| 574 |
) |
| 575 |
|
| 576 |
// all actions will fail atomically even if a single action fails |
| 577 |
setErr := func(err error) { |
| 578 |
for _, a := range actions { |
| 579 |
errs[a.Index] = err |
| 580 |
} |
| 581 |
} |
| 582 |
|
| 583 |
for _, a := range actions { |
| 584 |
var m mongo.WriteModel |
| 585 |
var err error |
| 586 |
var newID interface{} |
| 587 |
var rev string |
| 588 |
switch a.Kind { |
| 589 |
case driver.Create: |
| 590 |
m, newID, rev, err = c.newCreateModel(a) |
| 591 |
case driver.Delete: |
| 592 |
m, err = c.newDeleteModel(a) |
| 593 |
if err == nil { |
| 594 |
nDeletes++ |
| 595 |
} |
| 596 |
case driver.Replace, driver.Put: |
| 597 |
m, rev, err = c.newReplaceModel(a, a.Kind == driver.Put) |
| 598 |
if err == nil { |
| 599 |
nNonCreateWrite++ |
| 600 |
} |
| 601 |
case driver.Update: |
| 602 |
m, rev, err = c.newUpdateModel(a) |
| 603 |
if err == nil && m != nil { |
| 604 |
nNonCreateWrite++ |
| 605 |
} |
| 606 |
default: |
| 607 |
err = gcerr.Newf(gcerr.Internal, nil, "bad action %+v", a) |
| 608 |
} |
| 609 |
if err != nil { |
| 610 |
setErr(err) |
| 611 |
return nil |
| 612 |
} else if m != nil { // m can be nil for a no-op update |
| 613 |
models = append(models, m) |
| 614 |
modelActions = append(modelActions, a) |
| 615 |
newIDs = append(newIDs, newID) |
| 616 |
revs = append(revs, rev) |
| 617 |
} |
| 618 |
} |
| 619 |
if len(models) == 0 { |
| 620 |
return nil |
| 621 |
} |
| 622 |
|
| 623 |
bopts := options.BulkWrite().SetOrdered(true) |
| 624 |
if dopts.BeforeDo != nil { |
| 625 |
asFunc := func(target interface{}) bool { |
| 626 |
switch t := target.(type) { |
| 627 |
case *[]mongo.WriteModel: |
| 628 |
*t = models |
| 629 |
case **options.BulkWriteOptions: |
| 630 |
*t = bopts |
| 631 |
default: |
| 632 |
return false |
| 633 |
} |
| 634 |
return true |
| 635 |
} |
| 636 |
if err := dopts.BeforeDo(asFunc); err != nil { |
| 637 |
return []error{err} |
| 638 |
} |
| 639 |
} |
| 640 |
|
| 641 |
client := c.coll.Database().Client() |
| 642 |
session, err := client.StartSession() |
| 643 |
if err != nil { |
| 644 |
setErr(err) |
| 645 |
return nil |
| 646 |
} |
| 647 |
defer session.EndSession(ctx) |
| 648 |
|
| 649 |
callback := func(sessionCtx mongo.SessionContext) error { |
| 650 |
if err := session.StartTransaction(); err != nil { |
| 651 |
return err |
| 652 |
} |
| 653 |
|
| 654 |
res, err := c.coll.BulkWrite(sessionCtx, models, bopts) |
| 655 |
if res.DeletedCount != nDeletes { |
| 656 |
// Some Delete actions failed. It's not an error if a Delete failed because |
| 657 |
// the document didn't exist, but it is an error if it failed because of a |
| 658 |
// precondition mismatch. Find all the documents with revisions we tried to delete; if |
| 659 |
// any are still present, that's an error. |
| 660 |
if c.determineDeleteErrors(ctx, models, modelActions, errs) { |
| 661 |
setErr(gcerr.Newf(gcerr.FailedPrecondition, nil, |
| 662 |
"wrong revision for document to be deleted")) |
| 663 |
} |
| 664 |
} |
| 665 |
if res.MatchedCount+res.UpsertedCount != nNonCreateWrite { |
| 666 |
err = gcerr.Newf(gcerr.NotFound, nil, "some writes failed (replaced %d, upserted %d, out of total %d)", res.MatchedCount, res.UpsertedCount, nNonCreateWrite) |
| 667 |
} |
| 668 |
if err != nil { |
| 669 |
abortTxErr := session.AbortTransaction(context.Background()) |
| 670 |
if abortTxErr != nil { |
| 671 |
return err |
| 672 |
} |
| 673 |
return err |
| 674 |
} |
| 675 |
|
| 676 |
if err = session.CommitTransaction(sessionCtx); err != nil { |
| 677 |
return err |
| 678 |
} |
| 679 |
|
| 680 |
return nil |
| 681 |
} |
| 682 |
|
| 683 |
err = mongo.WithSession(ctx, session, callback) |
| 684 |
|
| 685 |
if err != nil { |
| 686 |
setErr(err) |
| 687 |
return nil |
| 688 |
} |
| 689 |
return nil |
| 690 |
} |
| 691 |
|
| 692 |
// determineDeleteErrors find the errors for the delete and return true if found any |
| 693 |
func (c *collection) determineDeleteErrors(ctx context.Context, models []mongo.WriteModel, actions []*driver.Action, errs []error) bool { |
| 694 |
// TODO(jba): do this concurrently. |
| 695 |
foundErr := false |
| 696 |
for i, m := range models { |
| 697 |
if dm, ok := m.(*mongo.DeleteOneModel); ok { |
| 698 |
filter := dm.Filter.(bson.D) |
| 699 |
if len(filter) > 1 { |
| 700 |
// Delete with both ID and revision. See if the document is still there. |
| 701 |
idOnlyFilter := filter[:1] |
| 702 |
// TODO(shantuo): use Find instead of FindOne. |
| 703 |
res := c.coll.FindOne(ctx, idOnlyFilter) |
| 704 |
|
| 705 |
// Assume an error means the document wasn't found. |
| 706 |
// That means either that it was deleted successfully, or that it never |
| 707 |
// existed. Either way, it's not an error. |
| 708 |
// TODO(jba): distinguish between not found and other errors. |
| 709 |
if res.Err() == nil { |
| 710 |
// The document exists, but we didn't delete it: assume we had the wrong |
| 711 |
// revision. |
| 712 |
errs[actions[i].Index] = gcerr.Newf(gcerr.FailedPrecondition, nil, |
| 713 |
"wrong revision for document with ID %v", actions[i].Key) |
| 714 |
foundErr = true |
| 715 |
} |
| 716 |
} |
| 717 |
} |
| 718 |
} |
| 719 |
return foundErr |
| 720 |
} |
| 721 |
|
| 722 |
func (c *collection) newCreateModel(a *driver.Action) (*mongo.InsertOneModel, interface{}, string, error) { |
| 723 |
mdoc, createdID, rev, err := c.prepareCreate(a) |
| 724 |
if err != nil { |
| 725 |
return nil, nil, "", err |
| 726 |
} |
| 727 |
return &mongo.InsertOneModel{Document: mdoc}, createdID, rev, nil |
| 728 |
} |
| 729 |
|
| 730 |
func (c *collection) newDeleteModel(a *driver.Action) (*mongo.DeleteOneModel, error) { |
| 731 |
id, err := encodeValue(a.Key) |
| 732 |
if err != nil { |
| 733 |
return nil, err |
| 734 |
} |
| 735 |
filter, _, err := c.makeFilter(id, a.Doc) |
| 736 |
if err != nil { |
| 737 |
return nil, err |
| 738 |
} |
| 739 |
return &mongo.DeleteOneModel{Filter: filter}, nil |
| 740 |
} |
| 741 |
|
| 742 |
func (c *collection) newReplaceModel(a *driver.Action, upsert bool) (*mongo.ReplaceOneModel, string, error) { |
| 743 |
filter, mdoc, rev, err := c.prepareReplace(a) |
| 744 |
if err != nil { |
| 745 |
return nil, "", err |
| 746 |
} |
| 747 |
return &mongo.ReplaceOneModel{ |
| 748 |
Filter: filter, |
| 749 |
Replacement: mdoc, |
| 750 |
Upsert: &upsert, |
| 751 |
}, rev, nil |
| 752 |
} |
| 753 |
|
| 754 |
func (c *collection) newUpdateModel(a *driver.Action) (*mongo.UpdateOneModel, string, error) { |
| 755 |
filter, updateDoc, rev, err := c.prepareUpdate(a) |
| 756 |
if err != nil { |
| 757 |
return nil, "", err |
| 758 |
} |
| 759 |
if filter == nil { // no-op |
| 760 |
return nil, "", nil |
| 761 |
} |
| 762 |
return &mongo.UpdateOneModel{Filter: filter, Update: updateDoc}, rev, nil |
| 763 |
} |
| 764 |
|
| 765 |
// RevisionToBytes implements driver.RevisionToBytes. |
| 766 |
func (c *collection) RevisionToBytes(rev interface{}) ([]byte, error) { |
| 767 |
s, ok := rev.(string) |
| 768 |
if !ok { |
| 769 |
return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "revision %v of type %[1]T is not a string", rev) |
| 770 |
} |
| 771 |
return []byte(s), nil |
| 772 |
} |
| 773 |
|
| 774 |
func (c *collection) hasField(doc driver.Document, field string) bool { |
| 775 |
if c.opts.LowercaseFields { |
| 776 |
return doc.HasFieldFold(field) |
| 777 |
} |
| 778 |
return doc.HasField(field) |
| 779 |
} |
| 780 |
|
| 781 |
// BytesToRevision implements driver.BytesToRevision. |
| 782 |
func (c *collection) BytesToRevision(b []byte) (interface{}, error) { |
| 783 |
return string(b), nil |
| 784 |
} |
| 785 |
|
| 786 |
// As implements driver.As. |
| 787 |
func (c *collection) As(i interface{}) bool { |
| 788 |
p, ok := i.(**mongo.Collection) |
| 789 |
if !ok { |
| 790 |
return false |
| 791 |
} |
| 792 |
*p = c.coll |
| 793 |
return true |
| 794 |
} |
| 795 |
|
| 796 |
// ErrorAs implements driver.Collection.ErrorAs |
| 797 |
func (c *collection) ErrorAs(err error, i interface{}) bool { |
| 798 |
switch e := err.(type) { |
| 799 |
case mongo.CommandError: |
| 800 |
if p, ok := i.(*mongo.CommandError); ok { |
| 801 |
*p = e |
| 802 |
return true |
| 803 |
} |
| 804 |
case mongo.BulkWriteError: |
| 805 |
if p, ok := i.(*mongo.BulkWriteError); ok { |
| 806 |
*p = e |
| 807 |
return true |
| 808 |
} |
| 809 |
case mongo.BulkWriteException: |
| 810 |
if p, ok := i.(*mongo.BulkWriteException); ok { |
| 811 |
*p = e |
| 812 |
return true |
| 813 |
} |
| 814 |
} |
| 815 |
return false |
| 816 |
} |
| 817 |
|
| 818 |
// ErrorCode implements driver.Collection.ErrorCode. |
| 819 |
func (c *collection) ErrorCode(err error) gcerrors.ErrorCode { |
| 820 |
if g, ok := err.(*gcerr.Error); ok { |
| 821 |
return g.Code |
| 822 |
} |
| 823 |
if err == mongo.ErrNoDocuments { |
| 824 |
return gcerrors.NotFound |
| 825 |
} |
| 826 |
if wexc, ok := err.(mongo.WriteException); ok && len(wexc.WriteErrors) > 0 { |
| 827 |
return translateMongoCode(wexc.WriteErrors[0].Code) |
| 828 |
} |
| 829 |
return gcerrors.Unknown |
| 830 |
} |
| 831 |
|
| 832 |
// Close implements driver.Collection.Close. |
| 833 |
func (c *collection) Close() error { return nil } |
| 834 |
|
| 835 |
// Error code for a write error when no documents match a filter. |
| 836 |
// (The Go mongo driver doesn't define an exported constant for this.) |
| 837 |
const mongoDupKeyCode = 11000 |
| 838 |
|
| 839 |
func translateMongoCode(code int) gcerrors.ErrorCode { |
| 840 |
switch code { |
| 841 |
case mongoDupKeyCode: |
| 842 |
return gcerrors.AlreadyExists |
| 843 |
default: |
| 844 |
return gcerrors.Unknown |
| 845 |
} |
| 846 |
} |