| 1 |
// Copyright 2012-present Oliver Eilhard. All rights reserved. |
| 2 |
// Use of this source code is governed by a MIT-license. |
| 3 |
// See http://olivere.mit-license.org/license.txt for details. |
| 4 |
|
| 5 |
package elastic |
| 6 |
|
| 7 |
import ( |
| 8 |
"bytes" |
| 9 |
"context" |
| 10 |
"errors" |
| 11 |
"fmt" |
| 12 |
"net/url" |
| 13 |
|
| 14 |
"gopkg.in/olivere/elastic.v5/uritemplates" |
| 15 |
) |
| 16 |
|
| 17 |
// BulkService allows for batching bulk requests and sending them to |
| 18 |
// Elasticsearch in one roundtrip. Use the Add method with BulkIndexRequest, |
| 19 |
// BulkUpdateRequest, and BulkDeleteRequest to add bulk requests to a batch, |
| 20 |
// then use Do to send them to Elasticsearch. |
| 21 |
// |
| 22 |
// BulkService will be reset after each Do call. In other words, you can |
| 23 |
// reuse BulkService to send many batches. You do not have to create a new |
| 24 |
// BulkService for each batch. |
| 25 |
// |
| 26 |
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-bulk.html |
| 27 |
// for more details. |
| 28 |
type BulkService struct { |
| 29 |
client *Client |
| 30 |
retrier Retrier |
| 31 |
|
| 32 |
index string |
| 33 |
typ string |
| 34 |
requests []BulkableRequest |
| 35 |
pipeline string |
| 36 |
timeout string |
| 37 |
refresh string |
| 38 |
routing string |
| 39 |
waitForActiveShards string |
| 40 |
pretty bool |
| 41 |
|
| 42 |
// estimated bulk size in bytes, up to the request index sizeInBytesCursor |
| 43 |
sizeInBytes int64 |
| 44 |
sizeInBytesCursor int |
| 45 |
} |
| 46 |
|
| 47 |
// NewBulkService initializes a new BulkService. |
| 48 |
func NewBulkService(client *Client) *BulkService { |
| 49 |
builder := &BulkService{ |
| 50 |
client: client, |
| 51 |
} |
| 52 |
return builder |
| 53 |
} |
| 54 |
|
| 55 |
// Reset cleans up the request queue |
| 56 |
func (s *BulkService) Reset() { |
| 57 |
s.requests = make([]BulkableRequest, 0) |
| 58 |
s.sizeInBytes = 0 |
| 59 |
s.sizeInBytesCursor = 0 |
| 60 |
} |
| 61 |
|
| 62 |
// Retrier allows to set specific retry logic for this BulkService. |
| 63 |
// If not specified, it will use the client's default retrier. |
| 64 |
func (s *BulkService) Retrier(retrier Retrier) *BulkService { |
| 65 |
s.retrier = retrier |
| 66 |
return s |
| 67 |
} |
| 68 |
|
| 69 |
// Index specifies the index to use for all batches. You may also leave |
| 70 |
// this blank and specify the index in the individual bulk requests. |
| 71 |
func (s *BulkService) Index(index string) *BulkService { |
| 72 |
s.index = index |
| 73 |
return s |
| 74 |
} |
| 75 |
|
| 76 |
// Type specifies the type to use for all batches. You may also leave |
| 77 |
// this blank and specify the type in the individual bulk requests. |
| 78 |
func (s *BulkService) Type(typ string) *BulkService { |
| 79 |
s.typ = typ |
| 80 |
return s |
| 81 |
} |
| 82 |
|
| 83 |
// Timeout is a global timeout for processing bulk requests. This is a |
| 84 |
// server-side timeout, i.e. it tells Elasticsearch the time after which |
| 85 |
// it should stop processing. |
| 86 |
func (s *BulkService) Timeout(timeout string) *BulkService { |
| 87 |
s.timeout = timeout |
| 88 |
return s |
| 89 |
} |
| 90 |
|
| 91 |
// Refresh controls when changes made by this request are made visible |
| 92 |
// to search. The allowed values are: "true" (refresh the relevant |
| 93 |
// primary and replica shards immediately), "wait_for" (wait for the |
| 94 |
// changes to be made visible by a refresh before applying), or "false" |
| 95 |
// (no refresh related actions). |
| 96 |
func (s *BulkService) Refresh(refresh string) *BulkService { |
| 97 |
s.refresh = refresh |
| 98 |
return s |
| 99 |
} |
| 100 |
|
| 101 |
// Routing specifies the routing value. |
| 102 |
func (s *BulkService) Routing(routing string) *BulkService { |
| 103 |
s.routing = routing |
| 104 |
return s |
| 105 |
} |
| 106 |
|
| 107 |
// Pipeline specifies the pipeline id to preprocess incoming documents with. |
| 108 |
func (s *BulkService) Pipeline(pipeline string) *BulkService { |
| 109 |
s.pipeline = pipeline |
| 110 |
return s |
| 111 |
} |
| 112 |
|
| 113 |
// WaitForActiveShards sets the number of shard copies that must be active |
| 114 |
// before proceeding with the bulk operation. Defaults to 1, meaning the |
| 115 |
// primary shard only. Set to `all` for all shard copies, otherwise set to |
| 116 |
// any non-negative value less than or equal to the total number of copies |
| 117 |
// for the shard (number of replicas + 1). |
| 118 |
func (s *BulkService) WaitForActiveShards(waitForActiveShards string) *BulkService { |
| 119 |
s.waitForActiveShards = waitForActiveShards |
| 120 |
return s |
| 121 |
} |
| 122 |
|
| 123 |
// Pretty tells Elasticsearch whether to return a formatted JSON response. |
| 124 |
func (s *BulkService) Pretty(pretty bool) *BulkService { |
| 125 |
s.pretty = pretty |
| 126 |
return s |
| 127 |
} |
| 128 |
|
| 129 |
// Add adds bulkable requests, i.e. BulkIndexRequest, BulkUpdateRequest, |
| 130 |
// and/or BulkDeleteRequest. |
| 131 |
func (s *BulkService) Add(requests ...BulkableRequest) *BulkService { |
| 132 |
for _, r := range requests { |
| 133 |
s.requests = append(s.requests, r) |
| 134 |
} |
| 135 |
return s |
| 136 |
} |
| 137 |
|
| 138 |
// EstimatedSizeInBytes returns the estimated size of all bulkable |
| 139 |
// requests added via Add. |
| 140 |
func (s *BulkService) EstimatedSizeInBytes() int64 { |
| 141 |
if s.sizeInBytesCursor == len(s.requests) { |
| 142 |
return s.sizeInBytes |
| 143 |
} |
| 144 |
for _, r := range s.requests[s.sizeInBytesCursor:] { |
| 145 |
s.sizeInBytes += s.estimateSizeInBytes(r) |
| 146 |
s.sizeInBytesCursor++ |
| 147 |
} |
| 148 |
return s.sizeInBytes |
| 149 |
} |
| 150 |
|
| 151 |
// estimateSizeInBytes returns the estimates size of the given |
| 152 |
// bulkable request, i.e. BulkIndexRequest, BulkUpdateRequest, and |
| 153 |
// BulkDeleteRequest. |
| 154 |
func (s *BulkService) estimateSizeInBytes(r BulkableRequest) int64 { |
| 155 |
lines, _ := r.Source() |
| 156 |
size := 0 |
| 157 |
for _, line := range lines { |
| 158 |
// +1 for the \n |
| 159 |
size += len(line) + 1 |
| 160 |
} |
| 161 |
return int64(size) |
| 162 |
} |
| 163 |
|
| 164 |
// NumberOfActions returns the number of bulkable requests that need to |
| 165 |
// be sent to Elasticsearch on the next batch. |
| 166 |
func (s *BulkService) NumberOfActions() int { |
| 167 |
return len(s.requests) |
| 168 |
} |
| 169 |
|
| 170 |
func (s *BulkService) bodyAsString() (string, error) { |
| 171 |
// Pre-allocate to reduce allocs |
| 172 |
buf := bytes.NewBuffer(make([]byte, 0, s.EstimatedSizeInBytes())) |
| 173 |
|
| 174 |
for _, req := range s.requests { |
| 175 |
source, err := req.Source() |
| 176 |
if err != nil { |
| 177 |
return "", err |
| 178 |
} |
| 179 |
for _, line := range source { |
| 180 |
buf.WriteString(line) |
| 181 |
buf.WriteByte('\n') |
| 182 |
} |
| 183 |
} |
| 184 |
|
| 185 |
return buf.String(), nil |
| 186 |
} |
| 187 |
|
| 188 |
// Do sends the batched requests to Elasticsearch. Note that, when successful, |
| 189 |
// you can reuse the BulkService for the next batch as the list of bulk |
| 190 |
// requests is cleared on success. |
| 191 |
func (s *BulkService) Do(ctx context.Context) (*BulkResponse, error) { |
| 192 |
// No actions? |
| 193 |
if s.NumberOfActions() == 0 { |
| 194 |
return nil, errors.New("elastic: No bulk actions to commit") |
| 195 |
} |
| 196 |
|
| 197 |
// Get body |
| 198 |
body, err := s.bodyAsString() |
| 199 |
if err != nil { |
| 200 |
return nil, err |
| 201 |
} |
| 202 |
|
| 203 |
// Build url |
| 204 |
path := "/" |
| 205 |
if len(s.index) > 0 { |
| 206 |
index, err := uritemplates.Expand("{index}", map[string]string{ |
| 207 |
"index": s.index, |
| 208 |
}) |
| 209 |
if err != nil { |
| 210 |
return nil, err |
| 211 |
} |
| 212 |
path += index + "/" |
| 213 |
} |
| 214 |
if len(s.typ) > 0 { |
| 215 |
typ, err := uritemplates.Expand("{type}", map[string]string{ |
| 216 |
"type": s.typ, |
| 217 |
}) |
| 218 |
if err != nil { |
| 219 |
return nil, err |
| 220 |
} |
| 221 |
path += typ + "/" |
| 222 |
} |
| 223 |
path += "_bulk" |
| 224 |
|
| 225 |
// Parameters |
| 226 |
params := make(url.Values) |
| 227 |
if s.pretty { |
| 228 |
params.Set("pretty", fmt.Sprintf("%v", s.pretty)) |
| 229 |
} |
| 230 |
if s.pipeline != "" { |
| 231 |
params.Set("pipeline", s.pipeline) |
| 232 |
} |
| 233 |
if s.refresh != "" { |
| 234 |
params.Set("refresh", s.refresh) |
| 235 |
} |
| 236 |
if s.routing != "" { |
| 237 |
params.Set("routing", s.routing) |
| 238 |
} |
| 239 |
if s.timeout != "" { |
| 240 |
params.Set("timeout", s.timeout) |
| 241 |
} |
| 242 |
if s.waitForActiveShards != "" { |
| 243 |
params.Set("wait_for_active_shards", s.waitForActiveShards) |
| 244 |
} |
| 245 |
|
| 246 |
// Get response |
| 247 |
res, err := s.client.PerformRequestWithOptions(ctx, PerformRequestOptions{ |
| 248 |
Method: "POST", |
| 249 |
Path: path, |
| 250 |
Params: params, |
| 251 |
Body: body, |
| 252 |
ContentType: "application/x-ndjson", |
| 253 |
Retrier: s.retrier, |
| 254 |
}) |
| 255 |
if err != nil { |
| 256 |
return nil, err |
| 257 |
} |
| 258 |
|
| 259 |
// Return results |
| 260 |
ret := new(BulkResponse) |
| 261 |
if err := s.client.decoder.Decode(res.Body, ret); err != nil { |
| 262 |
return nil, err |
| 263 |
} |
| 264 |
|
| 265 |
// Reset so the request can be reused |
| 266 |
s.Reset() |
| 267 |
|
| 268 |
return ret, nil |
| 269 |
} |
| 270 |
|
| 271 |
// BulkResponse is a response to a bulk execution. |
| 272 |
// |
| 273 |
// Example: |
| 274 |
// { |
| 275 |
// "took":3, |
| 276 |
// "errors":false, |
| 277 |
// "items":[{ |
| 278 |
// "index":{ |
| 279 |
// "_index":"index1", |
| 280 |
// "_type":"tweet", |
| 281 |
// "_id":"1", |
| 282 |
// "_version":3, |
| 283 |
// "status":201 |
| 284 |
// } |
| 285 |
// },{ |
| 286 |
// "index":{ |
| 287 |
// "_index":"index2", |
| 288 |
// "_type":"tweet", |
| 289 |
// "_id":"2", |
| 290 |
// "_version":3, |
| 291 |
// "status":200 |
| 292 |
// } |
| 293 |
// },{ |
| 294 |
// "delete":{ |
| 295 |
// "_index":"index1", |
| 296 |
// "_type":"tweet", |
| 297 |
// "_id":"1", |
| 298 |
// "_version":4, |
| 299 |
// "status":200, |
| 300 |
// "found":true |
| 301 |
// } |
| 302 |
// },{ |
| 303 |
// "update":{ |
| 304 |
// "_index":"index2", |
| 305 |
// "_type":"tweet", |
| 306 |
// "_id":"2", |
| 307 |
// "_version":4, |
| 308 |
// "status":200 |
| 309 |
// } |
| 310 |
// }] |
| 311 |
// } |
| 312 |
type BulkResponse struct { |
| 313 |
Took int `json:"took,omitempty"` |
| 314 |
Errors bool `json:"errors,omitempty"` |
| 315 |
Items []map[string]*BulkResponseItem `json:"items,omitempty"` |
| 316 |
} |
| 317 |
|
| 318 |
// BulkResponseItem is the result of a single bulk request. |
| 319 |
type BulkResponseItem struct { |
| 320 |
Index string `json:"_index,omitempty"` |
| 321 |
Type string `json:"_type,omitempty"` |
| 322 |
Id string `json:"_id,omitempty"` |
| 323 |
Version int64 `json:"_version,omitempty"` |
| 324 |
Status int `json:"status,omitempty"` |
| 325 |
Result string `json:"result,omitempty"` |
| 326 |
ForcedRefresh bool `json:"forced_refresh,omitempty"` |
| 327 |
Found bool `json:"found,omitempty"` |
| 328 |
Error *ErrorDetails `json:"error,omitempty"` |
| 329 |
GetResult *GetResult `json:"get,omitempty"` |
| 330 |
} |
| 331 |
|
| 332 |
// Indexed returns all bulk request results of "index" actions. |
| 333 |
func (r *BulkResponse) Indexed() []*BulkResponseItem { |
| 334 |
return r.ByAction("index") |
| 335 |
} |
| 336 |
|
| 337 |
// Created returns all bulk request results of "create" actions. |
| 338 |
func (r *BulkResponse) Created() []*BulkResponseItem { |
| 339 |
return r.ByAction("create") |
| 340 |
} |
| 341 |
|
| 342 |
// Updated returns all bulk request results of "update" actions. |
| 343 |
func (r *BulkResponse) Updated() []*BulkResponseItem { |
| 344 |
return r.ByAction("update") |
| 345 |
} |
| 346 |
|
| 347 |
// Deleted returns all bulk request results of "delete" actions. |
| 348 |
func (r *BulkResponse) Deleted() []*BulkResponseItem { |
| 349 |
return r.ByAction("delete") |
| 350 |
} |
| 351 |
|
| 352 |
// ByAction returns all bulk request results of a certain action, |
| 353 |
// e.g. "index" or "delete". |
| 354 |
func (r *BulkResponse) ByAction(action string) []*BulkResponseItem { |
| 355 |
if r.Items == nil { |
| 356 |
return nil |
| 357 |
} |
| 358 |
var items []*BulkResponseItem |
| 359 |
for _, item := range r.Items { |
| 360 |
if result, found := item[action]; found { |
| 361 |
items = append(items, result) |
| 362 |
} |
| 363 |
} |
| 364 |
return items |
| 365 |
} |
| 366 |
|
| 367 |
// ById returns all bulk request results of a given document id, |
| 368 |
// regardless of the action ("index", "delete" etc.). |
| 369 |
func (r *BulkResponse) ById(id string) []*BulkResponseItem { |
| 370 |
if r.Items == nil { |
| 371 |
return nil |
| 372 |
} |
| 373 |
var items []*BulkResponseItem |
| 374 |
for _, item := range r.Items { |
| 375 |
for _, result := range item { |
| 376 |
if result.Id == id { |
| 377 |
items = append(items, result) |
| 378 |
} |
| 379 |
} |
| 380 |
} |
| 381 |
return items |
| 382 |
} |
| 383 |
|
| 384 |
// Failed returns those items of a bulk response that have errors, |
| 385 |
// i.e. those that don't have a status code between 200 and 299. |
| 386 |
func (r *BulkResponse) Failed() []*BulkResponseItem { |
| 387 |
if r.Items == nil { |
| 388 |
return nil |
| 389 |
} |
| 390 |
var errors []*BulkResponseItem |
| 391 |
for _, item := range r.Items { |
| 392 |
for _, result := range item { |
| 393 |
if !(result.Status >= 200 && result.Status <= 299) { |
| 394 |
errors = append(errors, result) |
| 395 |
} |
| 396 |
} |
| 397 |
} |
| 398 |
return errors |
| 399 |
} |
| 400 |
|
| 401 |
// Succeeded returns those items of a bulk response that have no errors, |
| 402 |
// i.e. those have a status code between 200 and 299. |
| 403 |
func (r *BulkResponse) Succeeded() []*BulkResponseItem { |
| 404 |
if r.Items == nil { |
| 405 |
return nil |
| 406 |
} |
| 407 |
var succeeded []*BulkResponseItem |
| 408 |
for _, item := range r.Items { |
| 409 |
for _, result := range item { |
| 410 |
if result.Status >= 200 && result.Status <= 299 { |
| 411 |
succeeded = append(succeeded, result) |
| 412 |
} |
| 413 |
} |
| 414 |
} |
| 415 |
return succeeded |
| 416 |
} |