Debian Codegraph @

bulk.go

raw
1 // Copyright 2012-present Oliver Eilhard. All rights reserved.
2 // Use of this source code is governed by a MIT-license.
3 // See http://olivere.mit-license.org/license.txt for details.
4
5 package elastic
6
7 import (
8 "bytes"
9 "context"
10 "errors"
11 "fmt"
12 "net/url"
13
14 "gopkg.in/olivere/elastic.v5/uritemplates"
15 )
16
17 // BulkService allows for batching bulk requests and sending them to
18 // Elasticsearch in one roundtrip. Use the Add method with BulkIndexRequest,
19 // BulkUpdateRequest, and BulkDeleteRequest to add bulk requests to a batch,
20 // then use Do to send them to Elasticsearch.
21 //
22 // BulkService will be reset after each Do call. In other words, you can
23 // reuse BulkService to send many batches. You do not have to create a new
24 // BulkService for each batch.
25 //
26 // See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-bulk.html
27 // for more details.
28 type BulkService struct {
29 client *Client
30 retrier Retrier
31
32 index string
33 typ string
34 requests []BulkableRequest
35 pipeline string
36 timeout string
37 refresh string
38 routing string
39 waitForActiveShards string
40 pretty bool
41
42 // estimated bulk size in bytes, up to the request index sizeInBytesCursor
43 sizeInBytes int64
44 sizeInBytesCursor int
45 }
46
47 // NewBulkService initializes a new BulkService.
48 func NewBulkService(client *Client) *BulkService {
49 builder := &BulkService{
50 client: client,
51 }
52 return builder
53 }
54
55 // Reset cleans up the request queue
56 func (s *BulkService) Reset() {
57 s.requests = make([]BulkableRequest, 0)
58 s.sizeInBytes = 0
59 s.sizeInBytesCursor = 0
60 }
61
62 // Retrier allows to set specific retry logic for this BulkService.
63 // If not specified, it will use the client's default retrier.
64 func (s *BulkService) Retrier(retrier Retrier) *BulkService {
65 s.retrier = retrier
66 return s
67 }
68
69 // Index specifies the index to use for all batches. You may also leave
70 // this blank and specify the index in the individual bulk requests.
71 func (s *BulkService) Index(index string) *BulkService {
72 s.index = index
73 return s
74 }
75
76 // Type specifies the type to use for all batches. You may also leave
77 // this blank and specify the type in the individual bulk requests.
78 func (s *BulkService) Type(typ string) *BulkService {
79 s.typ = typ
80 return s
81 }
82
83 // Timeout is a global timeout for processing bulk requests. This is a
84 // server-side timeout, i.e. it tells Elasticsearch the time after which
85 // it should stop processing.
86 func (s *BulkService) Timeout(timeout string) *BulkService {
87 s.timeout = timeout
88 return s
89 }
90
91 // Refresh controls when changes made by this request are made visible
92 // to search. The allowed values are: "true" (refresh the relevant
93 // primary and replica shards immediately), "wait_for" (wait for the
94 // changes to be made visible by a refresh before applying), or "false"
95 // (no refresh related actions).
96 func (s *BulkService) Refresh(refresh string) *BulkService {
97 s.refresh = refresh
98 return s
99 }
100
101 // Routing specifies the routing value.
102 func (s *BulkService) Routing(routing string) *BulkService {
103 s.routing = routing
104 return s
105 }
106
107 // Pipeline specifies the pipeline id to preprocess incoming documents with.
108 func (s *BulkService) Pipeline(pipeline string) *BulkService {
109 s.pipeline = pipeline
110 return s
111 }
112
113 // WaitForActiveShards sets the number of shard copies that must be active
114 // before proceeding with the bulk operation. Defaults to 1, meaning the
115 // primary shard only. Set to `all` for all shard copies, otherwise set to
116 // any non-negative value less than or equal to the total number of copies
117 // for the shard (number of replicas + 1).
118 func (s *BulkService) WaitForActiveShards(waitForActiveShards string) *BulkService {
119 s.waitForActiveShards = waitForActiveShards
120 return s
121 }
122
123 // Pretty tells Elasticsearch whether to return a formatted JSON response.
124 func (s *BulkService) Pretty(pretty bool) *BulkService {
125 s.pretty = pretty
126 return s
127 }
128
129 // Add adds bulkable requests, i.e. BulkIndexRequest, BulkUpdateRequest,
130 // and/or BulkDeleteRequest.
131 func (s *BulkService) Add(requests ...BulkableRequest) *BulkService {
132 for _, r := range requests {
133 s.requests = append(s.requests, r)
134 }
135 return s
136 }
137
138 // EstimatedSizeInBytes returns the estimated size of all bulkable
139 // requests added via Add.
140 func (s *BulkService) EstimatedSizeInBytes() int64 {
141 if s.sizeInBytesCursor == len(s.requests) {
142 return s.sizeInBytes
143 }
144 for _, r := range s.requests[s.sizeInBytesCursor:] {
145 s.sizeInBytes += s.estimateSizeInBytes(r)
146 s.sizeInBytesCursor++
147 }
148 return s.sizeInBytes
149 }
150
151 // estimateSizeInBytes returns the estimates size of the given
152 // bulkable request, i.e. BulkIndexRequest, BulkUpdateRequest, and
153 // BulkDeleteRequest.
154 func (s *BulkService) estimateSizeInBytes(r BulkableRequest) int64 {
155 lines, _ := r.Source()
156 size := 0
157 for _, line := range lines {
158 // +1 for the \n
159 size += len(line) + 1
160 }
161 return int64(size)
162 }
163
164 // NumberOfActions returns the number of bulkable requests that need to
165 // be sent to Elasticsearch on the next batch.
166 func (s *BulkService) NumberOfActions() int {
167 return len(s.requests)
168 }
169
170 func (s *BulkService) bodyAsString() (string, error) {
171 // Pre-allocate to reduce allocs
172 buf := bytes.NewBuffer(make([]byte, 0, s.EstimatedSizeInBytes()))
173
174 for _, req := range s.requests {
175 source, err := req.Source()
176 if err != nil {
177 return "", err
178 }
179 for _, line := range source {
180 buf.WriteString(line)
181 buf.WriteByte('\n')
182 }
183 }
184
185 return buf.String(), nil
186 }
187
188 // Do sends the batched requests to Elasticsearch. Note that, when successful,
189 // you can reuse the BulkService for the next batch as the list of bulk
190 // requests is cleared on success.
191 func (s *BulkService) Do(ctx context.Context) (*BulkResponse, error) {
192 // No actions?
193 if s.NumberOfActions() == 0 {
194 return nil, errors.New("elastic: No bulk actions to commit")
195 }
196
197 // Get body
198 body, err := s.bodyAsString()
199 if err != nil {
200 return nil, err
201 }
202
203 // Build url
204 path := "/"
205 if len(s.index) > 0 {
206 index, err := uritemplates.Expand("{index}", map[string]string{
207 "index": s.index,
208 })
209 if err != nil {
210 return nil, err
211 }
212 path += index + "/"
213 }
214 if len(s.typ) > 0 {
215 typ, err := uritemplates.Expand("{type}", map[string]string{
216 "type": s.typ,
217 })
218 if err != nil {
219 return nil, err
220 }
221 path += typ + "/"
222 }
223 path += "_bulk"
224
225 // Parameters
226 params := make(url.Values)
227 if s.pretty {
228 params.Set("pretty", fmt.Sprintf("%v", s.pretty))
229 }
230 if s.pipeline != "" {
231 params.Set("pipeline", s.pipeline)
232 }
233 if s.refresh != "" {
234 params.Set("refresh", s.refresh)
235 }
236 if s.routing != "" {
237 params.Set("routing", s.routing)
238 }
239 if s.timeout != "" {
240 params.Set("timeout", s.timeout)
241 }
242 if s.waitForActiveShards != "" {
243 params.Set("wait_for_active_shards", s.waitForActiveShards)
244 }
245
246 // Get response
247 res, err := s.client.PerformRequestWithOptions(ctx, PerformRequestOptions{
248 Method: "POST",
249 Path: path,
250 Params: params,
251 Body: body,
252 ContentType: "application/x-ndjson",
253 Retrier: s.retrier,
254 })
255 if err != nil {
256 return nil, err
257 }
258
259 // Return results
260 ret := new(BulkResponse)
261 if err := s.client.decoder.Decode(res.Body, ret); err != nil {
262 return nil, err
263 }
264
265 // Reset so the request can be reused
266 s.Reset()
267
268 return ret, nil
269 }
270
271 // BulkResponse is a response to a bulk execution.
272 //
273 // Example:
274 // {
275 // "took":3,
276 // "errors":false,
277 // "items":[{
278 // "index":{
279 // "_index":"index1",
280 // "_type":"tweet",
281 // "_id":"1",
282 // "_version":3,
283 // "status":201
284 // }
285 // },{
286 // "index":{
287 // "_index":"index2",
288 // "_type":"tweet",
289 // "_id":"2",
290 // "_version":3,
291 // "status":200
292 // }
293 // },{
294 // "delete":{
295 // "_index":"index1",
296 // "_type":"tweet",
297 // "_id":"1",
298 // "_version":4,
299 // "status":200,
300 // "found":true
301 // }
302 // },{
303 // "update":{
304 // "_index":"index2",
305 // "_type":"tweet",
306 // "_id":"2",
307 // "_version":4,
308 // "status":200
309 // }
310 // }]
311 // }
312 type BulkResponse struct {
313 Took int `json:"took,omitempty"`
314 Errors bool `json:"errors,omitempty"`
315 Items []map[string]*BulkResponseItem `json:"items,omitempty"`
316 }
317
318 // BulkResponseItem is the result of a single bulk request.
319 type BulkResponseItem struct {
320 Index string `json:"_index,omitempty"`
321 Type string `json:"_type,omitempty"`
322 Id string `json:"_id,omitempty"`
323 Version int64 `json:"_version,omitempty"`
324 Status int `json:"status,omitempty"`
325 Result string `json:"result,omitempty"`
326 ForcedRefresh bool `json:"forced_refresh,omitempty"`
327 Found bool `json:"found,omitempty"`
328 Error *ErrorDetails `json:"error,omitempty"`
329 GetResult *GetResult `json:"get,omitempty"`
330 }
331
332 // Indexed returns all bulk request results of "index" actions.
333 func (r *BulkResponse) Indexed() []*BulkResponseItem {
334 return r.ByAction("index")
335 }
336
337 // Created returns all bulk request results of "create" actions.
338 func (r *BulkResponse) Created() []*BulkResponseItem {
339 return r.ByAction("create")
340 }
341
342 // Updated returns all bulk request results of "update" actions.
343 func (r *BulkResponse) Updated() []*BulkResponseItem {
344 return r.ByAction("update")
345 }
346
347 // Deleted returns all bulk request results of "delete" actions.
348 func (r *BulkResponse) Deleted() []*BulkResponseItem {
349 return r.ByAction("delete")
350 }
351
352 // ByAction returns all bulk request results of a certain action,
353 // e.g. "index" or "delete".
354 func (r *BulkResponse) ByAction(action string) []*BulkResponseItem {
355 if r.Items == nil {
356 return nil
357 }
358 var items []*BulkResponseItem
359 for _, item := range r.Items {
360 if result, found := item[action]; found {
361 items = append(items, result)
362 }
363 }
364 return items
365 }
366
367 // ById returns all bulk request results of a given document id,
368 // regardless of the action ("index", "delete" etc.).
369 func (r *BulkResponse) ById(id string) []*BulkResponseItem {
370 if r.Items == nil {
371 return nil
372 }
373 var items []*BulkResponseItem
374 for _, item := range r.Items {
375 for _, result := range item {
376 if result.Id == id {
377 items = append(items, result)
378 }
379 }
380 }
381 return items
382 }
383
384 // Failed returns those items of a bulk response that have errors,
385 // i.e. those that don't have a status code between 200 and 299.
386 func (r *BulkResponse) Failed() []*BulkResponseItem {
387 if r.Items == nil {
388 return nil
389 }
390 var errors []*BulkResponseItem
391 for _, item := range r.Items {
392 for _, result := range item {
393 if !(result.Status >= 200 && result.Status <= 299) {
394 errors = append(errors, result)
395 }
396 }
397 }
398 return errors
399 }
400
401 // Succeeded returns those items of a bulk response that have no errors,
402 // i.e. those have a status code between 200 and 299.
403 func (r *BulkResponse) Succeeded() []*BulkResponseItem {
404 if r.Items == nil {
405 return nil
406 }
407 var succeeded []*BulkResponseItem
408 for _, item := range r.Items {
409 for _, result := range item {
410 if result.Status >= 200 && result.Status <= 299 {
411 succeeded = append(succeeded, result)
412 }
413 }
414 }
415 return succeeded
416 }