Debian Codegraph @

bulk_processor.go

raw
1 // Copyright 2012-present Oliver Eilhard. All rights reserved.
2 // Use of this source code is governed by a MIT-license.
3 // See http://olivere.mit-license.org/license.txt for details.
4
5 package elastic
6
7 import (
8 "context"
9 "net"
10 "sync"
11 "sync/atomic"
12 "time"
13 )
14
15 // BulkProcessorService allows to easily process bulk requests. It allows setting
16 // policies when to flush new bulk requests, e.g. based on a number of actions,
17 // on the size of the actions, and/or to flush periodically. It also allows
18 // to control the number of concurrent bulk requests allowed to be executed
19 // in parallel.
20 //
21 // BulkProcessorService, by default, commits either every 1000 requests or when the
22 // (estimated) size of the bulk requests exceeds 5 MB. However, it does not
23 // commit periodically. BulkProcessorService also does retry by default, using
24 // an exponential backoff algorithm.
25 //
26 // The caller is responsible for setting the index and type on every
27 // bulk request added to BulkProcessorService.
28 //
29 // BulkProcessorService takes ideas from the BulkProcessor of the
30 // Elasticsearch Java API as documented in
31 // https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-docs-bulk-processor.html.
32 type BulkProcessorService struct {
33 c *Client
34 beforeFn BulkBeforeFunc
35 afterFn BulkAfterFunc
36 name string // name of processor
37 numWorkers int // # of workers (>= 1)
38 bulkActions int // # of requests after which to commit
39 bulkSize int // # of bytes after which to commit
40 flushInterval time.Duration // periodic flush interval
41 wantStats bool // indicates whether to gather statistics
42 backoff Backoff // a custom Backoff to use for errors
43 }
44
45 // NewBulkProcessorService creates a new BulkProcessorService.
46 func NewBulkProcessorService(client *Client) *BulkProcessorService {
47 return &BulkProcessorService{
48 c: client,
49 numWorkers: 1,
50 bulkActions: 1000,
51 bulkSize: 5 << 20, // 5 MB
52 backoff: NewExponentialBackoff(
53 time.Duration(200)*time.Millisecond,
54 time.Duration(10000)*time.Millisecond,
55 ),
56 }
57 }
58
59 // BulkBeforeFunc defines the signature of callbacks that are executed
60 // before a commit to Elasticsearch.
61 type BulkBeforeFunc func(executionId int64, requests []BulkableRequest)
62
63 // BulkAfterFunc defines the signature of callbacks that are executed
64 // after a commit to Elasticsearch. The err parameter signals an error.
65 type BulkAfterFunc func(executionId int64, requests []BulkableRequest, response *BulkResponse, err error)
66
67 // Before specifies a function to be executed before bulk requests get comitted
68 // to Elasticsearch.
69 func (s *BulkProcessorService) Before(fn BulkBeforeFunc) *BulkProcessorService {
70 s.beforeFn = fn
71 return s
72 }
73
74 // After specifies a function to be executed when bulk requests have been
75 // comitted to Elasticsearch. The After callback executes both when the
76 // commit was successful as well as on failures.
77 func (s *BulkProcessorService) After(fn BulkAfterFunc) *BulkProcessorService {
78 s.afterFn = fn
79 return s
80 }
81
82 // Name is an optional name to identify this bulk processor.
83 func (s *BulkProcessorService) Name(name string) *BulkProcessorService {
84 s.name = name
85 return s
86 }
87
88 // Workers is the number of concurrent workers allowed to be
89 // executed. Defaults to 1 and must be greater or equal to 1.
90 func (s *BulkProcessorService) Workers(num int) *BulkProcessorService {
91 s.numWorkers = num
92 return s
93 }
94
95 // BulkActions specifies when to flush based on the number of actions
96 // currently added. Defaults to 1000 and can be set to -1 to be disabled.
97 func (s *BulkProcessorService) BulkActions(bulkActions int) *BulkProcessorService {
98 s.bulkActions = bulkActions
99 return s
100 }
101
102 // BulkSize specifies when to flush based on the size (in bytes) of the actions
103 // currently added. Defaults to 5 MB and can be set to -1 to be disabled.
104 func (s *BulkProcessorService) BulkSize(bulkSize int) *BulkProcessorService {
105 s.bulkSize = bulkSize
106 return s
107 }
108
109 // FlushInterval specifies when to flush at the end of the given interval.
110 // This is disabled by default. If you want the bulk processor to
111 // operate completely asynchronously, set both BulkActions and BulkSize to
112 // -1 and set the FlushInterval to a meaningful interval.
113 func (s *BulkProcessorService) FlushInterval(interval time.Duration) *BulkProcessorService {
114 s.flushInterval = interval
115 return s
116 }
117
118 // Stats tells bulk processor to gather stats while running.
119 // Use Stats to return the stats. This is disabled by default.
120 func (s *BulkProcessorService) Stats(wantStats bool) *BulkProcessorService {
121 s.wantStats = wantStats
122 return s
123 }
124
125 // Backoff sets the backoff strategy to use for errors
126 func (s *BulkProcessorService) Backoff(backoff Backoff) *BulkProcessorService {
127 s.backoff = backoff
128 return s
129 }
130
131 // Do creates a new BulkProcessor and starts it.
132 // Consider the BulkProcessor as a running instance that accepts bulk requests
133 // and commits them to Elasticsearch, spreading the work across one or more
134 // workers.
135 //
136 // You can interoperate with the BulkProcessor returned by Do, e.g. Start and
137 // Stop (or Close) it.
138 //
139 // Context is an optional context that is passed into the bulk request
140 // service calls. In contrast to other operations, this context is used in
141 // a long running process. You could use it to pass e.g. loggers, but you
142 // shouldn't use it for cancellation.
143 //
144 // Calling Do several times returns new BulkProcessors. You probably don't
145 // want to do this. BulkProcessorService implements just a builder pattern.
146 func (s *BulkProcessorService) Do(ctx context.Context) (*BulkProcessor, error) {
147 p := newBulkProcessor(
148 s.c,
149 s.beforeFn,
150 s.afterFn,
151 s.name,
152 s.numWorkers,
153 s.bulkActions,
154 s.bulkSize,
155 s.flushInterval,
156 s.wantStats,
157 s.backoff)
158
159 err := p.Start(ctx)
160 if err != nil {
161 return nil, err
162 }
163 return p, nil
164 }
165
166 // -- Bulk Processor Statistics --
167
168 // BulkProcessorStats contains various statistics of a bulk processor
169 // while it is running. Use the Stats func to return it while running.
170 type BulkProcessorStats struct {
171 Flushed int64 // number of times the flush interval has been invoked
172 Committed int64 // # of times workers committed bulk requests
173 Indexed int64 // # of requests indexed
174 Created int64 // # of requests that ES reported as creates (201)
175 Updated int64 // # of requests that ES reported as updates
176 Deleted int64 // # of requests that ES reported as deletes
177 Succeeded int64 // # of requests that ES reported as successful
178 Failed int64 // # of requests that ES reported as failed
179
180 Workers []*BulkProcessorWorkerStats // stats for each worker
181 }
182
183 // BulkProcessorWorkerStats represents per-worker statistics.
184 type BulkProcessorWorkerStats struct {
185 Queued int64 // # of requests queued in this worker
186 LastDuration time.Duration // duration of last commit
187 }
188
189 // newBulkProcessorStats initializes and returns a BulkProcessorStats struct.
190 func newBulkProcessorStats(workers int) *BulkProcessorStats {
191 stats := &BulkProcessorStats{
192 Workers: make([]*BulkProcessorWorkerStats, workers),
193 }
194 for i := 0; i < workers; i++ {
195 stats.Workers[i] = &BulkProcessorWorkerStats{}
196 }
197 return stats
198 }
199
200 func (st *BulkProcessorStats) dup() *BulkProcessorStats {
201 dst := new(BulkProcessorStats)
202 dst.Flushed = st.Flushed
203 dst.Committed = st.Committed
204 dst.Indexed = st.Indexed
205 dst.Created = st.Created
206 dst.Updated = st.Updated
207 dst.Deleted = st.Deleted
208 dst.Succeeded = st.Succeeded
209 dst.Failed = st.Failed
210 for _, src := range st.Workers {
211 dst.Workers = append(dst.Workers, src.dup())
212 }
213 return dst
214 }
215
216 func (st *BulkProcessorWorkerStats) dup() *BulkProcessorWorkerStats {
217 dst := new(BulkProcessorWorkerStats)
218 dst.Queued = st.Queued
219 dst.LastDuration = st.LastDuration
220 return dst
221 }
222
223 // -- Bulk Processor --
224
225 // BulkProcessor encapsulates a task that accepts bulk requests and
226 // orchestrates committing them to Elasticsearch via one or more workers.
227 //
228 // BulkProcessor is returned by setting up a BulkProcessorService and
229 // calling the Do method.
230 type BulkProcessor struct {
231 c *Client
232 beforeFn BulkBeforeFunc
233 afterFn BulkAfterFunc
234 name string
235 bulkActions int
236 bulkSize int
237 numWorkers int
238 executionId int64
239 requestsC chan BulkableRequest
240 workerWg sync.WaitGroup
241 workers []*bulkWorker
242 flushInterval time.Duration
243 flusherStopC chan struct{}
244 wantStats bool
245 backoff Backoff
246
247 startedMu sync.Mutex // guards the following block
248 started bool
249
250 statsMu sync.Mutex // guards the following block
251 stats *BulkProcessorStats
252
253 stopReconnC chan struct{} // channel to signal stop reconnection attempts
254 }
255
256 func newBulkProcessor(
257 client *Client,
258 beforeFn BulkBeforeFunc,
259 afterFn BulkAfterFunc,
260 name string,
261 numWorkers int,
262 bulkActions int,
263 bulkSize int,
264 flushInterval time.Duration,
265 wantStats bool,
266 backoff Backoff) *BulkProcessor {
267 return &BulkProcessor{
268 c: client,
269 beforeFn: beforeFn,
270 afterFn: afterFn,
271 name: name,
272 numWorkers: numWorkers,
273 bulkActions: bulkActions,
274 bulkSize: bulkSize,
275 flushInterval: flushInterval,
276 wantStats: wantStats,
277 backoff: backoff,
278 }
279 }
280
281 // Start starts the bulk processor. If the processor is already started,
282 // nil is returned.
283 func (p *BulkProcessor) Start(ctx context.Context) error {
284 p.startedMu.Lock()
285 defer p.startedMu.Unlock()
286
287 if p.started {
288 return nil
289 }
290
291 // We must have at least one worker.
292 if p.numWorkers < 1 {
293 p.numWorkers = 1
294 }
295
296 p.requestsC = make(chan BulkableRequest)
297 p.executionId = 0
298 p.stats = newBulkProcessorStats(p.numWorkers)
299 p.stopReconnC = make(chan struct{})
300
301 // Create and start up workers.
302 p.workers = make([]*bulkWorker, p.numWorkers)
303 for i := 0; i < p.numWorkers; i++ {
304 p.workerWg.Add(1)
305 p.workers[i] = newBulkWorker(p, i)
306 go p.workers[i].work(ctx)
307 }
308
309 // Start the ticker for flush (if enabled)
310 if int64(p.flushInterval) > 0 {
311 p.flusherStopC = make(chan struct{})
312 go p.flusher(p.flushInterval)
313 }
314
315 p.started = true
316
317 return nil
318 }
319
320 // Stop is an alias for Close.
321 func (p *BulkProcessor) Stop() error {
322 return p.Close()
323 }
324
325 // Close stops the bulk processor previously started with Do.
326 // If it is already stopped, this is a no-op and nil is returned.
327 //
328 // By implementing Close, BulkProcessor implements the io.Closer interface.
329 func (p *BulkProcessor) Close() error {
330 p.startedMu.Lock()
331 defer p.startedMu.Unlock()
332
333 // Already stopped? Do nothing.
334 if !p.started {
335 return nil
336 }
337
338 // Tell connection checkers to stop
339 if p.stopReconnC != nil {
340 close(p.stopReconnC)
341 p.stopReconnC = nil
342 }
343
344 // Stop flusher (if enabled)
345 if p.flusherStopC != nil {
346 p.flusherStopC <- struct{}{}
347 <-p.flusherStopC
348 close(p.flusherStopC)
349 p.flusherStopC = nil
350 }
351
352 // Stop all workers.
353 close(p.requestsC)
354 p.workerWg.Wait()
355
356 p.started = false
357
358 return nil
359 }
360
361 // Stats returns the latest bulk processor statistics.
362 // Collecting stats must be enabled first by calling Stats(true) on
363 // the service that created this processor.
364 func (p *BulkProcessor) Stats() BulkProcessorStats {
365 p.statsMu.Lock()
366 defer p.statsMu.Unlock()
367 return *p.stats.dup()
368 }
369
370 // Add adds a single request to commit by the BulkProcessorService.
371 //
372 // The caller is responsible for setting the index and type on the request.
373 func (p *BulkProcessor) Add(request BulkableRequest) {
374 p.requestsC <- request
375 }
376
377 // Flush manually asks all workers to commit their outstanding requests.
378 // It returns only when all workers acknowledge completion.
379 func (p *BulkProcessor) Flush() error {
380 p.statsMu.Lock()
381 p.stats.Flushed++
382 p.statsMu.Unlock()
383
384 for _, w := range p.workers {
385 w.flushC <- struct{}{}
386 <-w.flushAckC // wait for completion
387 }
388 return nil
389 }
390
391 // flusher is a single goroutine that periodically asks all workers to
392 // commit their outstanding bulk requests. It is only started if
393 // FlushInterval is greater than 0.
394 func (p *BulkProcessor) flusher(interval time.Duration) {
395 ticker := time.NewTicker(interval)
396 defer ticker.Stop()
397
398 for {
399 select {
400 case <-ticker.C: // Periodic flush
401 p.Flush() // TODO swallow errors here?
402
403 case <-p.flusherStopC:
404 p.flusherStopC <- struct{}{}
405 return
406 }
407 }
408 }
409
410 // -- Bulk Worker --
411
412 // bulkWorker encapsulates a single worker, running in a goroutine,
413 // receiving bulk requests and eventually committing them to Elasticsearch.
414 // It is strongly bound to a BulkProcessor.
415 type bulkWorker struct {
416 p *BulkProcessor
417 i int
418 bulkActions int
419 bulkSize int
420 service *BulkService
421 flushC chan struct{}
422 flushAckC chan struct{}
423 }
424
425 // newBulkWorker creates a new bulkWorker instance.
426 func newBulkWorker(p *BulkProcessor, i int) *bulkWorker {
427 return &bulkWorker{
428 p: p,
429 i: i,
430 bulkActions: p.bulkActions,
431 bulkSize: p.bulkSize,
432 service: NewBulkService(p.c),
433 flushC: make(chan struct{}),
434 flushAckC: make(chan struct{}),
435 }
436 }
437
438 // work waits for bulk requests and manual flush calls on the respective
439 // channels and is invoked as a goroutine when the bulk processor is started.
440 func (w *bulkWorker) work(ctx context.Context) {
441 defer func() {
442 w.p.workerWg.Done()
443 close(w.flushAckC)
444 close(w.flushC)
445 }()
446
447 var stop bool
448 for !stop {
449 var err error
450 select {
451 case req, open := <-w.p.requestsC:
452 if open {
453 // Received a new request
454 w.service.Add(req)
455 if w.commitRequired() {
456 err = w.commit(ctx)
457 }
458 } else {
459 // Channel closed: Stop.
460 stop = true
461 if w.service.NumberOfActions() > 0 {
462 err = w.commit(ctx)
463 }
464 }
465 case <-w.flushC:
466 // Commit outstanding requests
467 if w.service.NumberOfActions() > 0 {
468 err = w.commit(ctx)
469 }
470 w.flushAckC <- struct{}{}
471 }
472 if !stop && err != nil {
473 waitForActive := func() {
474 // Add back pressure to prevent Add calls from filling up the request queue
475 ready := make(chan struct{})
476 go w.waitForActiveConnection(ready)
477 <-ready
478 }
479 if _, ok := err.(net.Error); ok {
480 waitForActive()
481 } else if IsConnErr(err) {
482 waitForActive()
483 }
484 }
485 }
486 }
487
488 // commit commits the bulk requests in the given service,
489 // invoking callbacks as specified.
490 func (w *bulkWorker) commit(ctx context.Context) error {
491 var res *BulkResponse
492
493 // commitFunc will commit bulk requests and, on failure, be retried
494 // via exponential backoff
495 commitFunc := func() error {
496 var err error
497 res, err = w.service.Do(ctx)
498 return err
499 }
500 // notifyFunc will be called if retry fails
501 notifyFunc := func(err error) {
502 w.p.c.errorf("elastic: bulk processor %q failed but may retry: %v", w.p.name, err)
503 }
504
505 id := atomic.AddInt64(&w.p.executionId, 1)
506
507 // Update # documents in queue before eventual retries
508 w.p.statsMu.Lock()
509 if w.p.wantStats {
510 w.p.stats.Workers[w.i].Queued = int64(len(w.service.requests))
511 }
512 w.p.statsMu.Unlock()
513
514 // Save requests because they will be reset in commitFunc
515 reqs := w.service.requests
516
517 // Invoke before callback
518 if w.p.beforeFn != nil {
519 w.p.beforeFn(id, reqs)
520 }
521
522 // Commit bulk requests
523 err := RetryNotify(commitFunc, w.p.backoff, notifyFunc)
524 w.updateStats(res)
525 if err != nil {
526 w.p.c.errorf("elastic: bulk processor %q failed: %v", w.p.name, err)
527 }
528
529 // Invoke after callback
530 if w.p.afterFn != nil {
531 w.p.afterFn(id, reqs, res, err)
532 }
533
534 return err
535 }
536
537 func (w *bulkWorker) waitForActiveConnection(ready chan<- struct{}) {
538 defer close(ready)
539
540 t := time.NewTicker(5 * time.Second)
541 defer t.Stop()
542
543 client := w.p.c
544 stopReconnC := w.p.stopReconnC
545 w.p.c.errorf("elastic: bulk processor %q is waiting for an active connection", w.p.name)
546
547 // loop until a health check finds at least 1 active connection or the reconnection channel is closed
548 for {
549 select {
550 case _, ok := <-stopReconnC:
551 if !ok {
552 w.p.c.errorf("elastic: bulk processor %q active connection check interrupted", w.p.name)
553 return
554 }
555 case <-t.C:
556 client.healthcheck(time.Duration(3)*time.Second, true)
557 if client.mustActiveConn() == nil {
558 // found an active connection
559 // exit and signal done to the WaitGroup
560 return
561 }
562 }
563 }
564 }
565
566 func (w *bulkWorker) updateStats(res *BulkResponse) {
567 // Update stats
568 if res != nil {
569 w.p.statsMu.Lock()
570 if w.p.wantStats {
571 w.p.stats.Committed++
572 if res != nil {
573 w.p.stats.Indexed += int64(len(res.Indexed()))
574 w.p.stats.Created += int64(len(res.Created()))
575 w.p.stats.Updated += int64(len(res.Updated()))
576 w.p.stats.Deleted += int64(len(res.Deleted()))
577 w.p.stats.Succeeded += int64(len(res.Succeeded()))
578 w.p.stats.Failed += int64(len(res.Failed()))
579 }
580 w.p.stats.Workers[w.i].Queued = int64(len(w.service.requests))
581 w.p.stats.Workers[w.i].LastDuration = time.Duration(int64(res.Took)) * time.Millisecond
582 }
583 w.p.statsMu.Unlock()
584 }
585 }
586
587 // commitRequired returns true if the service has to commit its
588 // bulk requests. This can be either because the number of actions
589 // or the estimated size in bytes is larger than specified in the
590 // BulkProcessorService.
591 func (w *bulkWorker) commitRequired() bool {
592 if w.bulkActions >= 0 && w.service.NumberOfActions() >= w.bulkActions {
593 return true
594 }
595 if w.bulkSize >= 0 && w.service.EstimatedSizeInBytes() >= int64(w.bulkSize) {
596 return true
597 }
598 return false
599 }