Debian Codegraph @

bulk_processor_test.go

raw
1 // Copyright 2012-present Oliver Eilhard. All rights reserved.
2 // Use of this source code is governed by a MIT-license.
3 // See http://olivere.mit-license.org/license.txt for details.
4
5 package elastic
6
7 import (
8 "context"
9 "fmt"
10 "math/rand"
11 "sync/atomic"
12 "testing"
13 "time"
14 )
15
16 func TestBulkProcessorDefaults(t *testing.T) {
17 client := setupTestClientAndCreateIndex(t)
18
19 p := client.BulkProcessor()
20 if p == nil {
21 t.Fatalf("expected BulkProcessorService; got: %v", p)
22 }
23 if got, want := p.name, ""; got != want {
24 t.Errorf("expected %q; got: %q", want, got)
25 }
26 if got, want := p.numWorkers, 1; got != want {
27 t.Errorf("expected %d; got: %d", want, got)
28 }
29 if got, want := p.bulkActions, 1000; got != want {
30 t.Errorf("expected %d; got: %d", want, got)
31 }
32 if got, want := p.bulkSize, 5*1024*1024; got != want {
33 t.Errorf("expected %d; got: %d", want, got)
34 }
35 if got, want := p.flushInterval, time.Duration(0); got != want {
36 t.Errorf("expected %v; got: %v", want, got)
37 }
38 if got, want := p.wantStats, false; got != want {
39 t.Errorf("expected %v; got: %v", want, got)
40 }
41 if p.backoff == nil {
42 t.Fatalf("expected non-nill backoff; got: %v", p.backoff)
43 }
44 }
45
46 func TestBulkProcessorCommitOnBulkActions(t *testing.T) {
47 //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
48 client := setupTestClientAndCreateIndex(t)
49
50 testBulkProcessor(t,
51 10000,
52 client.BulkProcessor().
53 Name("Actions-1").
54 Workers(1).
55 BulkActions(100).
56 BulkSize(-1),
57 )
58
59 testBulkProcessor(t,
60 10000,
61 client.BulkProcessor().
62 Name("Actions-2").
63 Workers(2).
64 BulkActions(100).
65 BulkSize(-1),
66 )
67 }
68
69 func TestBulkProcessorCommitOnBulkSize(t *testing.T) {
70 //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
71 client := setupTestClientAndCreateIndex(t)
72
73 testBulkProcessor(t,
74 10000,
75 client.BulkProcessor().
76 Name("Size-1").
77 Workers(1).
78 BulkActions(-1).
79 BulkSize(64*1024),
80 )
81
82 testBulkProcessor(t,
83 10000,
84 client.BulkProcessor().
85 Name("Size-2").
86 Workers(2).
87 BulkActions(-1).
88 BulkSize(64*1024),
89 )
90 }
91
92 func TestBulkProcessorBasedOnFlushInterval(t *testing.T) {
93 //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
94 client := setupTestClientAndCreateIndex(t)
95
96 var beforeRequests int64
97 var befores int64
98 var afters int64
99 var failures int64
100 var afterRequests int64
101
102 beforeFn := func(executionId int64, requests []BulkableRequest) {
103 atomic.AddInt64(&beforeRequests, int64(len(requests)))
104 atomic.AddInt64(&befores, 1)
105 }
106 afterFn := func(executionId int64, requests []BulkableRequest, response *BulkResponse, err error) {
107 atomic.AddInt64(&afters, 1)
108 if err != nil {
109 atomic.AddInt64(&failures, 1)
110 }
111 atomic.AddInt64(&afterRequests, int64(len(requests)))
112 }
113
114 svc := client.BulkProcessor().
115 Name("FlushInterval-1").
116 Workers(2).
117 BulkActions(-1).
118 BulkSize(-1).
119 FlushInterval(1 * time.Second).
120 Before(beforeFn).
121 After(afterFn)
122
123 p, err := svc.Do(context.Background())
124 if err != nil {
125 t.Fatal(err)
126 }
127
128 const numDocs = 1000 // low-enough number that flush should be invoked
129
130 for i := 1; i <= numDocs; i++ {
131 tweet := tweet{User: "olivere", Message: fmt.Sprintf("%d. %s", i, randomString(rand.Intn(64)))}
132 request := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id(fmt.Sprintf("%d", i)).Doc(tweet)
133 p.Add(request)
134 }
135
136 // Should flush at least once
137 time.Sleep(2 * time.Second)
138
139 err = p.Close()
140 if err != nil {
141 t.Fatal(err)
142 }
143
144 if p.stats.Flushed == 0 {
145 t.Errorf("expected at least 1 flush; got: %d", p.stats.Flushed)
146 }
147 if got, want := beforeRequests, int64(numDocs); got != want {
148 t.Errorf("expected %d requests to before callback; got: %d", want, got)
149 }
150 if got, want := afterRequests, int64(numDocs); got != want {
151 t.Errorf("expected %d requests to after callback; got: %d", want, got)
152 }
153 if befores == 0 {
154 t.Error("expected at least 1 call to before callback")
155 }
156 if afters == 0 {
157 t.Error("expected at least 1 call to after callback")
158 }
159 if failures != 0 {
160 t.Errorf("expected 0 calls to failure callback; got: %d", failures)
161 }
162
163 // Check number of documents that were bulk indexed
164 _, err = p.c.Flush(testIndexName).Do(context.TODO())
165 if err != nil {
166 t.Fatal(err)
167 }
168 count, err := p.c.Count(testIndexName).Do(context.TODO())
169 if err != nil {
170 t.Fatal(err)
171 }
172 if count != int64(numDocs) {
173 t.Fatalf("expected %d documents; got: %d", numDocs, count)
174 }
175 }
176
177 func TestBulkProcessorClose(t *testing.T) {
178 //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
179 client := setupTestClientAndCreateIndex(t)
180
181 var beforeRequests int64
182 var befores int64
183 var afters int64
184 var failures int64
185 var afterRequests int64
186
187 beforeFn := func(executionId int64, requests []BulkableRequest) {
188 atomic.AddInt64(&beforeRequests, int64(len(requests)))
189 atomic.AddInt64(&befores, 1)
190 }
191 afterFn := func(executionId int64, requests []BulkableRequest, response *BulkResponse, err error) {
192 atomic.AddInt64(&afters, 1)
193 if err != nil {
194 atomic.AddInt64(&failures, 1)
195 }
196 atomic.AddInt64(&afterRequests, int64(len(requests)))
197 }
198
199 p, err := client.BulkProcessor().
200 Name("FlushInterval-1").
201 Workers(2).
202 BulkActions(-1).
203 BulkSize(-1).
204 FlushInterval(30 * time.Second). // 30 seconds to flush
205 Before(beforeFn).After(afterFn).
206 Do(context.Background())
207 if err != nil {
208 t.Fatal(err)
209 }
210
211 const numDocs = 1000 // low-enough number that flush should be invoked
212
213 for i := 1; i <= numDocs; i++ {
214 tweet := tweet{User: "olivere", Message: fmt.Sprintf("%d. %s", i, randomString(rand.Intn(64)))}
215 request := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id(fmt.Sprintf("%d", i)).Doc(tweet)
216 p.Add(request)
217 }
218
219 // Should not flush because 30s > 1s
220 time.Sleep(1 * time.Second)
221
222 // Close should flush
223 err = p.Close()
224 if err != nil {
225 t.Fatal(err)
226 }
227
228 if p.stats.Flushed != 0 {
229 t.Errorf("expected no flush; got: %d", p.stats.Flushed)
230 }
231 if got, want := beforeRequests, int64(numDocs); got != want {
232 t.Errorf("expected %d requests to before callback; got: %d", want, got)
233 }
234 if got, want := afterRequests, int64(numDocs); got != want {
235 t.Errorf("expected %d requests to after callback; got: %d", want, got)
236 }
237 if befores == 0 {
238 t.Error("expected at least 1 call to before callback")
239 }
240 if afters == 0 {
241 t.Error("expected at least 1 call to after callback")
242 }
243 if failures != 0 {
244 t.Errorf("expected 0 calls to failure callback; got: %d", failures)
245 }
246
247 // Check number of documents that were bulk indexed
248 _, err = p.c.Flush(testIndexName).Do(context.TODO())
249 if err != nil {
250 t.Fatal(err)
251 }
252 count, err := p.c.Count(testIndexName).Do(context.TODO())
253 if err != nil {
254 t.Fatal(err)
255 }
256 if count != int64(numDocs) {
257 t.Fatalf("expected %d documents; got: %d", numDocs, count)
258 }
259 }
260
261 func TestBulkProcessorFlush(t *testing.T) {
262 //client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
263 client := setupTestClientAndCreateIndex(t)
264
265 p, err := client.BulkProcessor().
266 Name("ManualFlush").
267 Workers(10).
268 BulkActions(-1).
269 BulkSize(-1).
270 FlushInterval(30 * time.Second). // 30 seconds to flush
271 Stats(true).
272 Do(context.Background())
273 if err != nil {
274 t.Fatal(err)
275 }
276
277 const numDocs = 100
278
279 for i := 1; i <= numDocs; i++ {
280 tweet := tweet{User: "olivere", Message: fmt.Sprintf("%d. %s", i, randomString(rand.Intn(64)))}
281 request := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id(fmt.Sprintf("%d", i)).Doc(tweet)
282 p.Add(request)
283 }
284
285 // Should not flush because 30s > 1s
286 time.Sleep(1 * time.Second)
287
288 // No flush yet
289 stats := p.Stats()
290 if stats.Flushed != 0 {
291 t.Errorf("expected no flush; got: %d", p.stats.Flushed)
292 }
293
294 // Manual flush
295 err = p.Flush()
296 if err != nil {
297 t.Fatal(err)
298 }
299
300 time.Sleep(1 * time.Second)
301
302 // Now flushed
303 stats = p.Stats()
304 if got, want := p.stats.Flushed, int64(1); got != want {
305 t.Errorf("expected %d flush; got: %d", want, got)
306 }
307
308 // Close should not start another flush
309 err = p.Close()
310 if err != nil {
311 t.Fatal(err)
312 }
313
314 // Still 1 flush
315 stats = p.Stats()
316 if got, want := p.stats.Flushed, int64(1); got != want {
317 t.Errorf("expected %d flush; got: %d", want, got)
318 }
319
320 // Check number of documents that were bulk indexed
321 _, err = p.c.Flush(testIndexName).Do(context.TODO())
322 if err != nil {
323 t.Fatal(err)
324 }
325 count, err := p.c.Count(testIndexName).Do(context.TODO())
326 if err != nil {
327 t.Fatal(err)
328 }
329 if count != int64(numDocs) {
330 t.Fatalf("expected %d documents; got: %d", numDocs, count)
331 }
332 }
333
334 // -- Helper --
335
336 func testBulkProcessor(t *testing.T, numDocs int, svc *BulkProcessorService) {
337 var beforeRequests int64
338 var befores int64
339 var afters int64
340 var failures int64
341 var afterRequests int64
342
343 beforeFn := func(executionId int64, requests []BulkableRequest) {
344 atomic.AddInt64(&beforeRequests, int64(len(requests)))
345 atomic.AddInt64(&befores, 1)
346 }
347 afterFn := func(executionId int64, requests []BulkableRequest, response *BulkResponse, err error) {
348 atomic.AddInt64(&afters, 1)
349 if err != nil {
350 atomic.AddInt64(&failures, 1)
351 }
352 atomic.AddInt64(&afterRequests, int64(len(requests)))
353 }
354
355 p, err := svc.Before(beforeFn).After(afterFn).Stats(true).Do(context.Background())
356 if err != nil {
357 t.Fatal(err)
358 }
359
360 for i := 1; i <= numDocs; i++ {
361 tweet := tweet{User: "olivere", Message: fmt.Sprintf("%07d. %s", i, randomString(1+rand.Intn(63)))}
362 request := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id(fmt.Sprintf("%d", i)).Doc(tweet)
363 p.Add(request)
364 }
365
366 err = p.Close()
367 if err != nil {
368 t.Fatal(err)
369 }
370
371 stats := p.Stats()
372
373 if stats.Flushed != 0 {
374 t.Errorf("expected no flush; got: %d", stats.Flushed)
375 }
376 if stats.Committed <= 0 {
377 t.Errorf("expected committed > %d; got: %d", 0, stats.Committed)
378 }
379 if got, want := stats.Indexed, int64(numDocs); got != want {
380 t.Errorf("expected indexed = %d; got: %d", want, got)
381 }
382 if got, want := stats.Created, int64(0); got != want {
383 t.Errorf("expected created = %d; got: %d", want, got)
384 }
385 if got, want := stats.Updated, int64(0); got != want {
386 t.Errorf("expected updated = %d; got: %d", want, got)
387 }
388 if got, want := stats.Deleted, int64(0); got != want {
389 t.Errorf("expected deleted = %d; got: %d", want, got)
390 }
391 if got, want := stats.Succeeded, int64(numDocs); got != want {
392 t.Errorf("expected succeeded = %d; got: %d", want, got)
393 }
394 if got, want := stats.Failed, int64(0); got != want {
395 t.Errorf("expected failed = %d; got: %d", want, got)
396 }
397 if got, want := beforeRequests, int64(numDocs); got != want {
398 t.Errorf("expected %d requests to before callback; got: %d", want, got)
399 }
400 if got, want := afterRequests, int64(numDocs); got != want {
401 t.Errorf("expected %d requests to after callback; got: %d", want, got)
402 }
403 if befores == 0 {
404 t.Error("expected at least 1 call to before callback")
405 }
406 if afters == 0 {
407 t.Error("expected at least 1 call to after callback")
408 }
409 if failures != 0 {
410 t.Errorf("expected 0 calls to failure callback; got: %d", failures)
411 }
412
413 // Check number of documents that were bulk indexed
414 _, err = p.c.Flush(testIndexName).Do(context.TODO())
415 if err != nil {
416 t.Fatal(err)
417 }
418 count, err := p.c.Count(testIndexName).Do(context.TODO())
419 if err != nil {
420 t.Fatal(err)
421 }
422 if count != int64(numDocs) {
423 t.Fatalf("expected %d documents; got: %d", numDocs, count)
424 }
425 }