Debian Codegraph @

scroll_test.go

raw
1 // Copyright 2012-present Oliver Eilhard. All rights reserved.
2 // Use of this source code is governed by a MIT-license.
3 // See http://olivere.mit-license.org/license.txt for details.
4
5 package elastic
6
7 import (
8 "context"
9 "encoding/json"
10 "io"
11 _ "net/http"
12 "testing"
13 )
14
15 func TestScroll(t *testing.T) {
16 // client := setupTestClientAndCreateIndexAndLog(t)
17 client := setupTestClientAndCreateIndex(t)
18
19 tweet1 := tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."}
20 tweet2 := tweet{User: "olivere", Message: "Another unrelated topic."}
21 tweet3 := tweet{User: "sandrae", Message: "Cycling is fun."}
22
23 // Add all documents
24 _, err := client.Index().Index(testIndexName).Type("tweet").Id("1").BodyJson(&tweet1).Do(context.TODO())
25 if err != nil {
26 t.Fatal(err)
27 }
28
29 _, err = client.Index().Index(testIndexName).Type("tweet").Id("2").BodyJson(&tweet2).Do(context.TODO())
30 if err != nil {
31 t.Fatal(err)
32 }
33
34 _, err = client.Index().Index(testIndexName).Type("tweet").Id("3").BodyJson(&tweet3).Do(context.TODO())
35 if err != nil {
36 t.Fatal(err)
37 }
38
39 _, err = client.Flush().Index(testIndexName).Do(context.TODO())
40 if err != nil {
41 t.Fatal(err)
42 }
43
44 // Should return all documents. Just don't call Do yet!
45 svc := client.Scroll(testIndexName).Size(1)
46
47 pages := 0
48 docs := 0
49
50 for {
51 res, err := svc.Do(context.TODO())
52 if err == io.EOF {
53 break
54 }
55 if err != nil {
56 t.Fatal(err)
57 }
58 if res == nil {
59 t.Fatal("expected results != nil; got nil")
60 }
61 if res.Hits == nil {
62 t.Fatal("expected results.Hits != nil; got nil")
63 }
64 if want, have := int64(3), res.Hits.TotalHits; want != have {
65 t.Fatalf("expected results.Hits.TotalHits = %d; got %d", want, have)
66 }
67 if want, have := 1, len(res.Hits.Hits); want != have {
68 t.Fatalf("expected len(results.Hits.Hits) = %d; got %d", want, have)
69 }
70
71 pages++
72
73 for _, hit := range res.Hits.Hits {
74 if hit.Index != testIndexName {
75 t.Fatalf("expected SearchResult.Hits.Hit.Index = %q; got %q", testIndexName, hit.Index)
76 }
77 item := make(map[string]interface{})
78 err := json.Unmarshal(*hit.Source, &item)
79 if err != nil {
80 t.Fatal(err)
81 }
82 docs++
83 }
84
85 if len(res.ScrollId) == 0 {
86 t.Fatalf("expected scrollId in results; got %q", res.ScrollId)
87 }
88 }
89
90 if want, have := 3, pages; want != have {
91 t.Fatalf("expected to retrieve %d pages; got %d", want, have)
92 }
93 if want, have := 3, docs; want != have {
94 t.Fatalf("expected to retrieve %d hits; got %d", want, have)
95 }
96
97 err = svc.Clear(context.TODO())
98 if err != nil {
99 t.Fatal(err)
100 }
101
102 _, err = svc.Do(context.TODO())
103 if err == nil {
104 t.Fatal("expected to fail")
105 }
106 }
107
108 func TestScrollWithQueryAndSort(t *testing.T) {
109 client := setupTestClientAndCreateIndex(t)
110 // client := setupTestClientAndCreateIndexAndAddDocs(t, SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)))
111
112 tweet1 := tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."}
113 tweet2 := tweet{User: "olivere", Message: "Another unrelated topic."}
114 tweet3 := tweet{User: "sandrae", Message: "Cycling is fun."}
115
116 // Add all documents
117 _, err := client.Index().Index(testIndexName).Type("tweet").Id("1").BodyJson(&tweet1).Do(context.TODO())
118 if err != nil {
119 t.Fatal(err)
120 }
121
122 _, err = client.Index().Index(testIndexName).Type("tweet").Id("2").BodyJson(&tweet2).Do(context.TODO())
123 if err != nil {
124 t.Fatal(err)
125 }
126
127 _, err = client.Index().Index(testIndexName).Type("tweet").Id("3").BodyJson(&tweet3).Do(context.TODO())
128 if err != nil {
129 t.Fatal(err)
130 }
131
132 _, err = client.Flush().Index(testIndexName).Do(context.TODO())
133 if err != nil {
134 t.Fatal(err)
135 }
136
137 // Create a scroll service that returns tweets from user olivere
138 // and returns them sorted by "message", in reverse order.
139 //
140 // Just don't call Do yet!
141 svc := client.Scroll(testIndexName).
142 Query(NewTermQuery("user", "olivere")).
143 Sort("message", false).
144 Size(1)
145
146 docs := 0
147 pages := 0
148 for {
149 res, err := svc.Do(context.TODO())
150 if err == io.EOF {
151 break
152 }
153 if err != nil {
154 t.Fatal(err)
155 }
156 if err != nil {
157 t.Fatal(err)
158 }
159 if res == nil {
160 t.Fatal("expected results != nil; got nil")
161 }
162 if res.Hits == nil {
163 t.Fatal("expected results.Hits != nil; got nil")
164 }
165 if want, have := int64(2), res.Hits.TotalHits; want != have {
166 t.Fatalf("expected results.Hits.TotalHits = %d; got %d", want, have)
167 }
168 if want, have := 1, len(res.Hits.Hits); want != have {
169 t.Fatalf("expected len(results.Hits.Hits) = %d; got %d", want, have)
170 }
171
172 pages++
173
174 for _, hit := range res.Hits.Hits {
175 if hit.Index != testIndexName {
176 t.Fatalf("expected SearchResult.Hits.Hit.Index = %q; got %q", testIndexName, hit.Index)
177 }
178 item := make(map[string]interface{})
179 err := json.Unmarshal(*hit.Source, &item)
180 if err != nil {
181 t.Fatal(err)
182 }
183 docs++
184 }
185 }
186
187 if want, have := 2, pages; want != have {
188 t.Fatalf("expected to retrieve %d pages; got %d", want, have)
189 }
190 if want, have := 2, docs; want != have {
191 t.Fatalf("expected to retrieve %d hits; got %d", want, have)
192 }
193 }
194
195 func TestScrollWithBody(t *testing.T) {
196 // client := setupTestClientAndCreateIndexAndLog(t)
197 client := setupTestClientAndCreateIndex(t)
198
199 tweet1 := tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch.", Retweets: 4}
200 tweet2 := tweet{User: "olivere", Message: "Another unrelated topic.", Retweets: 10}
201 tweet3 := tweet{User: "sandrae", Message: "Cycling is fun.", Retweets: 3}
202
203 // Add all documents
204 _, err := client.Index().Index(testIndexName).Type("tweet").Id("1").BodyJson(&tweet1).Do(context.TODO())
205 if err != nil {
206 t.Fatal(err)
207 }
208
209 _, err = client.Index().Index(testIndexName).Type("tweet").Id("2").BodyJson(&tweet2).Do(context.TODO())
210 if err != nil {
211 t.Fatal(err)
212 }
213
214 _, err = client.Index().Index(testIndexName).Type("tweet").Id("3").BodyJson(&tweet3).Do(context.TODO())
215 if err != nil {
216 t.Fatal(err)
217 }
218
219 _, err = client.Flush().Index(testIndexName).Do(context.TODO())
220 if err != nil {
221 t.Fatal(err)
222 }
223
224 // Test with simple strings and a map
225 var tests = []struct {
226 Body interface{}
227 ExpectedTotalHits int64
228 ExpectedDocs int
229 ExpectedPages int
230 }{
231 {
232 Body: `{"query":{"match_all":{}}}`,
233 ExpectedTotalHits: 3,
234 ExpectedDocs: 3,
235 ExpectedPages: 3,
236 },
237 {
238 Body: `{"query":{"term":{"user":"olivere"}},"sort":["_doc"]}`,
239 ExpectedTotalHits: 2,
240 ExpectedDocs: 2,
241 ExpectedPages: 2,
242 },
243 {
244 Body: `{"query":{"term":{"user":"olivere"}},"sort":[{"retweets":"desc"}]}`,
245 ExpectedTotalHits: 2,
246 ExpectedDocs: 2,
247 ExpectedPages: 2,
248 },
249 {
250 Body: map[string]interface{}{
251 "query": map[string]interface{}{
252 "term": map[string]interface{}{
253 "user": "olivere",
254 },
255 },
256 "sort": []interface{}{"_doc"},
257 },
258 ExpectedTotalHits: 2,
259 ExpectedDocs: 2,
260 ExpectedPages: 2,
261 },
262 }
263
264 for i, tt := range tests {
265 // Should return all documents. Just don't call Do yet!
266 svc := client.Scroll(testIndexName).Size(1).Body(tt.Body)
267
268 pages := 0
269 docs := 0
270
271 for {
272 res, err := svc.Do(context.TODO())
273 if err == io.EOF {
274 break
275 }
276 if err != nil {
277 t.Fatal(err)
278 }
279 if res == nil {
280 t.Fatalf("#%d: expected results != nil; got nil", i)
281 }
282 if res.Hits == nil {
283 t.Fatalf("#%d: expected results.Hits != nil; got nil", i)
284 }
285 if want, have := tt.ExpectedTotalHits, res.Hits.TotalHits; want != have {
286 t.Fatalf("#%d: expected results.Hits.TotalHits = %d; got %d", i, want, have)
287 }
288 if want, have := 1, len(res.Hits.Hits); want != have {
289 t.Fatalf("#%d: expected len(results.Hits.Hits) = %d; got %d", i, want, have)
290 }
291
292 pages++
293
294 for _, hit := range res.Hits.Hits {
295 if hit.Index != testIndexName {
296 t.Fatalf("#%d: expected SearchResult.Hits.Hit.Index = %q; got %q", i, testIndexName, hit.Index)
297 }
298 item := make(map[string]interface{})
299 err := json.Unmarshal(*hit.Source, &item)
300 if err != nil {
301 t.Fatalf("#%d: %v", i, err)
302 }
303 docs++
304 }
305
306 if len(res.ScrollId) == 0 {
307 t.Fatalf("#%d: expected scrollId in results; got %q", i, res.ScrollId)
308 }
309 }
310
311 if want, have := tt.ExpectedPages, pages; want != have {
312 t.Fatalf("#%d: expected to retrieve %d pages; got %d", i, want, have)
313 }
314 if want, have := tt.ExpectedDocs, docs; want != have {
315 t.Fatalf("#%d: expected to retrieve %d hits; got %d", i, want, have)
316 }
317
318 err = svc.Clear(context.TODO())
319 if err != nil {
320 t.Fatalf("#%d: failed to clear scroll context: %v", i, err)
321 }
322
323 _, err = svc.Do(context.TODO())
324 if err == nil {
325 t.Fatalf("#%d: expected to fail", i)
326 }
327 }
328 }
329
330 func TestScrollWithSlice(t *testing.T) {
331 client := setupTestClientAndCreateIndexAndAddDocs(t) //, SetTraceLog(log.New(os.Stdout, "", 0)))
332
333 // Should return all documents. Just don't call Do yet!
334 sliceQuery := NewSliceQuery().Id(0).Max(2)
335 svc := client.Scroll(testIndexName).Slice(sliceQuery).Size(1)
336
337 pages := 0
338 docs := 0
339
340 for {
341 res, err := svc.Do(context.TODO())
342 if err == io.EOF {
343 break
344 }
345 if err != nil {
346 t.Fatal(err)
347 }
348 if res == nil {
349 t.Fatal("expected results != nil; got nil")
350 }
351 if res.Hits == nil {
352 t.Fatal("expected results.Hits != nil; got nil")
353 }
354
355 pages++
356
357 for _, hit := range res.Hits.Hits {
358 if hit.Index != testIndexName {
359 t.Fatalf("expected SearchResult.Hits.Hit.Index = %q; got %q", testIndexName, hit.Index)
360 }
361 item := make(map[string]interface{})
362 err := json.Unmarshal(*hit.Source, &item)
363 if err != nil {
364 t.Fatal(err)
365 }
366 docs++
367 }
368
369 if len(res.ScrollId) == 0 {
370 t.Fatalf("expected scrollId in results; got %q", res.ScrollId)
371 }
372 }
373
374 if pages == 0 {
375 t.Fatal("expected to retrieve some pages")
376 }
377 if docs == 0 {
378 t.Fatal("expected to retrieve some hits")
379 }
380
381 if err := svc.Clear(context.TODO()); err != nil {
382 t.Fatal(err)
383 }
384
385 if _, err := svc.Do(context.TODO()); err == nil {
386 t.Fatal("expected to fail")
387 }
388 }
389
390 func TestScrollWithFilterPath(t *testing.T) {
391 // client := setupTestClientAndCreateIndexAndLog(t)
392 client := setupTestClientAndCreateIndex(t)
393
394 tweet1 := tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."}
395 tweet2 := tweet{User: "olivere", Message: "Another unrelated topic."}
396 tweet3 := tweet{User: "sandrae", Message: "Cycling is fun."}
397
398 // Add all documents
399 _, err := client.Index().Index(testIndexName).Type("doc").Id("1").BodyJson(&tweet1).Do(context.TODO())
400 if err != nil {
401 t.Fatal(err)
402 }
403
404 _, err = client.Index().Index(testIndexName).Type("doc").Id("2").BodyJson(&tweet2).Do(context.TODO())
405 if err != nil {
406 t.Fatal(err)
407 }
408
409 _, err = client.Index().Index(testIndexName).Type("doc").Id("3").BodyJson(&tweet3).Do(context.TODO())
410 if err != nil {
411 t.Fatal(err)
412 }
413
414 _, err = client.Flush().Index(testIndexName).Do(context.TODO())
415 if err != nil {
416 t.Fatal(err)
417 }
418
419 // Should return all documents. Just don't call Do yet!
420 // Notice that we don't have to add "_scroll_id" to the FilterPath here:
421 // It's been added automatically by the ScrollService.
422 svc := client.Scroll(testIndexName).Size(1).
423 FilterPath("hits.total", "hits.hits._index", "hits.hits._id", "hits.hits._source")
424
425 pages := 0
426 docs := 0
427
428 for {
429 res, err := svc.Do(context.TODO())
430 if err == io.EOF {
431 break
432 }
433 if err != nil {
434 t.Fatal(err)
435 }
436 if res == nil {
437 t.Fatal("expected results != nil; got nil")
438 }
439 if res.Hits == nil {
440 t.Fatal("expected results.Hits != nil; got nil")
441 }
442 if want, have := int64(3), res.Hits.TotalHits; want != have {
443 t.Fatalf("expected results.Hits.TotalHits = %d; got %d", want, have)
444 }
445 if want, have := 1, len(res.Hits.Hits); want != have {
446 t.Fatalf("expected len(results.Hits.Hits) = %d; got %d", want, have)
447 }
448
449 pages++
450
451 for _, hit := range res.Hits.Hits {
452 if hit.Index != testIndexName {
453 t.Fatalf("expected SearchResult.Hits.Hit.Index = %q; got %q", testIndexName, hit.Index)
454 }
455 item := make(map[string]interface{})
456 err := json.Unmarshal(*hit.Source, &item)
457 if err != nil {
458 t.Fatal(err)
459 }
460 docs++
461 }
462
463 if len(res.ScrollId) == 0 {
464 t.Fatalf("expected scrollId in results; got %q", res.ScrollId)
465 }
466 }
467
468 if want, have := 3, pages; want != have {
469 t.Fatalf("expected to retrieve %d pages; got %d", want, have)
470 }
471 if want, have := 3, docs; want != have {
472 t.Fatalf("expected to retrieve %d hits; got %d", want, have)
473 }
474
475 err = svc.Clear(context.TODO())
476 if err != nil {
477 t.Fatal(err)
478 }
479
480 _, err = svc.Do(context.TODO())
481 if err == nil {
482 t.Fatal("expected to fail")
483 }
484 }