Debian Codegraph @

recipes/sliced_scroll/sliced_scroll.go

raw
1 // Copyright 2012-present Oliver Eilhard. All rights reserved.
2 // Use of this source code is governed by a MIT-license.
3 // See http://olivere.mit-license.org/license.txt for details.
4
5 // SlicedScroll illustrates scrolling through a set of documents
6 // in parallel. It uses the sliced scrolling feature introduced
7 // in Elasticsearch 5.0 to create a number of Goroutines, each
8 // scrolling through a slice of the total results. A second goroutine
9 // receives the hits from the set of goroutines scrolling through
10 // the slices and simply counts the total number and the number of
11 // documents received per slice.
12 //
13 // The speedup of sliced scrolling can be significant but is very
14 // dependent on the specific use case.
15 //
16 // See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-request-scroll.html#sliced-scroll
17 // for details on sliced scrolling in Elasticsearch.
18 //
19 // Example
20 //
21 // Scroll with 4 parallel slices through an index called "products".
22 // Use "_uid" as the default field:
23 //
24 // sliced_scroll -index=products -n=4
25 //
26 package main
27
28 import (
29 "context"
30 "flag"
31 "fmt"
32 "io"
33 "log"
34 "sync"
35 "sync/atomic"
36 "time"
37
38 "golang.org/x/sync/errgroup"
39 "gopkg.in/olivere/elastic.v5"
40 )
41
42 func main() {
43 var (
44 url = flag.String("url", "http://localhost:9200", "Elasticsearch URL")
45 index = flag.String("index", "", "Elasticsearch index name")
46 typ = flag.String("type", "", "Elasticsearch type name")
47 field = flag.String("field", "", "Slice field (must be numeric)")
48 numSlices = flag.Int("n", 2, "Number of slices to use in parallel")
49 sniff = flag.Bool("sniff", true, "Enable or disable sniffing")
50 )
51 flag.Parse()
52 log.SetFlags(0)
53
54 if *url == "" {
55 log.Fatal("missing url parameter")
56 }
57 if *index == "" {
58 log.Fatal("missing index parameter")
59 }
60 if *numSlices <= 0 {
61 log.Fatal("n must be greater than zero")
62 }
63
64 // Create an Elasticsearch client
65 client, err := elastic.NewClient(elastic.SetURL(*url), elastic.SetSniff(*sniff))
66 if err != nil {
67 log.Fatal(err)
68 }
69
70 // Setup a group of goroutines from the excellent errgroup package
71 g, ctx := errgroup.WithContext(context.TODO())
72
73 // Hits channel will be sent to from the first set of goroutines and consumed by the second
74 type hit struct {
75 Slice int
76 Hit elastic.SearchHit
77 }
78 hitsc := make(chan hit)
79
80 begin := time.Now()
81
82 // Start a number of goroutines to parallelize scrolling
83 var wg sync.WaitGroup
84 for i := 0; i < *numSlices; i++ {
85 wg.Add(1)
86
87 slice := i
88
89 // Prepare the query
90 var query elastic.Query
91 if *typ == "" {
92 query = elastic.NewMatchAllQuery()
93 } else {
94 query = elastic.NewTypeQuery(*typ)
95 }
96
97 // Prepare the slice
98 sliceQuery := elastic.NewSliceQuery().Id(i).Max(*numSlices)
99 if *field != "" {
100 sliceQuery = sliceQuery.Field(*field)
101 }
102
103 // Start goroutine for this sliced scroll
104 g.Go(func() error {
105 defer wg.Done()
106 svc := client.Scroll(*index).Query(query).Slice(sliceQuery)
107 for {
108 res, err := svc.Do(ctx)
109 if err == io.EOF {
110 break
111 }
112 if err != nil {
113 return err
114 }
115 for _, searchHit := range res.Hits.Hits {
116 // Pass the hit to the hits channel, which will be consumed below
117 select {
118 case hitsc <- hit{Slice: slice, Hit: *searchHit}:
119 case <-ctx.Done():
120 return ctx.Err()
121 }
122 }
123 }
124 return nil
125 })
126 }
127 go func() {
128 // Wait until all scrolling is done
129 wg.Wait()
130 close(hitsc)
131 }()
132
133 // Second goroutine will consume the hits sent from the workers in first set of goroutines
134 var total uint64
135 totals := make([]uint64, *numSlices)
136 g.Go(func() error {
137 for hit := range hitsc {
138 // We simply count the hits here.
139 atomic.AddUint64(&totals[hit.Slice], 1)
140 current := atomic.AddUint64(&total, 1)
141 sec := int(time.Since(begin).Seconds())
142 fmt.Printf("%8d | %02d:%02d\r", current, sec/60, sec%60)
143 select {
144 default:
145 case <-ctx.Done():
146 return ctx.Err()
147 }
148 }
149 return nil
150 })
151
152 // Wait until all goroutines are finished
153 if err := g.Wait(); err != nil {
154 log.Fatal(err)
155 }
156
157 fmt.Printf("Scrolled through a total of %d documents in %v\n", total, time.Since(begin))
158 for i := 0; i < *numSlices; i++ {
159 fmt.Printf("Slice %2d received %d documents\n", i, totals[i])
160 }
161 }