Debian Codegraph @

scroll.go

raw
1 // Copyright 2012-present Oliver Eilhard. All rights reserved.
2 // Use of this source code is governed by a MIT-license.
3 // See http://olivere.mit-license.org/license.txt for details.
4
5 package elastic
6
7 import (
8 "context"
9 "fmt"
10 "io"
11 "net/url"
12 "strings"
13 "sync"
14
15 "gopkg.in/olivere/elastic.v5/uritemplates"
16 )
17
18 const (
19 // DefaultScrollKeepAlive is the default time a scroll cursor will be kept alive.
20 DefaultScrollKeepAlive = "5m"
21 )
22
23 // ScrollService iterates over pages of search results from Elasticsearch.
24 type ScrollService struct {
25 client *Client
26 retrier Retrier
27 indices []string
28 types []string
29 keepAlive string
30 body interface{}
31 ss *SearchSource
32 size *int
33 pretty bool
34 routing string
35 preference string
36 ignoreUnavailable *bool
37 allowNoIndices *bool
38 expandWildcards string
39 filterPath []string
40
41 mu sync.RWMutex
42 scrollId string
43 }
44
45 // NewScrollService initializes and returns a new ScrollService.
46 func NewScrollService(client *Client) *ScrollService {
47 builder := &ScrollService{
48 client: client,
49 ss: NewSearchSource(),
50 keepAlive: DefaultScrollKeepAlive,
51 }
52 return builder
53 }
54
55 // Retrier allows to set specific retry logic for this ScrollService.
56 // If not specified, it will use the client's default retrier.
57 func (s *ScrollService) Retrier(retrier Retrier) *ScrollService {
58 s.retrier = retrier
59 return s
60 }
61
62 // Index sets the name of one or more indices to iterate over.
63 func (s *ScrollService) Index(indices ...string) *ScrollService {
64 if s.indices == nil {
65 s.indices = make([]string, 0)
66 }
67 s.indices = append(s.indices, indices...)
68 return s
69 }
70
71 // Type sets the name of one or more types to iterate over.
72 func (s *ScrollService) Type(types ...string) *ScrollService {
73 if s.types == nil {
74 s.types = make([]string, 0)
75 }
76 s.types = append(s.types, types...)
77 return s
78 }
79
80 // Scroll is an alias for KeepAlive, the time to keep
81 // the cursor alive (e.g. "5m" for 5 minutes).
82 func (s *ScrollService) Scroll(keepAlive string) *ScrollService {
83 s.keepAlive = keepAlive
84 return s
85 }
86
87 // KeepAlive sets the maximum time after which the cursor will expire.
88 // It is "2m" by default.
89 func (s *ScrollService) KeepAlive(keepAlive string) *ScrollService {
90 s.keepAlive = keepAlive
91 return s
92 }
93
94 // Size specifies the number of documents Elasticsearch should return
95 // from each shard, per page.
96 func (s *ScrollService) Size(size int) *ScrollService {
97 s.size = &size
98 return s
99 }
100
101 // Body sets the raw body to send to Elasticsearch. This can be e.g. a string,
102 // a map[string]interface{} or anything that can be serialized into JSON.
103 // Notice that setting the body disables the use of SearchSource and many
104 // other properties of the SearchService.
105 func (s *ScrollService) Body(body interface{}) *ScrollService {
106 s.body = body
107 return s
108 }
109
110 // SearchSource sets the search source builder to use with this iterator.
111 // Notice that only a certain number of properties can be used when scrolling,
112 // e.g. query and sorting.
113 func (s *ScrollService) SearchSource(searchSource *SearchSource) *ScrollService {
114 s.ss = searchSource
115 if s.ss == nil {
116 s.ss = NewSearchSource()
117 }
118 return s
119 }
120
121 // Query sets the query to perform, e.g. a MatchAllQuery.
122 func (s *ScrollService) Query(query Query) *ScrollService {
123 s.ss = s.ss.Query(query)
124 return s
125 }
126
127 // PostFilter is executed as the last filter. It only affects the
128 // search hits but not facets. See
129 // https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-request-post-filter.html
130 // for details.
131 func (s *ScrollService) PostFilter(postFilter Query) *ScrollService {
132 s.ss = s.ss.PostFilter(postFilter)
133 return s
134 }
135
136 // Slice allows slicing the scroll request into several batches.
137 // This is supported in Elasticsearch 5.0 or later.
138 // See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-request-scroll.html#sliced-scroll
139 // for details.
140 func (s *ScrollService) Slice(sliceQuery Query) *ScrollService {
141 s.ss = s.ss.Slice(sliceQuery)
142 return s
143 }
144
145 // FetchSource indicates whether the response should contain the stored
146 // _source for every hit.
147 func (s *ScrollService) FetchSource(fetchSource bool) *ScrollService {
148 s.ss = s.ss.FetchSource(fetchSource)
149 return s
150 }
151
152 // FetchSourceContext indicates how the _source should be fetched.
153 func (s *ScrollService) FetchSourceContext(fetchSourceContext *FetchSourceContext) *ScrollService {
154 s.ss = s.ss.FetchSourceContext(fetchSourceContext)
155 return s
156 }
157
158 // Version can be set to true to return a version for each search hit.
159 // See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-request-version.html.
160 func (s *ScrollService) Version(version bool) *ScrollService {
161 s.ss = s.ss.Version(version)
162 return s
163 }
164
165 // Sort adds a sort order. This can have negative effects on the performance
166 // of the scroll operation as Elasticsearch needs to sort first.
167 func (s *ScrollService) Sort(field string, ascending bool) *ScrollService {
168 s.ss = s.ss.Sort(field, ascending)
169 return s
170 }
171
172 // SortWithInfo specifies a sort order. Notice that sorting can have a
173 // negative impact on scroll performance.
174 func (s *ScrollService) SortWithInfo(info SortInfo) *ScrollService {
175 s.ss = s.ss.SortWithInfo(info)
176 return s
177 }
178
179 // SortBy specifies a sort order. Notice that sorting can have a
180 // negative impact on scroll performance.
181 func (s *ScrollService) SortBy(sorter ...Sorter) *ScrollService {
182 s.ss = s.ss.SortBy(sorter...)
183 return s
184 }
185
186 // Pretty asks Elasticsearch to pretty-print the returned JSON.
187 func (s *ScrollService) Pretty(pretty bool) *ScrollService {
188 s.pretty = pretty
189 return s
190 }
191
192 // Routing is a list of specific routing values to control the shards
193 // the search will be executed on.
194 func (s *ScrollService) Routing(routings ...string) *ScrollService {
195 s.routing = strings.Join(routings, ",")
196 return s
197 }
198
199 // Preference sets the preference to execute the search. Defaults to
200 // randomize across shards ("random"). Can be set to "_local" to prefer
201 // local shards, "_primary" to execute on primary shards only,
202 // or a custom value which guarantees that the same order will be used
203 // across different requests.
204 func (s *ScrollService) Preference(preference string) *ScrollService {
205 s.preference = preference
206 return s
207 }
208
209 // IgnoreUnavailable indicates whether the specified concrete indices
210 // should be ignored when unavailable (missing or closed).
211 func (s *ScrollService) IgnoreUnavailable(ignoreUnavailable bool) *ScrollService {
212 s.ignoreUnavailable = &ignoreUnavailable
213 return s
214 }
215
216 // AllowNoIndices indicates whether to ignore if a wildcard indices
217 // expression resolves into no concrete indices. (This includes `_all` string
218 // or when no indices have been specified).
219 func (s *ScrollService) AllowNoIndices(allowNoIndices bool) *ScrollService {
220 s.allowNoIndices = &allowNoIndices
221 return s
222 }
223
224 // ExpandWildcards indicates whether to expand wildcard expression to
225 // concrete indices that are open, closed or both.
226 func (s *ScrollService) ExpandWildcards(expandWildcards string) *ScrollService {
227 s.expandWildcards = expandWildcards
228 return s
229 }
230
231 // FilterPath allows reducing the response, a mechanism known as
232 // response filtering and described here:
233 // https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#common-options-response-filtering.
234 func (s *ScrollService) FilterPath(filterPath ...string) *ScrollService {
235 s.filterPath = append(s.filterPath, filterPath...)
236 return s
237 }
238
239 // ScrollId specifies the identifier of a scroll in action.
240 func (s *ScrollService) ScrollId(scrollId string) *ScrollService {
241 s.mu.Lock()
242 s.scrollId = scrollId
243 s.mu.Unlock()
244 return s
245 }
246
247 // Do returns the next search result. It will return io.EOF as error if there
248 // are no more search results.
249 func (s *ScrollService) Do(ctx context.Context) (*SearchResult, error) {
250 s.mu.RLock()
251 nextScrollId := s.scrollId
252 s.mu.RUnlock()
253 if len(nextScrollId) == 0 {
254 return s.first(ctx)
255 }
256 return s.next(ctx)
257 }
258
259 // Clear cancels the current scroll operation. If you don't do this manually,
260 // the scroll will be expired automatically by Elasticsearch. You can control
261 // how long a scroll cursor is kept alive with the KeepAlive func.
262 func (s *ScrollService) Clear(ctx context.Context) error {
263 s.mu.RLock()
264 scrollId := s.scrollId
265 s.mu.RUnlock()
266 if len(scrollId) == 0 {
267 return nil
268 }
269
270 path := "/_search/scroll"
271 params := url.Values{}
272 body := struct {
273 ScrollId []string `json:"scroll_id,omitempty"`
274 }{
275 ScrollId: []string{scrollId},
276 }
277
278 _, err := s.client.PerformRequestWithOptions(ctx, PerformRequestOptions{
279 Method: "DELETE",
280 Path: path,
281 Params: params,
282 Body: body,
283 Retrier: s.retrier,
284 })
285 if err != nil {
286 return err
287 }
288
289 return nil
290 }
291
292 // -- First --
293
294 // first takes the first page of search results.
295 func (s *ScrollService) first(ctx context.Context) (*SearchResult, error) {
296 // Get URL and parameters for request
297 path, params, err := s.buildFirstURL()
298 if err != nil {
299 return nil, err
300 }
301
302 // Get HTTP request body
303 body, err := s.bodyFirst()
304 if err != nil {
305 return nil, err
306 }
307
308 // Get HTTP response
309 res, err := s.client.PerformRequestWithOptions(ctx, PerformRequestOptions{
310 Method: "POST",
311 Path: path,
312 Params: params,
313 Body: body,
314 Retrier: s.retrier,
315 })
316 if err != nil {
317 return nil, err
318 }
319
320 // Return operation response
321 ret := new(SearchResult)
322 if err := s.client.decoder.Decode(res.Body, ret); err != nil {
323 return nil, err
324 }
325 s.mu.Lock()
326 s.scrollId = ret.ScrollId
327 s.mu.Unlock()
328 if ret.Hits == nil || len(ret.Hits.Hits) == 0 {
329 return ret, io.EOF
330 }
331 return ret, nil
332 }
333
334 // buildFirstURL builds the URL for retrieving the first page.
335 func (s *ScrollService) buildFirstURL() (string, url.Values, error) {
336 // Build URL
337 var err error
338 var path string
339 if len(s.indices) == 0 && len(s.types) == 0 {
340 path = "/_search"
341 } else if len(s.indices) > 0 && len(s.types) == 0 {
342 path, err = uritemplates.Expand("/{index}/_search", map[string]string{
343 "index": strings.Join(s.indices, ","),
344 })
345 } else if len(s.indices) == 0 && len(s.types) > 0 {
346 path, err = uritemplates.Expand("/_all/{typ}/_search", map[string]string{
347 "typ": strings.Join(s.types, ","),
348 })
349 } else {
350 path, err = uritemplates.Expand("/{index}/{typ}/_search", map[string]string{
351 "index": strings.Join(s.indices, ","),
352 "typ": strings.Join(s.types, ","),
353 })
354 }
355 if err != nil {
356 return "", url.Values{}, err
357 }
358
359 // Add query string parameters
360 params := url.Values{}
361 if s.pretty {
362 params.Set("pretty", "1")
363 }
364 if s.size != nil && *s.size > 0 {
365 params.Set("size", fmt.Sprintf("%d", *s.size))
366 }
367 if len(s.keepAlive) > 0 {
368 params.Set("scroll", s.keepAlive)
369 }
370 if len(s.routing) > 0 {
371 params.Set("routing", s.routing)
372 }
373 if len(s.preference) > 0 {
374 params.Set("preference", s.preference)
375 }
376 if s.allowNoIndices != nil {
377 params.Set("allow_no_indices", fmt.Sprintf("%v", *s.allowNoIndices))
378 }
379 if len(s.expandWildcards) > 0 {
380 params.Set("expand_wildcards", s.expandWildcards)
381 }
382 if s.ignoreUnavailable != nil {
383 params.Set("ignore_unavailable", fmt.Sprintf("%v", *s.ignoreUnavailable))
384 }
385 if len(s.filterPath) > 0 {
386 // Always add "hits._scroll_id", otherwise we cannot scroll
387 s.filterPath = append(s.filterPath, "_scroll_id")
388 params.Set("filter_path", strings.Join(s.filterPath, ","))
389 }
390
391 return path, params, nil
392 }
393
394 // bodyFirst returns the request to fetch the first batch of results.
395 func (s *ScrollService) bodyFirst() (interface{}, error) {
396 var err error
397 var body interface{}
398
399 if s.body != nil {
400 body = s.body
401 } else {
402 // Use _doc sort by default if none is specified
403 if !s.ss.hasSort() {
404 // Use efficient sorting when no user-defined query/body is specified
405 s.ss = s.ss.SortBy(SortByDoc{})
406 }
407
408 // Body from search source
409 body, err = s.ss.Source()
410 if err != nil {
411 return nil, err
412 }
413 }
414
415 return body, nil
416 }
417
418 // -- Next --
419
420 func (s *ScrollService) next(ctx context.Context) (*SearchResult, error) {
421 // Get URL for request
422 path, params, err := s.buildNextURL()
423 if err != nil {
424 return nil, err
425 }
426
427 // Setup HTTP request body
428 body, err := s.bodyNext()
429 if err != nil {
430 return nil, err
431 }
432
433 // Get HTTP response
434 res, err := s.client.PerformRequestWithOptions(ctx, PerformRequestOptions{
435 Method: "POST",
436 Path: path,
437 Params: params,
438 Body: body,
439 Retrier: s.retrier,
440 })
441 if err != nil {
442 return nil, err
443 }
444
445 // Return operation response
446 ret := new(SearchResult)
447 if err := s.client.decoder.Decode(res.Body, ret); err != nil {
448 return nil, err
449 }
450 s.mu.Lock()
451 s.scrollId = ret.ScrollId
452 s.mu.Unlock()
453 if ret.Hits == nil || len(ret.Hits.Hits) == 0 {
454 return ret, io.EOF
455 }
456 return ret, nil
457 }
458
459 // buildNextURL builds the URL for the operation.
460 func (s *ScrollService) buildNextURL() (string, url.Values, error) {
461 path := "/_search/scroll"
462
463 // Add query string parameters
464 params := url.Values{}
465 if s.pretty {
466 params.Set("pretty", "1")
467 }
468
469 return path, params, nil
470 }
471
472 // body returns the request to fetch the next batch of results.
473 func (s *ScrollService) bodyNext() (interface{}, error) {
474 s.mu.RLock()
475 body := struct {
476 Scroll string `json:"scroll"`
477 ScrollId string `json:"scroll_id,omitempty"`
478 }{
479 Scroll: s.keepAlive,
480 ScrollId: s.scrollId,
481 }
482 s.mu.RUnlock()
483 return body, nil
484 }