Debian Codegraph @

reindex.go

raw
1 // Copyright 2012-present Oliver Eilhard. All rights reserved.
2 // Use of this source code is governed by a MIT-license.
3 // See http://olivere.mit-license.org/license.txt for details.
4
5 package elastic
6
7 import (
8 "context"
9 "fmt"
10 "net/url"
11 )
12
13 // ReindexService is a method to copy documents from one index to another.
14 // It is documented at https://www.elastic.co/guide/en/elasticsearch/reference/5.0/docs-reindex.html.
15 type ReindexService struct {
16 client *Client
17 pretty bool
18 refresh string
19 timeout string
20 waitForActiveShards string
21 waitForCompletion *bool
22 requestsPerSecond *int
23 slices *int
24 body interface{}
25 source *ReindexSource
26 destination *ReindexDestination
27 conflicts string
28 size *int
29 script *Script
30 }
31
32 // NewReindexService creates a new ReindexService.
33 func NewReindexService(client *Client) *ReindexService {
34 return &ReindexService{
35 client: client,
36 }
37 }
38
39 // WaitForActiveShards sets the number of shard copies that must be active before
40 // proceeding with the reindex operation. Defaults to 1, meaning the primary shard only.
41 // Set to `all` for all shard copies, otherwise set to any non-negative value less than or
42 // equal to the total number of copies for the shard (number of replicas + 1).
43 func (s *ReindexService) WaitForActiveShards(waitForActiveShards string) *ReindexService {
44 s.waitForActiveShards = waitForActiveShards
45 return s
46 }
47
48 // RequestsPerSecond specifies the throttle to set on this request in sub-requests per second.
49 // -1 means set no throttle as does "unlimited" which is the only non-float this accepts.
50 func (s *ReindexService) RequestsPerSecond(requestsPerSecond int) *ReindexService {
51 s.requestsPerSecond = &requestsPerSecond
52 return s
53 }
54
55 // Slices specifies the number of slices this task should be divided into. Defaults to 1.
56 func (s *ReindexService) Slices(slices int) *ReindexService {
57 s.slices = &slices
58 return s
59 }
60
61 // Refresh indicates whether Elasticsearch should refresh the effected indexes
62 // immediately.
63 func (s *ReindexService) Refresh(refresh string) *ReindexService {
64 s.refresh = refresh
65 return s
66 }
67
68 // Timeout is the time each individual bulk request should wait for shards
69 // that are unavailable.
70 func (s *ReindexService) Timeout(timeout string) *ReindexService {
71 s.timeout = timeout
72 return s
73 }
74
75 // WaitForCompletion indicates whether Elasticsearch should block until the
76 // reindex is complete.
77 func (s *ReindexService) WaitForCompletion(waitForCompletion bool) *ReindexService {
78 s.waitForCompletion = &waitForCompletion
79 return s
80 }
81
82 // Pretty indicates that the JSON response be indented and human readable.
83 func (s *ReindexService) Pretty(pretty bool) *ReindexService {
84 s.pretty = pretty
85 return s
86 }
87
88 // Source specifies the source of the reindexing process.
89 func (s *ReindexService) Source(source *ReindexSource) *ReindexService {
90 s.source = source
91 return s
92 }
93
94 // SourceIndex specifies the source index of the reindexing process.
95 func (s *ReindexService) SourceIndex(index string) *ReindexService {
96 if s.source == nil {
97 s.source = NewReindexSource()
98 }
99 s.source = s.source.Index(index)
100 return s
101 }
102
103 // Destination specifies the destination of the reindexing process.
104 func (s *ReindexService) Destination(destination *ReindexDestination) *ReindexService {
105 s.destination = destination
106 return s
107 }
108
109 // DestinationIndex specifies the destination index of the reindexing process.
110 func (s *ReindexService) DestinationIndex(index string) *ReindexService {
111 if s.destination == nil {
112 s.destination = NewReindexDestination()
113 }
114 s.destination = s.destination.Index(index)
115 return s
116 }
117
118 // DestinationIndexAndType specifies both the destination index and type
119 // of the reindexing process.
120 func (s *ReindexService) DestinationIndexAndType(index, typ string) *ReindexService {
121 if s.destination == nil {
122 s.destination = NewReindexDestination()
123 }
124 s.destination = s.destination.Index(index)
125 s.destination = s.destination.Type(typ)
126 return s
127 }
128
129 // Conflicts indicates what to do when the process detects version conflicts.
130 // Possible values are "proceed" and "abort".
131 func (s *ReindexService) Conflicts(conflicts string) *ReindexService {
132 s.conflicts = conflicts
133 return s
134 }
135
136 // AbortOnVersionConflict aborts the request on version conflicts.
137 // It is an alias to setting Conflicts("abort").
138 func (s *ReindexService) AbortOnVersionConflict() *ReindexService {
139 s.conflicts = "abort"
140 return s
141 }
142
143 // ProceedOnVersionConflict aborts the request on version conflicts.
144 // It is an alias to setting Conflicts("proceed").
145 func (s *ReindexService) ProceedOnVersionConflict() *ReindexService {
146 s.conflicts = "proceed"
147 return s
148 }
149
150 // Size sets an upper limit for the number of processed documents.
151 func (s *ReindexService) Size(size int) *ReindexService {
152 s.size = &size
153 return s
154 }
155
156 // Script allows for modification of the documents as they are reindexed
157 // from source to destination.
158 func (s *ReindexService) Script(script *Script) *ReindexService {
159 s.script = script
160 return s
161 }
162
163 // Body specifies the body of the request to send to Elasticsearch.
164 // It overrides settings specified with other setters, e.g. Query.
165 func (s *ReindexService) Body(body interface{}) *ReindexService {
166 s.body = body
167 return s
168 }
169
170 // buildURL builds the URL for the operation.
171 func (s *ReindexService) buildURL() (string, url.Values, error) {
172 // Build URL path
173 path := "/_reindex"
174
175 // Add query string parameters
176 params := url.Values{}
177 if s.pretty {
178 params.Set("pretty", "1")
179 }
180 if s.refresh != "" {
181 params.Set("refresh", s.refresh)
182 }
183 if s.timeout != "" {
184 params.Set("timeout", s.timeout)
185 }
186 if s.requestsPerSecond != nil {
187 params.Set("requests_per_second", fmt.Sprintf("%v", *s.requestsPerSecond))
188 }
189 if s.slices != nil {
190 params.Set("slices", fmt.Sprintf("%v", *s.slices))
191 }
192 if s.waitForActiveShards != "" {
193 params.Set("wait_for_active_shards", s.waitForActiveShards)
194 }
195 if s.waitForCompletion != nil {
196 params.Set("wait_for_completion", fmt.Sprintf("%v", *s.waitForCompletion))
197 }
198 return path, params, nil
199 }
200
201 // Validate checks if the operation is valid.
202 func (s *ReindexService) Validate() error {
203 var invalid []string
204 if s.body != nil {
205 return nil
206 }
207 if s.source == nil {
208 invalid = append(invalid, "Source")
209 } else {
210 if len(s.source.indices) == 0 {
211 invalid = append(invalid, "Source.Index")
212 }
213 }
214 if s.destination == nil {
215 invalid = append(invalid, "Destination")
216 }
217 if len(invalid) > 0 {
218 return fmt.Errorf("missing required fields: %v", invalid)
219 }
220 return nil
221 }
222
223 // getBody returns the body part of the document request.
224 func (s *ReindexService) getBody() (interface{}, error) {
225 if s.body != nil {
226 return s.body, nil
227 }
228
229 body := make(map[string]interface{})
230
231 if s.conflicts != "" {
232 body["conflicts"] = s.conflicts
233 }
234 if s.size != nil {
235 body["size"] = *s.size
236 }
237 if s.script != nil {
238 out, err := s.script.Source()
239 if err != nil {
240 return nil, err
241 }
242 body["script"] = out
243 }
244
245 src, err := s.source.Source()
246 if err != nil {
247 return nil, err
248 }
249 body["source"] = src
250
251 dst, err := s.destination.Source()
252 if err != nil {
253 return nil, err
254 }
255 body["dest"] = dst
256
257 return body, nil
258 }
259
260 // Do executes the operation.
261 func (s *ReindexService) Do(ctx context.Context) (*BulkIndexByScrollResponse, error) {
262 // Check pre-conditions
263 if err := s.Validate(); err != nil {
264 return nil, err
265 }
266
267 // Get URL for request
268 path, params, err := s.buildURL()
269 if err != nil {
270 return nil, err
271 }
272
273 // Setup HTTP request body
274 body, err := s.getBody()
275 if err != nil {
276 return nil, err
277 }
278
279 // Get HTTP response
280 res, err := s.client.PerformRequest(ctx, "POST", path, params, body)
281 if err != nil {
282 return nil, err
283 }
284
285 // Return operation response
286 ret := new(BulkIndexByScrollResponse)
287 if err := s.client.decoder.Decode(res.Body, ret); err != nil {
288 return nil, err
289 }
290 return ret, nil
291 }
292
293 // DoAsync executes the reindexing operation asynchronously by starting a new task.
294 // Callers need to use the Task Management API to watch the outcome of the reindexing
295 // operation.
296 func (s *ReindexService) DoAsync(ctx context.Context) (*StartTaskResult, error) {
297 // Check pre-conditions
298 if err := s.Validate(); err != nil {
299 return nil, err
300 }
301
302 // DoAsync only makes sense with WaitForCompletion set to true
303 if s.waitForCompletion != nil && *s.waitForCompletion {
304 return nil, fmt.Errorf("cannot start a task with WaitForCompletion set to true")
305 }
306 f := false
307 s.waitForCompletion = &f
308
309 // Get URL for request
310 path, params, err := s.buildURL()
311 if err != nil {
312 return nil, err
313 }
314
315 // Setup HTTP request body
316 body, err := s.getBody()
317 if err != nil {
318 return nil, err
319 }
320
321 // Get HTTP response
322 res, err := s.client.PerformRequest(ctx, "POST", path, params, body)
323 if err != nil {
324 return nil, err
325 }
326
327 // Return operation response
328 ret := new(StartTaskResult)
329 if err := s.client.decoder.Decode(res.Body, ret); err != nil {
330 return nil, err
331 }
332 return ret, nil
333 }
334
335 // -- Source of Reindex --
336
337 // ReindexSource specifies the source of a Reindex process.
338 type ReindexSource struct {
339 searchType string
340 indices []string
341 types []string
342 routing *string
343 preference *string
344 requestCache *bool
345 scroll string
346 query Query
347 sorts []SortInfo
348 sorters []Sorter
349 searchSource *SearchSource
350 remoteInfo *ReindexRemoteInfo
351 }
352
353 // NewReindexSource creates a new ReindexSource.
354 func NewReindexSource() *ReindexSource {
355 return &ReindexSource{}
356 }
357
358 // SearchType is the search operation type. Possible values are
359 // "query_then_fetch" and "dfs_query_then_fetch".
360 func (r *ReindexSource) SearchType(searchType string) *ReindexSource {
361 r.searchType = searchType
362 return r
363 }
364
365 func (r *ReindexSource) SearchTypeDfsQueryThenFetch() *ReindexSource {
366 return r.SearchType("dfs_query_then_fetch")
367 }
368
369 func (r *ReindexSource) SearchTypeQueryThenFetch() *ReindexSource {
370 return r.SearchType("query_then_fetch")
371 }
372
373 func (r *ReindexSource) Index(indices ...string) *ReindexSource {
374 r.indices = append(r.indices, indices...)
375 return r
376 }
377
378 func (r *ReindexSource) Type(types ...string) *ReindexSource {
379 r.types = append(r.types, types...)
380 return r
381 }
382
383 func (r *ReindexSource) Preference(preference string) *ReindexSource {
384 r.preference = &preference
385 return r
386 }
387
388 func (r *ReindexSource) RequestCache(requestCache bool) *ReindexSource {
389 r.requestCache = &requestCache
390 return r
391 }
392
393 func (r *ReindexSource) Scroll(scroll string) *ReindexSource {
394 r.scroll = scroll
395 return r
396 }
397
398 func (r *ReindexSource) Query(query Query) *ReindexSource {
399 r.query = query
400 return r
401 }
402
403 // Sort adds a sort order.
404 func (s *ReindexSource) Sort(field string, ascending bool) *ReindexSource {
405 s.sorts = append(s.sorts, SortInfo{Field: field, Ascending: ascending})
406 return s
407 }
408
409 // SortWithInfo adds a sort order.
410 func (s *ReindexSource) SortWithInfo(info SortInfo) *ReindexSource {
411 s.sorts = append(s.sorts, info)
412 return s
413 }
414
415 // SortBy adds a sort order.
416 func (s *ReindexSource) SortBy(sorter ...Sorter) *ReindexSource {
417 s.sorters = append(s.sorters, sorter...)
418 return s
419 }
420
421 // RemoteInfo sets up reindexing from a remote cluster.
422 func (s *ReindexSource) RemoteInfo(ri *ReindexRemoteInfo) *ReindexSource {
423 s.remoteInfo = ri
424 return s
425 }
426
427 // Source returns a serializable JSON request for the request.
428 func (r *ReindexSource) Source() (interface{}, error) {
429 source := make(map[string]interface{})
430
431 if r.query != nil {
432 src, err := r.query.Source()
433 if err != nil {
434 return nil, err
435 }
436 source["query"] = src
437 } else if r.searchSource != nil {
438 src, err := r.searchSource.Source()
439 if err != nil {
440 return nil, err
441 }
442 source["source"] = src
443 }
444
445 if r.searchType != "" {
446 source["search_type"] = r.searchType
447 }
448
449 switch len(r.indices) {
450 case 0:
451 case 1:
452 source["index"] = r.indices[0]
453 default:
454 source["index"] = r.indices
455 }
456
457 switch len(r.types) {
458 case 0:
459 case 1:
460 source["type"] = r.types[0]
461 default:
462 source["type"] = r.types
463 }
464
465 if r.preference != nil && *r.preference != "" {
466 source["preference"] = *r.preference
467 }
468
469 if r.requestCache != nil {
470 source["request_cache"] = fmt.Sprintf("%v", *r.requestCache)
471 }
472
473 if r.scroll != "" {
474 source["scroll"] = r.scroll
475 }
476
477 if r.remoteInfo != nil {
478 src, err := r.remoteInfo.Source()
479 if err != nil {
480 return nil, err
481 }
482 source["remote"] = src
483 }
484
485 if len(r.sorters) > 0 {
486 var sortarr []interface{}
487 for _, sorter := range r.sorters {
488 src, err := sorter.Source()
489 if err != nil {
490 return nil, err
491 }
492 sortarr = append(sortarr, src)
493 }
494 source["sort"] = sortarr
495 } else if len(r.sorts) > 0 {
496 var sortarr []interface{}
497 for _, sort := range r.sorts {
498 src, err := sort.Source()
499 if err != nil {
500 return nil, err
501 }
502 sortarr = append(sortarr, src)
503 }
504 source["sort"] = sortarr
505 }
506
507 return source, nil
508 }
509
510 // ReindexRemoteInfo contains information for reindexing from a remote cluster.
511 type ReindexRemoteInfo struct {
512 host string
513 username string
514 password string
515 socketTimeout string // e.g. "1m" or "30s"
516 connectTimeout string // e.g. "1m" or "30s"
517 }
518
519 // NewReindexRemoteInfo creates a new ReindexRemoteInfo.
520 func NewReindexRemoteInfo() *ReindexRemoteInfo {
521 return &ReindexRemoteInfo{}
522 }
523
524 // Host sets the host information of the remote cluster.
525 // It must be of the form "http(s)://<hostname>:<port>"
526 func (ri *ReindexRemoteInfo) Host(host string) *ReindexRemoteInfo {
527 ri.host = host
528 return ri
529 }
530
531 // Username sets the username to authenticate with the remote cluster.
532 func (ri *ReindexRemoteInfo) Username(username string) *ReindexRemoteInfo {
533 ri.username = username
534 return ri
535 }
536
537 // Password sets the password to authenticate with the remote cluster.
538 func (ri *ReindexRemoteInfo) Password(password string) *ReindexRemoteInfo {
539 ri.password = password
540 return ri
541 }
542
543 // SocketTimeout sets the socket timeout to connect with the remote cluster.
544 // Use ES compatible values like e.g. "30s" or "1m".
545 func (ri *ReindexRemoteInfo) SocketTimeout(timeout string) *ReindexRemoteInfo {
546 ri.socketTimeout = timeout
547 return ri
548 }
549
550 // ConnectTimeout sets the connection timeout to connect with the remote cluster.
551 // Use ES compatible values like e.g. "30s" or "1m".
552 func (ri *ReindexRemoteInfo) ConnectTimeout(timeout string) *ReindexRemoteInfo {
553 ri.connectTimeout = timeout
554 return ri
555 }
556
557 // Source returns the serializable JSON data for the request.
558 func (ri *ReindexRemoteInfo) Source() (interface{}, error) {
559 res := make(map[string]interface{})
560 res["host"] = ri.host
561 if len(ri.username) > 0 {
562 res["username"] = ri.username
563 }
564 if len(ri.password) > 0 {
565 res["password"] = ri.password
566 }
567 if len(ri.socketTimeout) > 0 {
568 res["socket_timeout"] = ri.socketTimeout
569 }
570 if len(ri.connectTimeout) > 0 {
571 res["connect_timeout"] = ri.connectTimeout
572 }
573 return res, nil
574 }
575
576 // -source Destination of Reindex --
577
578 // ReindexDestination is the destination of a Reindex API call.
579 // It is basically the meta data of a BulkIndexRequest.
580 //
581 // See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-reindex.html
582 // fsourcer details.
583 type ReindexDestination struct {
584 index string
585 typ string
586 routing string
587 parent string
588 opType string
589 version int64 // default is MATCH_ANY
590 versionType string // default is "internal"
591 }
592
593 // NewReindexDestination returns a new ReindexDestination.
594 func NewReindexDestination() *ReindexDestination {
595 return &ReindexDestination{}
596 }
597
598 // Index specifies name of the Elasticsearch index to use as the destination
599 // of a reindexing process.
600 func (r *ReindexDestination) Index(index string) *ReindexDestination {
601 r.index = index
602 return r
603 }
604
605 // Type specifies the Elasticsearch type to use for reindexing.
606 func (r *ReindexDestination) Type(typ string) *ReindexDestination {
607 r.typ = typ
608 return r
609 }
610
611 // Routing specifies a routing value for the reindexing request.
612 // It can be "keep", "discard", or start with "=". The latter specifies
613 // the routing on the bulk request.
614 func (r *ReindexDestination) Routing(routing string) *ReindexDestination {
615 r.routing = routing
616 return r
617 }
618
619 // Keep sets the routing on the bulk request sent for each match to the routing
620 // of the match (the default).
621 func (r *ReindexDestination) Keep() *ReindexDestination {
622 r.routing = "keep"
623 return r
624 }
625
626 // Discard sets the routing on the bulk request sent for each match to null.
627 func (r *ReindexDestination) Discard() *ReindexDestination {
628 r.routing = "discard"
629 return r
630 }
631
632 // Parent specifies the identifier of the parent document (if available).
633 func (r *ReindexDestination) Parent(parent string) *ReindexDestination {
634 r.parent = parent
635 return r
636 }
637
638 // OpType specifies if this request should follow create-only or upsert
639 // behavior. This follows the OpType of the standard document index API.
640 // See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-index_.html#operation-type
641 // for details.
642 func (r *ReindexDestination) OpType(opType string) *ReindexDestination {
643 r.opType = opType
644 return r
645 }
646
647 // Version indicates the version of the document as part of an optimistic
648 // concurrency model.
649 func (r *ReindexDestination) Version(version int64) *ReindexDestination {
650 r.version = version
651 return r
652 }
653
654 // VersionType specifies how versions are created.
655 func (r *ReindexDestination) VersionType(versionType string) *ReindexDestination {
656 r.versionType = versionType
657 return r
658 }
659
660 // Source returns a serializable JSON request for the request.
661 func (r *ReindexDestination) Source() (interface{}, error) {
662 source := make(map[string]interface{})
663 if r.index != "" {
664 source["index"] = r.index
665 }
666 if r.typ != "" {
667 source["type"] = r.typ
668 }
669 if r.routing != "" {
670 source["routing"] = r.routing
671 }
672 if r.opType != "" {
673 source["op_type"] = r.opType
674 }
675 if r.parent != "" {
676 source["parent"] = r.parent
677 }
678 if r.version > 0 {
679 source["version"] = r.version
680 }
681 if r.versionType != "" {
682 source["version_type"] = r.versionType
683 }
684 return source, nil
685 }