Debian Codegraph @

recipes/bulk_insert/bulk_insert.go

raw
1 // Copyright 2012-present Oliver Eilhard. All rights reserved.
2 // Use of this source code is governed by a MIT-license.
3 // See http://olivere.mit-license.org/license.txt for details.
4
5 // BulkInsert illustrates how to bulk insert documents into Elasticsearch.
6 //
7 // It uses two goroutines to do so. The first creates a simple document
8 // and sends it to the second via a channel. The second goroutine collects
9 // those documents, creates a bulk request that is added to a Bulk service
10 // and committed to Elasticsearch after reaching a number of documents.
11 // The number of documents after which a commit happens can be specified
12 // via the "bulk-size" flag.
13 //
14 // See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-bulk.html
15 // for details on the Bulk API in Elasticsearch.
16 //
17 // Example
18 //
19 // Bulk index 100.000 documents into the index "warehouse", type "product",
20 // committing every set of 1.000 documents.
21 //
22 // bulk_insert -index=warehouse -type=product -n=100000 -bulk-size=1000
23 //
24 package main
25
26 import (
27 "context"
28 "encoding/base64"
29 "errors"
30 "flag"
31 "fmt"
32 "log"
33 "math/rand"
34 "sync/atomic"
35 "time"
36
37 "golang.org/x/sync/errgroup"
38 "gopkg.in/olivere/elastic.v5"
39 )
40
41 func main() {
42 var (
43 url = flag.String("url", "http://localhost:9200", "Elasticsearch URL")
44 index = flag.String("index", "", "Elasticsearch index name")
45 typ = flag.String("type", "", "Elasticsearch type name")
46 sniff = flag.Bool("sniff", true, "Enable or disable sniffing")
47 n = flag.Int("n", 0, "Number of documents to bulk insert")
48 bulkSize = flag.Int("bulk-size", 0, "Number of documents to collect before committing")
49 )
50 flag.Parse()
51 log.SetFlags(0)
52 rand.Seed(time.Now().UnixNano())
53
54 if *url == "" {
55 log.Fatal("missing url parameter")
56 }
57 if *index == "" {
58 log.Fatal("missing index parameter")
59 }
60 if *typ == "" {
61 log.Fatal("missing type parameter")
62 }
63 if *n <= 0 {
64 log.Fatal("n must be a positive number")
65 }
66 if *bulkSize <= 0 {
67 log.Fatal("bulk-size must be a positive number")
68 }
69
70 // Create an Elasticsearch client
71 client, err := elastic.NewClient(elastic.SetURL(*url), elastic.SetSniff(*sniff))
72 if err != nil {
73 log.Fatal(err)
74 }
75
76 // Setup a group of goroutines from the excellent errgroup package
77 g, ctx := errgroup.WithContext(context.TODO())
78
79 // The first goroutine will emit documents and send it to the second goroutine
80 // via the docsc channel.
81 // The second Goroutine will simply bulk insert the documents.
82 type doc struct {
83 ID string `json:"id"`
84 Timestamp time.Time `json:"@timestamp"`
85 }
86 docsc := make(chan doc)
87
88 begin := time.Now()
89
90 // Goroutine to create documents
91 g.Go(func() error {
92 defer close(docsc)
93
94 buf := make([]byte, 32)
95 for i := 0; i < *n; i++ {
96 // Generate a random ID
97 _, err := rand.Read(buf)
98 if err != nil {
99 return err
100 }
101 id := base64.URLEncoding.EncodeToString(buf)
102
103 // Construct the document
104 d := doc{
105 ID: id,
106 Timestamp: time.Now(),
107 }
108
109 // Send over to 2nd goroutine, or cancel
110 select {
111 case docsc <- d:
112 case <-ctx.Done():
113 return ctx.Err()
114 }
115 }
116 return nil
117 })
118
119 // Second goroutine will consume the documents sent from the first and bulk insert into ES
120 var total uint64
121 g.Go(func() error {
122 bulk := client.Bulk().Index(*index).Type(*typ)
123 for d := range docsc {
124 // Simple progress
125 current := atomic.AddUint64(&total, 1)
126 dur := time.Since(begin).Seconds()
127 sec := int(dur)
128 pps := int64(float64(current) / dur)
129 fmt.Printf("%10d | %6d req/s | %02d:%02d\r", current, pps, sec/60, sec%60)
130
131 // Enqueue the document
132 bulk.Add(elastic.NewBulkIndexRequest().Id(d.ID).Doc(d))
133 if bulk.NumberOfActions() >= *bulkSize {
134 // Commit
135 res, err := bulk.Do(ctx)
136 if err != nil {
137 return err
138 }
139 if res.Errors {
140 // Look up the failed documents with res.Failed(), and e.g. recommit
141 return errors.New("bulk commit failed")
142 }
143 // "bulk" is reset after Do, so you can reuse it
144 }
145
146 select {
147 default:
148 case <-ctx.Done():
149 return ctx.Err()
150 }
151 }
152
153 // Commit the final batch before exiting
154 if bulk.NumberOfActions() > 0 {
155 _, err = bulk.Do(ctx)
156 if err != nil {
157 return err
158 }
159 }
160 return nil
161 })
162
163 // Wait until all goroutines are finished
164 if err := g.Wait(); err != nil {
165 log.Fatal(err)
166 }
167
168 // Final results
169 dur := time.Since(begin).Seconds()
170 sec := int(dur)
171 pps := int64(float64(total) / dur)
172 fmt.Printf("%10d | %6d req/s | %02d:%02d\n", total, pps, sec/60, sec%60)
173 }