Debian Codegraph @

recipes/bulk_processor/main.go

raw
1 // Copyright 2012-present Oliver Eilhard. All rights reserved.
2 // Use of this source code is governed by a MIT-license.
3 // See http://olivere.mit-license.org/license.txt for details.
4
5 // BulkProcessor runs a bulk processing job that fills an index
6 // given certain criteria like flush interval etc.
7 //
8 // Example
9 //
10 // bulk_processor -url=http://127.0.0.1:9200/bulk-processor-test?sniff=false -n=100000 -flush-interval=1s
11 //
12 package main
13
14 import (
15 "context"
16 "flag"
17 "fmt"
18 "log"
19 "math/rand"
20 "os"
21 "os/signal"
22 "sync/atomic"
23 "syscall"
24 "time"
25
26 "github.com/google/uuid"
27
28 elastic "gopkg.in/olivere/elastic.v5"
29 "gopkg.in/olivere/elastic.v5/config"
30 )
31
32 func main() {
33 var (
34 url = flag.String("url", "http://localhost:9200/bulk-processor-test", "Elasticsearch URL")
35 numWorkers = flag.Int("num-workers", 4, "Number of workers")
36 n = flag.Int64("n", -1, "Number of documents to process (-1 for unlimited)")
37 flushInterval = flag.Duration("flush-interval", 1*time.Second, "Flush interval")
38 bulkActions = flag.Int("bulk-actions", 0, "Number of bulk actions before committing")
39 bulkSize = flag.Int("bulk-size", 0, "Size of bulk requests before committing")
40 )
41 flag.Parse()
42 log.SetFlags(0)
43
44 rand.Seed(time.Now().UnixNano())
45
46 // Parse configuration from URL
47 cfg, err := config.Parse(*url)
48 if err != nil {
49 log.Fatal(err)
50 }
51
52 // Create an Elasticsearch client from the parsed config
53 client, err := elastic.NewClientFromConfig(cfg)
54 if err != nil {
55 log.Fatal(err)
56 }
57
58 // Drop old index
59 exists, err := client.IndexExists(cfg.Index).Do(context.Background())
60 if err != nil {
61 log.Fatal(err)
62 }
63 if exists {
64 _, err = client.DeleteIndex(cfg.Index).Do(context.Background())
65 if err != nil {
66 log.Fatal(err)
67 }
68 }
69
70 // Create processor
71 bulkp := elastic.NewBulkProcessorService(client).
72 Name("bulk-test-processor").
73 Stats(true).
74 Backoff(elastic.StopBackoff{}).
75 FlushInterval(*flushInterval).
76 Workers(*numWorkers)
77 if *bulkActions > 0 {
78 bulkp = bulkp.BulkActions(*bulkActions)
79 }
80 if *bulkSize > 0 {
81 bulkp = bulkp.BulkSize(*bulkSize)
82 }
83 p, err := bulkp.Do(context.Background())
84 if err != nil {
85 log.Fatal(err)
86 }
87
88 var created int64
89 errc := make(chan error, 1)
90 go func() {
91 c := make(chan os.Signal, 1)
92 signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
93 <-c
94 errc <- nil
95 }()
96
97 go func() {
98 defer func() {
99 if err := p.Close(); err != nil {
100 errc <- err
101 }
102 }()
103
104 type Doc struct {
105 Timestamp time.Time `json:"@timestamp"`
106 }
107
108 for {
109 current := atomic.AddInt64(&created, 1)
110 if *n > 0 && current >= *n {
111 errc <- nil
112 return
113 }
114 r := elastic.NewBulkIndexRequest().
115 Index(cfg.Index).
116 Type("doc").
117 Id(uuid.New().String()).
118 Doc(Doc{Timestamp: time.Now()})
119 p.Add(r)
120
121 time.Sleep(time.Duration(rand.Intn(1000)) * time.Microsecond)
122 }
123 }()
124
125 go func() {
126 t := time.NewTicker(1 * time.Second)
127 defer t.Stop()
128 for range t.C {
129 stats := p.Stats()
130 written := atomic.LoadInt64(&created)
131 var queued int64
132 for _, w := range stats.Workers {
133 queued += w.Queued
134 }
135 fmt.Printf("Queued=%5d Written=%8d Succeeded=%8d Failed=%8d Comitted=%6d Flushed=%6d\n",
136 queued,
137 written,
138 stats.Succeeded,
139 stats.Failed,
140 stats.Committed,
141 stats.Flushed,
142 )
143 }
144 }()
145
146 if err := <-errc; err != nil {
147 log.Fatal(err)
148 }
149 }