| 1 |
// Copyright 2012-present Oliver Eilhard. All rights reserved. |
| 2 |
// Use of this source code is governed by a MIT-license. |
| 3 |
// See http://olivere.mit-license.org/license.txt for details. |
| 4 |
|
| 5 |
// BulkProcessor runs a bulk processing job that fills an index |
| 6 |
// given certain criteria like flush interval etc. |
| 7 |
// |
| 8 |
// Example |
| 9 |
// |
| 10 |
// bulk_processor -url=http://127.0.0.1:9200/bulk-processor-test?sniff=false -n=100000 -flush-interval=1s |
| 11 |
// |
| 12 |
package main |
| 13 |
|
| 14 |
import ( |
| 15 |
"context" |
| 16 |
"flag" |
| 17 |
"fmt" |
| 18 |
"log" |
| 19 |
"math/rand" |
| 20 |
"os" |
| 21 |
"os/signal" |
| 22 |
"sync/atomic" |
| 23 |
"syscall" |
| 24 |
"time" |
| 25 |
|
| 26 |
"github.com/google/uuid" |
| 27 |
|
| 28 |
elastic "gopkg.in/olivere/elastic.v5" |
| 29 |
"gopkg.in/olivere/elastic.v5/config" |
| 30 |
) |
| 31 |
|
| 32 |
func main() { |
| 33 |
var ( |
| 34 |
url = flag.String("url", "http://localhost:9200/bulk-processor-test", "Elasticsearch URL") |
| 35 |
numWorkers = flag.Int("num-workers", 4, "Number of workers") |
| 36 |
n = flag.Int64("n", -1, "Number of documents to process (-1 for unlimited)") |
| 37 |
flushInterval = flag.Duration("flush-interval", 1*time.Second, "Flush interval") |
| 38 |
bulkActions = flag.Int("bulk-actions", 0, "Number of bulk actions before committing") |
| 39 |
bulkSize = flag.Int("bulk-size", 0, "Size of bulk requests before committing") |
| 40 |
) |
| 41 |
flag.Parse() |
| 42 |
log.SetFlags(0) |
| 43 |
|
| 44 |
rand.Seed(time.Now().UnixNano()) |
| 45 |
|
| 46 |
// Parse configuration from URL |
| 47 |
cfg, err := config.Parse(*url) |
| 48 |
if err != nil { |
| 49 |
log.Fatal(err) |
| 50 |
} |
| 51 |
|
| 52 |
// Create an Elasticsearch client from the parsed config |
| 53 |
client, err := elastic.NewClientFromConfig(cfg) |
| 54 |
if err != nil { |
| 55 |
log.Fatal(err) |
| 56 |
} |
| 57 |
|
| 58 |
// Drop old index |
| 59 |
exists, err := client.IndexExists(cfg.Index).Do(context.Background()) |
| 60 |
if err != nil { |
| 61 |
log.Fatal(err) |
| 62 |
} |
| 63 |
if exists { |
| 64 |
_, err = client.DeleteIndex(cfg.Index).Do(context.Background()) |
| 65 |
if err != nil { |
| 66 |
log.Fatal(err) |
| 67 |
} |
| 68 |
} |
| 69 |
|
| 70 |
// Create processor |
| 71 |
bulkp := elastic.NewBulkProcessorService(client). |
| 72 |
Name("bulk-test-processor"). |
| 73 |
Stats(true). |
| 74 |
Backoff(elastic.StopBackoff{}). |
| 75 |
FlushInterval(*flushInterval). |
| 76 |
Workers(*numWorkers) |
| 77 |
if *bulkActions > 0 { |
| 78 |
bulkp = bulkp.BulkActions(*bulkActions) |
| 79 |
} |
| 80 |
if *bulkSize > 0 { |
| 81 |
bulkp = bulkp.BulkSize(*bulkSize) |
| 82 |
} |
| 83 |
p, err := bulkp.Do(context.Background()) |
| 84 |
if err != nil { |
| 85 |
log.Fatal(err) |
| 86 |
} |
| 87 |
|
| 88 |
var created int64 |
| 89 |
errc := make(chan error, 1) |
| 90 |
go func() { |
| 91 |
c := make(chan os.Signal, 1) |
| 92 |
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) |
| 93 |
<-c |
| 94 |
errc <- nil |
| 95 |
}() |
| 96 |
|
| 97 |
go func() { |
| 98 |
defer func() { |
| 99 |
if err := p.Close(); err != nil { |
| 100 |
errc <- err |
| 101 |
} |
| 102 |
}() |
| 103 |
|
| 104 |
type Doc struct { |
| 105 |
Timestamp time.Time `json:"@timestamp"` |
| 106 |
} |
| 107 |
|
| 108 |
for { |
| 109 |
current := atomic.AddInt64(&created, 1) |
| 110 |
if *n > 0 && current >= *n { |
| 111 |
errc <- nil |
| 112 |
return |
| 113 |
} |
| 114 |
r := elastic.NewBulkIndexRequest(). |
| 115 |
Index(cfg.Index). |
| 116 |
Type("doc"). |
| 117 |
Id(uuid.New().String()). |
| 118 |
Doc(Doc{Timestamp: time.Now()}) |
| 119 |
p.Add(r) |
| 120 |
|
| 121 |
time.Sleep(time.Duration(rand.Intn(1000)) * time.Microsecond) |
| 122 |
} |
| 123 |
}() |
| 124 |
|
| 125 |
go func() { |
| 126 |
t := time.NewTicker(1 * time.Second) |
| 127 |
defer t.Stop() |
| 128 |
for range t.C { |
| 129 |
stats := p.Stats() |
| 130 |
written := atomic.LoadInt64(&created) |
| 131 |
var queued int64 |
| 132 |
for _, w := range stats.Workers { |
| 133 |
queued += w.Queued |
| 134 |
} |
| 135 |
fmt.Printf("Queued=%5d Written=%8d Succeeded=%8d Failed=%8d Comitted=%6d Flushed=%6d\n", |
| 136 |
queued, |
| 137 |
written, |
| 138 |
stats.Succeeded, |
| 139 |
stats.Failed, |
| 140 |
stats.Committed, |
| 141 |
stats.Flushed, |
| 142 |
) |
| 143 |
} |
| 144 |
}() |
| 145 |
|
| 146 |
if err := <-errc; err != nil { |
| 147 |
log.Fatal(err) |
| 148 |
} |
| 149 |
} |