Debian Codegraph @

tools/kafka-producer-performance/main.go

raw
1 package main
2
3 import (
4 "context"
5 "crypto/rand"
6 "flag"
7 "fmt"
8 "io"
9 "os"
10 "strings"
11 gosync "sync"
12 "time"
13
14 "github.com/Shopify/sarama"
15 metrics "github.com/rcrowley/go-metrics"
16 )
17
18 var (
19 sync = flag.Bool(
20 "sync",
21 false,
22 "Use a synchronous producer.",
23 )
24 messageLoad = flag.Int(
25 "message-load",
26 0,
27 "REQUIRED: The number of messages to produce to -topic.",
28 )
29 messageSize = flag.Int(
30 "message-size",
31 0,
32 "REQUIRED: The approximate size (in bytes) of each message to produce to -topic.",
33 )
34 brokers = flag.String(
35 "brokers",
36 "",
37 "REQUIRED: A comma separated list of broker addresses.",
38 )
39 topic = flag.String(
40 "topic",
41 "",
42 "REQUIRED: The topic to run the performance test on.",
43 )
44 partition = flag.Int(
45 "partition",
46 -1,
47 "The partition of -topic to run the performance test on.",
48 )
49 throughput = flag.Int(
50 "throughput",
51 0,
52 "The maximum number of messages to send per second (0 for no limit).",
53 )
54 maxMessageBytes = flag.Int(
55 "max-message-bytes",
56 1000000,
57 "The max permitted size of a message.",
58 )
59 requiredAcks = flag.Int(
60 "required-acks",
61 1,
62 "The required number of acks needed from the broker (-1: all, 0: none, 1: local).",
63 )
64 timeout = flag.Duration(
65 "timeout",
66 10*time.Second,
67 "The duration the producer will wait to receive -required-acks.",
68 )
69 partitioner = flag.String(
70 "partitioner",
71 "roundrobin",
72 "The partitioning scheme to use (hash, manual, random, roundrobin).",
73 )
74 compression = flag.String(
75 "compression",
76 "none",
77 "The compression method to use (none, gzip, snappy, lz4).",
78 )
79 flushFrequency = flag.Duration(
80 "flush-frequency",
81 0,
82 "The best-effort frequency of flushes.",
83 )
84 flushBytes = flag.Int(
85 "flush-bytes",
86 0,
87 "The best-effort number of bytes needed to trigger a flush.",
88 )
89 flushMessages = flag.Int(
90 "flush-messages",
91 0,
92 "The best-effort number of messages needed to trigger a flush.",
93 )
94 flushMaxMessages = flag.Int(
95 "flush-max-messages",
96 0,
97 "The maximum number of messages the producer will send in a single request.",
98 )
99 retryMax = flag.Int(
100 "retry-max",
101 3,
102 "The total number of times to retry sending a message.",
103 )
104 retryBackoff = flag.Duration(
105 "retry-backoff",
106 100*time.Millisecond,
107 "The duration the producer will wait for the cluster to settle between retries.",
108 )
109 clientID = flag.String(
110 "client-id",
111 "sarama",
112 "The client ID sent with every request to the brokers.",
113 )
114 channelBufferSize = flag.Int(
115 "channel-buffer-size",
116 256,
117 "The number of events to buffer in internal and external channels.",
118 )
119 routines = flag.Int(
120 "routines",
121 1,
122 "The number of routines to send the messages from (-sync only).",
123 )
124 version = flag.String(
125 "version",
126 "0.8.2.0",
127 "The assumed version of Kafka.",
128 )
129 )
130
131 func parseCompression(scheme string) sarama.CompressionCodec {
132 switch scheme {
133 case "none":
134 return sarama.CompressionNone
135 case "gzip":
136 return sarama.CompressionGZIP
137 case "snappy":
138 return sarama.CompressionSnappy
139 case "lz4":
140 return sarama.CompressionLZ4
141 default:
142 printUsageErrorAndExit(fmt.Sprintf("Unknown -compression: %s", scheme))
143 }
144 panic("should not happen")
145 }
146
147 func parsePartitioner(scheme string, partition int) sarama.PartitionerConstructor {
148 if partition < 0 && scheme == "manual" {
149 printUsageErrorAndExit("-partition must not be -1 for -partitioning=manual")
150 }
151 switch scheme {
152 case "manual":
153 return sarama.NewManualPartitioner
154 case "hash":
155 return sarama.NewHashPartitioner
156 case "random":
157 return sarama.NewRandomPartitioner
158 case "roundrobin":
159 return sarama.NewRoundRobinPartitioner
160 default:
161 printUsageErrorAndExit(fmt.Sprintf("Unknown -partitioning: %s", scheme))
162 }
163 panic("should not happen")
164 }
165
166 func parseVersion(version string) sarama.KafkaVersion {
167 result, err := sarama.ParseKafkaVersion(version)
168 if err != nil {
169 printUsageErrorAndExit(fmt.Sprintf("unknown -version: %s", version))
170 }
171 return result
172 }
173
174 func generateMessages(topic string, partition, messageLoad, messageSize int) []*sarama.ProducerMessage {
175 messages := make([]*sarama.ProducerMessage, messageLoad)
176 for i := 0; i < messageLoad; i++ {
177 payload := make([]byte, messageSize)
178 if _, err := rand.Read(payload); err != nil {
179 printErrorAndExit(69, "Failed to generate message payload: %s", err)
180 }
181 messages[i] = &sarama.ProducerMessage{
182 Topic: topic,
183 Partition: int32(partition),
184 Value: sarama.ByteEncoder(payload),
185 }
186 }
187 return messages
188 }
189
190 func main() {
191 flag.Parse()
192
193 if *brokers == "" {
194 printUsageErrorAndExit("-brokers is required")
195 }
196 if *topic == "" {
197 printUsageErrorAndExit("-topic is required")
198 }
199 if *messageLoad <= 0 {
200 printUsageErrorAndExit("-message-load must be greater than 0")
201 }
202 if *messageSize <= 0 {
203 printUsageErrorAndExit("-message-size must be greater than 0")
204 }
205 if *routines < 1 || *routines > *messageLoad {
206 printUsageErrorAndExit("-routines must be greater than 0 and less than or equal to -message-load")
207 }
208
209 config := sarama.NewConfig()
210
211 config.Producer.MaxMessageBytes = *maxMessageBytes
212 config.Producer.RequiredAcks = sarama.RequiredAcks(*requiredAcks)
213 config.Producer.Timeout = *timeout
214 config.Producer.Partitioner = parsePartitioner(*partitioner, *partition)
215 config.Producer.Compression = parseCompression(*compression)
216 config.Producer.Flush.Frequency = *flushFrequency
217 config.Producer.Flush.Bytes = *flushBytes
218 config.Producer.Flush.Messages = *flushMessages
219 config.Producer.Flush.MaxMessages = *flushMaxMessages
220 config.Producer.Return.Successes = true
221 config.ClientID = *clientID
222 config.ChannelBufferSize = *channelBufferSize
223 config.Version = parseVersion(*version)
224
225 if err := config.Validate(); err != nil {
226 printErrorAndExit(69, "Invalid configuration: %s", err)
227 }
228
229 // Print out metrics periodically.
230 done := make(chan struct{})
231 ctx, cancel := context.WithCancel(context.Background())
232 go func(ctx context.Context) {
233 defer close(done)
234 t := time.Tick(5 * time.Second)
235 for {
236 select {
237 case <-t:
238 printMetrics(os.Stdout, config.MetricRegistry)
239 case <-ctx.Done():
240 return
241 }
242 }
243 }(ctx)
244
245 brokers := strings.Split(*brokers, ",")
246 if *sync {
247 runSyncProducer(*topic, *partition, *messageLoad, *messageSize, *routines,
248 config, brokers, *throughput)
249 } else {
250 runAsyncProducer(*topic, *partition, *messageLoad, *messageSize,
251 config, brokers, *throughput)
252 }
253
254 cancel()
255 <-done
256
257 // Print final metrics.
258 printMetrics(os.Stdout, config.MetricRegistry)
259 }
260
261 func runAsyncProducer(topic string, partition, messageLoad, messageSize int,
262 config *sarama.Config, brokers []string, throughput int) {
263 producer, err := sarama.NewAsyncProducer(brokers, config)
264 if err != nil {
265 printErrorAndExit(69, "Failed to create producer: %s", err)
266 }
267 defer func() {
268 if err := producer.Close(); err != nil {
269 printErrorAndExit(69, "Failed to close producer: %s", err)
270 }
271 }()
272
273 messages := generateMessages(topic, partition, messageLoad, messageSize)
274
275 messagesDone := make(chan struct{})
276 go func() {
277 for i := 0; i < messageLoad; i++ {
278 select {
279 case <-producer.Successes():
280 case err = <-producer.Errors():
281 printErrorAndExit(69, "%s", err)
282 }
283 }
284 messagesDone <- struct{}{}
285 }()
286
287 if throughput > 0 {
288 ticker := time.NewTicker(time.Second)
289 for _, message := range messages {
290 for i := 0; i < throughput; i++ {
291 producer.Input() <- message
292 }
293 <-ticker.C
294 }
295 ticker.Stop()
296 } else {
297 for _, message := range messages {
298 producer.Input() <- message
299 }
300 }
301
302 <-messagesDone
303 close(messagesDone)
304 }
305
306 func runSyncProducer(topic string, partition, messageLoad, messageSize, routines int,
307 config *sarama.Config, brokers []string, throughput int) {
308 producer, err := sarama.NewSyncProducer(brokers, config)
309 if err != nil {
310 printErrorAndExit(69, "Failed to create producer: %s", err)
311 }
312 defer func() {
313 if err := producer.Close(); err != nil {
314 printErrorAndExit(69, "Failed to close producer: %s", err)
315 }
316 }()
317
318 messages := make([][]*sarama.ProducerMessage, routines)
319 for i := 0; i < routines; i++ {
320 if i == routines-1 {
321 messages[i] = generateMessages(topic, partition, messageLoad/routines+messageLoad%routines, messageSize)
322 } else {
323 messages[i] = generateMessages(topic, partition, messageLoad/routines, messageSize)
324 }
325 }
326
327 var wg gosync.WaitGroup
328 if throughput > 0 {
329 for _, messages := range messages {
330 messages := messages
331 wg.Add(1)
332 go func() {
333 ticker := time.NewTicker(time.Second)
334 for _, message := range messages {
335 for i := 0; i < throughput; i++ {
336 _, _, err = producer.SendMessage(message)
337 if err != nil {
338 printErrorAndExit(69, "Failed to send message: %s", err)
339 }
340 }
341 <-ticker.C
342 }
343 ticker.Stop()
344 wg.Done()
345 }()
346 }
347 } else {
348 for _, messages := range messages {
349 messages := messages
350 wg.Add(1)
351 go func() {
352 for _, message := range messages {
353 _, _, err = producer.SendMessage(message)
354 if err != nil {
355 printErrorAndExit(69, "Failed to send message: %s", err)
356 }
357 }
358 wg.Done()
359 }()
360 }
361 }
362 wg.Wait()
363 }
364
365 func printMetrics(w io.Writer, r metrics.Registry) {
366 if r.Get("record-send-rate") == nil || r.Get("request-latency-in-ms") == nil {
367 return
368 }
369 recordSendRate := r.Get("record-send-rate").(metrics.Meter).Snapshot()
370 requestLatency := r.Get("request-latency-in-ms").(metrics.Histogram).Snapshot()
371 requestLatencyPercentiles := requestLatency.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
372 fmt.Fprintf(w, "%d records sent, %.1f records/sec (%.2f MB/sec), "+
373 "%.1f ms avg latency, %.1f ms stddev, %.1f ms 50th, %.1f ms 75th, "+
374 "%.1f ms 95th, %.1f ms 99th, %.1f ms 99.9th\n",
375 recordSendRate.Count(),
376 recordSendRate.RateMean(),
377 recordSendRate.RateMean()*float64(*messageSize)/1024/1024,
378 requestLatency.Mean(),
379 requestLatency.StdDev(),
380 requestLatencyPercentiles[0],
381 requestLatencyPercentiles[1],
382 requestLatencyPercentiles[2],
383 requestLatencyPercentiles[3],
384 requestLatencyPercentiles[4],
385 )
386 }
387
388 func printUsageErrorAndExit(message string) {
389 fmt.Fprintln(os.Stderr, "ERROR:", message)
390 fmt.Fprintln(os.Stderr)
391 fmt.Fprintln(os.Stderr, "Available command line options:")
392 flag.PrintDefaults()
393 os.Exit(64)
394 }
395
396 func printErrorAndExit(code int, format string, values ...interface{}) {
397 fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
398 fmt.Fprintln(os.Stderr)
399 os.Exit(code)
400 }