Debian Codegraph @

tools/kafka-producer-performance/main.go

raw
1 package main
2
3 import (
4 "context"
5 "crypto/rand"
6 "crypto/x509"
7 "flag"
8 "fmt"
9 "io"
10 "log"
11 "os"
12 "strings"
13 gosync "sync"
14 "time"
15
16 "github.com/rcrowley/go-metrics"
17
18 "github.com/IBM/sarama"
19 "github.com/IBM/sarama/tools/tls"
20 )
21
22 var (
23 sync = flag.Bool(
24 "sync",
25 false,
26 "Use a synchronous producer.",
27 )
28 messageLoad = flag.Int(
29 "message-load",
30 0,
31 "REQUIRED: The number of messages to produce to -topic.",
32 )
33 messageSize = flag.Int(
34 "message-size",
35 0,
36 "REQUIRED: The approximate size (in bytes) of each message to produce to -topic.",
37 )
38 brokers = flag.String(
39 "brokers",
40 "",
41 "REQUIRED: A comma separated list of broker addresses.",
42 )
43 securityProtocol = flag.String(
44 "security-protocol",
45 "PLAINTEXT",
46 "The name of the security protocol to talk to Kafka (PLAINTEXT, SSL).",
47 )
48 tlsRootCACerts = flag.String(
49 "tls-ca-certs",
50 "",
51 "The path to a file that contains a set of root certificate authorities in PEM format "+
52 "to trust when verifying broker certificates when -security-protocol=SSL "+
53 "(leave empty to use the host's root CA set).",
54 )
55 tlsClientCert = flag.String(
56 "tls-client-cert",
57 "",
58 "The path to a file that contains the client certificate to send to the broker "+
59 "in PEM format if client authentication is required when -security-protocol=SSL "+
60 "(leave empty to disable client authentication).",
61 )
62 tlsClientKey = flag.String(
63 "tls-client-key",
64 "",
65 "The path to a file that contains the client private key linked to the client certificate "+
66 "in PEM format when -security-protocol=SSL (REQUIRED if tls-client-cert is provided).",
67 )
68 topic = flag.String(
69 "topic",
70 "",
71 "REQUIRED: The topic to run the performance test on.",
72 )
73 partition = flag.Int(
74 "partition",
75 -1,
76 "The partition of -topic to run the performance test on.",
77 )
78 throughput = flag.Int(
79 "throughput",
80 0,
81 "The maximum number of messages to send per second (0 for no limit).",
82 )
83 maxOpenRequests = flag.Int(
84 "max-open-requests",
85 5,
86 "The maximum number of unacknowledged requests the client will send on a single connection before blocking.",
87 )
88 maxMessageBytes = flag.Int(
89 "max-message-bytes",
90 1000000,
91 "The max permitted size of a message.",
92 )
93 requiredAcks = flag.Int(
94 "required-acks",
95 1,
96 "The required number of acks needed from the broker (-1: all, 0: none, 1: local).",
97 )
98 timeout = flag.Duration(
99 "timeout",
100 10*time.Second,
101 "The duration the producer will wait to receive -required-acks.",
102 )
103 partitioner = flag.String(
104 "partitioner",
105 "roundrobin",
106 "The partitioning scheme to use (hash, manual, random, roundrobin).",
107 )
108 compression = flag.String(
109 "compression",
110 "none",
111 "The compression method to use (none, gzip, snappy, lz4).",
112 )
113 flushFrequency = flag.Duration(
114 "flush-frequency",
115 0,
116 "The best-effort frequency of flushes.",
117 )
118 flushBytes = flag.Int(
119 "flush-bytes",
120 0,
121 "The best-effort number of bytes needed to trigger a flush.",
122 )
123 flushMessages = flag.Int(
124 "flush-messages",
125 0,
126 "The best-effort number of messages needed to trigger a flush.",
127 )
128 flushMaxMessages = flag.Int(
129 "flush-max-messages",
130 0,
131 "The maximum number of messages the producer will send in a single request.",
132 )
133 clientID = flag.String(
134 "client-id",
135 "sarama",
136 "The client ID sent with every request to the brokers.",
137 )
138 channelBufferSize = flag.Int(
139 "channel-buffer-size",
140 256,
141 "The number of events to buffer in internal and external channels.",
142 )
143 routines = flag.Int(
144 "routines",
145 1,
146 "The number of routines to send the messages from (-sync only).",
147 )
148 version = flag.String(
149 "version",
150 "0.8.2.0",
151 "The assumed version of Kafka.",
152 )
153 verbose = flag.Bool(
154 "verbose",
155 false,
156 "Turn on sarama logging to stderr",
157 )
158 )
159
160 func parseCompression(scheme string) sarama.CompressionCodec {
161 switch scheme {
162 case "none":
163 return sarama.CompressionNone
164 case "gzip":
165 return sarama.CompressionGZIP
166 case "snappy":
167 return sarama.CompressionSnappy
168 case "lz4":
169 return sarama.CompressionLZ4
170 default:
171 printUsageErrorAndExit(fmt.Sprintf("Unknown -compression: %s", scheme))
172 }
173 panic("should not happen")
174 }
175
176 func parsePartitioner(scheme string, partition int) sarama.PartitionerConstructor {
177 if partition < 0 && scheme == "manual" {
178 printUsageErrorAndExit("-partition must not be -1 for -partitioning=manual")
179 }
180 switch scheme {
181 case "manual":
182 return sarama.NewManualPartitioner
183 case "hash":
184 return sarama.NewHashPartitioner
185 case "random":
186 return sarama.NewRandomPartitioner
187 case "roundrobin":
188 return sarama.NewRoundRobinPartitioner
189 default:
190 printUsageErrorAndExit(fmt.Sprintf("Unknown -partitioning: %s", scheme))
191 }
192 panic("should not happen")
193 }
194
195 func parseVersion(version string) sarama.KafkaVersion {
196 result, err := sarama.ParseKafkaVersion(version)
197 if err != nil {
198 printUsageErrorAndExit(fmt.Sprintf("unknown -version: %s", version))
199 }
200 return result
201 }
202
203 func generateMessages(topic string, partition, messageLoad, messageSize int) []*sarama.ProducerMessage {
204 messages := make([]*sarama.ProducerMessage, messageLoad)
205 for i := 0; i < messageLoad; i++ {
206 payload := make([]byte, messageSize)
207 if _, err := rand.Read(payload); err != nil {
208 printErrorAndExit(69, "Failed to generate message payload: %s", err)
209 }
210 messages[i] = &sarama.ProducerMessage{
211 Topic: topic,
212 Partition: int32(partition),
213 Value: sarama.ByteEncoder(payload),
214 }
215 }
216 return messages
217 }
218
219 func main() {
220 flag.Parse()
221
222 if *brokers == "" {
223 printUsageErrorAndExit("-brokers is required")
224 }
225 if *topic == "" {
226 printUsageErrorAndExit("-topic is required")
227 }
228 if *messageLoad <= 0 {
229 printUsageErrorAndExit("-message-load must be greater than 0")
230 }
231 if *messageSize <= 0 {
232 printUsageErrorAndExit("-message-size must be greater than 0")
233 }
234 if *routines < 1 || *routines > *messageLoad {
235 printUsageErrorAndExit("-routines must be greater than 0 and less than or equal to -message-load")
236 }
237 if *securityProtocol != "PLAINTEXT" && *securityProtocol != "SSL" {
238 printUsageErrorAndExit(fmt.Sprintf("-security-protocol %q is not supported", *securityProtocol))
239 }
240 if *verbose {
241 sarama.Logger = log.New(os.Stderr, "", log.LstdFlags)
242 }
243
244 config := sarama.NewConfig()
245
246 config.Net.MaxOpenRequests = *maxOpenRequests
247 config.Producer.MaxMessageBytes = *maxMessageBytes
248 config.Producer.RequiredAcks = sarama.RequiredAcks(*requiredAcks)
249 config.Producer.Timeout = *timeout
250 config.Producer.Partitioner = parsePartitioner(*partitioner, *partition)
251 config.Producer.Compression = parseCompression(*compression)
252 config.Producer.Flush.Frequency = *flushFrequency
253 config.Producer.Flush.Bytes = *flushBytes
254 config.Producer.Flush.Messages = *flushMessages
255 config.Producer.Flush.MaxMessages = *flushMaxMessages
256 config.Producer.Return.Successes = true
257 config.ClientID = *clientID
258 config.ChannelBufferSize = *channelBufferSize
259 config.Version = parseVersion(*version)
260
261 if *securityProtocol == "SSL" {
262 tlsConfig, err := tls.NewConfig(*tlsClientCert, *tlsClientKey)
263 if err != nil {
264 printErrorAndExit(69, "failed to load client certificate from: %s and private key from: %s: %v",
265 *tlsClientCert, *tlsClientKey, err)
266 }
267
268 if *tlsRootCACerts != "" {
269 rootCAsBytes, err := os.ReadFile(*tlsRootCACerts)
270 if err != nil {
271 printErrorAndExit(69, "failed to read root CA certificates: %v", err)
272 }
273 certPool := x509.NewCertPool()
274 if !certPool.AppendCertsFromPEM(rootCAsBytes) {
275 printErrorAndExit(69, "failed to load root CA certificates from file: %s", *tlsRootCACerts)
276 }
277 // Use specific root CA set vs the host's set
278 tlsConfig.RootCAs = certPool
279 }
280
281 config.Net.TLS.Enable = true
282 config.Net.TLS.Config = tlsConfig
283 }
284
285 if err := config.Validate(); err != nil {
286 printErrorAndExit(69, "Invalid configuration: %s", err)
287 }
288
289 // Print out metrics periodically.
290 done := make(chan struct{})
291 ctx, cancel := context.WithCancel(context.Background())
292 go func(ctx context.Context) {
293 defer close(done)
294 t := time.Tick(5 * time.Second)
295 for {
296 select {
297 case <-t:
298 printMetrics(os.Stdout, config.MetricRegistry)
299 case <-ctx.Done():
300 return
301 }
302 }
303 }(ctx)
304
305 brokers := strings.Split(*brokers, ",")
306 if *sync {
307 runSyncProducer(*topic, *partition, *messageLoad, *messageSize, *routines,
308 config, brokers, *throughput)
309 } else {
310 runAsyncProducer(*topic, *partition, *messageLoad, *messageSize,
311 config, brokers, *throughput)
312 }
313
314 cancel()
315 <-done
316 }
317
318 func runAsyncProducer(topic string, partition, messageLoad, messageSize int,
319 config *sarama.Config, brokers []string, throughput int) {
320 producer, err := sarama.NewAsyncProducer(brokers, config)
321 if err != nil {
322 printErrorAndExit(69, "Failed to create producer: %s", err)
323 }
324 defer func() {
325 // Print final metrics.
326 printMetrics(os.Stdout, config.MetricRegistry)
327 if err := producer.Close(); err != nil {
328 printErrorAndExit(69, "Failed to close producer: %s", err)
329 }
330 }()
331
332 messages := generateMessages(topic, partition, messageLoad, messageSize)
333
334 messagesDone := make(chan struct{})
335 go func() {
336 for i := 0; i < messageLoad; i++ {
337 select {
338 case <-producer.Successes():
339 case err = <-producer.Errors():
340 printErrorAndExit(69, "%s", err)
341 }
342 }
343 messagesDone <- struct{}{}
344 }()
345
346 if throughput > 0 {
347 ticker := time.NewTicker(time.Second)
348 for idx, message := range messages {
349 producer.Input() <- message
350 if (idx+1)%throughput == 0 {
351 <-ticker.C
352 }
353 }
354 ticker.Stop()
355 } else {
356 for _, message := range messages {
357 producer.Input() <- message
358 }
359 }
360
361 <-messagesDone
362 close(messagesDone)
363 }
364
365 func runSyncProducer(topic string, partition, messageLoad, messageSize, routines int,
366 config *sarama.Config, brokers []string, throughput int) {
367 producer, err := sarama.NewSyncProducer(brokers, config)
368 if err != nil {
369 printErrorAndExit(69, "Failed to create producer: %s", err)
370 }
371 defer func() {
372 // Print final metrics.
373 printMetrics(os.Stdout, config.MetricRegistry)
374 if err := producer.Close(); err != nil {
375 printErrorAndExit(69, "Failed to close producer: %s", err)
376 }
377 }()
378
379 messages := make([][]*sarama.ProducerMessage, routines)
380 for i := 0; i < routines; i++ {
381 if i == routines-1 {
382 messages[i] = generateMessages(topic, partition, messageLoad/routines+messageLoad%routines, messageSize)
383 } else {
384 messages[i] = generateMessages(topic, partition, messageLoad/routines, messageSize)
385 }
386 }
387
388 var wg gosync.WaitGroup
389 if throughput > 0 {
390 for _, messages := range messages {
391 wg.Add(1)
392 go func() {
393 ticker := time.NewTicker(time.Second)
394 for _, message := range messages {
395 for i := 0; i < throughput; i++ {
396 _, _, err = producer.SendMessage(message)
397 if err != nil {
398 printErrorAndExit(69, "Failed to send message: %s", err)
399 }
400 }
401 <-ticker.C
402 }
403 ticker.Stop()
404 wg.Done()
405 }()
406 }
407 } else {
408 for _, messages := range messages {
409 wg.Add(1)
410 go func() {
411 for _, message := range messages {
412 _, _, err = producer.SendMessage(message)
413 if err != nil {
414 printErrorAndExit(69, "Failed to send message: %s", err)
415 }
416 }
417 wg.Done()
418 }()
419 }
420 }
421 wg.Wait()
422 }
423
424 func printMetrics(w io.Writer, r metrics.Registry) {
425 recordSendRateMetric := r.Get("record-send-rate")
426 requestLatencyMetric := r.Get("request-latency-in-ms")
427 outgoingByteRateMetric := r.Get("outgoing-byte-rate")
428 requestsInFlightMetric := r.Get("requests-in-flight")
429
430 if recordSendRateMetric == nil || requestLatencyMetric == nil || outgoingByteRateMetric == nil ||
431 requestsInFlightMetric == nil {
432 return
433 }
434 recordSendRate := recordSendRateMetric.(metrics.Meter).Snapshot()
435 requestLatency := requestLatencyMetric.(metrics.Histogram).Snapshot()
436 requestLatencyPercentiles := requestLatency.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
437 outgoingByteRate := outgoingByteRateMetric.(metrics.Meter).Snapshot()
438 requestsInFlight := requestsInFlightMetric.(metrics.Counter).Count()
439 fmt.Fprintf(w, "%d records sent, %.1f records/sec (%.2f MiB/sec ingress, %.2f MiB/sec egress), "+
440 "%.1f ms avg latency, %.1f ms stddev, %.1f ms 50th, %.1f ms 75th, "+
441 "%.1f ms 95th, %.1f ms 99th, %.1f ms 99.9th, %d total req. in flight\n",
442 recordSendRate.Count(),
443 recordSendRate.RateMean(),
444 recordSendRate.RateMean()*float64(*messageSize)/1024/1024,
445 outgoingByteRate.RateMean()/1024/1024,
446 requestLatency.Mean(),
447 requestLatency.StdDev(),
448 requestLatencyPercentiles[0],
449 requestLatencyPercentiles[1],
450 requestLatencyPercentiles[2],
451 requestLatencyPercentiles[3],
452 requestLatencyPercentiles[4],
453 requestsInFlight,
454 )
455 }
456
457 func printUsageErrorAndExit(message string) {
458 fmt.Fprintln(os.Stderr, "ERROR:", message)
459 fmt.Fprintln(os.Stderr)
460 fmt.Fprintln(os.Stderr, "Available command line options:")
461 flag.PrintDefaults()
462 os.Exit(64)
463 }
464
465 func printErrorAndExit(code int, format string, values ...interface{}) {
466 fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
467 fmt.Fprintln(os.Stderr)
468 os.Exit(code)
469 }