Debian Codegraph @

tools/kafka-console-consumer/kafka-console-consumer.go

raw
1 package main
2
3 import (
4 "flag"
5 "fmt"
6 "log"
7 "os"
8 "os/signal"
9 "strconv"
10 "strings"
11 "sync"
12
13 "github.com/Shopify/sarama"
14 "github.com/Shopify/sarama/tools/tls"
15 )
16
17 var (
18 brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster")
19 topic = flag.String("topic", "", "REQUIRED: the topic to consume")
20 partitions = flag.String("partitions", "all", "The partitions to consume, can be 'all' or comma-separated numbers")
21 offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`")
22 verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging")
23 tlsEnabled = flag.Bool("tls-enabled", false, "Whether to enable TLS")
24 tlsSkipVerify = flag.Bool("tls-skip-verify", false, "Whether skip TLS server cert verification")
25 tlsClientCert = flag.String("tls-client-cert", "", "Client cert for client authentication (use with -tls-enabled and -tls-client-key)")
26 tlsClientKey = flag.String("tls-client-key", "", "Client key for client authentication (use with tls-enabled and -tls-client-cert)")
27
28 bufferSize = flag.Int("buffer-size", 256, "The buffer size of the message channel.")
29
30 logger = log.New(os.Stderr, "", log.LstdFlags)
31 )
32
33 func main() {
34 flag.Parse()
35
36 if *brokerList == "" {
37 printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.")
38 }
39
40 if *topic == "" {
41 printUsageErrorAndExit("-topic is required")
42 }
43
44 if *verbose {
45 sarama.Logger = logger
46 }
47
48 var initialOffset int64
49 switch *offset {
50 case "oldest":
51 initialOffset = sarama.OffsetOldest
52 case "newest":
53 initialOffset = sarama.OffsetNewest
54 default:
55 printUsageErrorAndExit("-offset should be `oldest` or `newest`")
56 }
57
58 config := sarama.NewConfig()
59 if *tlsEnabled {
60 tlsConfig, err := tls.NewConfig(*tlsClientCert, *tlsClientKey)
61 if err != nil {
62 printErrorAndExit(69, "Failed to create TLS config: %s", err)
63 }
64
65 config.Net.TLS.Enable = true
66 config.Net.TLS.Config = tlsConfig
67 config.Net.TLS.Config.InsecureSkipVerify = *tlsSkipVerify
68 }
69
70 c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), config)
71 if err != nil {
72 printErrorAndExit(69, "Failed to start consumer: %s", err)
73 }
74
75 partitionList, err := getPartitions(c)
76 if err != nil {
77 printErrorAndExit(69, "Failed to get the list of partitions: %s", err)
78 }
79
80 var (
81 messages = make(chan *sarama.ConsumerMessage, *bufferSize)
82 closing = make(chan struct{})
83 wg sync.WaitGroup
84 )
85
86 go func() {
87 signals := make(chan os.Signal, 1)
88 signal.Notify(signals, os.Kill, os.Interrupt)
89 <-signals
90 logger.Println("Initiating shutdown of consumer...")
91 close(closing)
92 }()
93
94 for _, partition := range partitionList {
95 pc, err := c.ConsumePartition(*topic, partition, initialOffset)
96 if err != nil {
97 printErrorAndExit(69, "Failed to start consumer for partition %d: %s", partition, err)
98 }
99
100 go func(pc sarama.PartitionConsumer) {
101 <-closing
102 pc.AsyncClose()
103 }(pc)
104
105 wg.Add(1)
106 go func(pc sarama.PartitionConsumer) {
107 defer wg.Done()
108 for message := range pc.Messages() {
109 messages <- message
110 }
111 }(pc)
112 }
113
114 go func() {
115 for msg := range messages {
116 fmt.Printf("Partition:\t%d\n", msg.Partition)
117 fmt.Printf("Offset:\t%d\n", msg.Offset)
118 fmt.Printf("Key:\t%s\n", string(msg.Key))
119 fmt.Printf("Value:\t%s\n", string(msg.Value))
120 fmt.Println()
121 }
122 }()
123
124 wg.Wait()
125 logger.Println("Done consuming topic", *topic)
126 close(messages)
127
128 if err := c.Close(); err != nil {
129 logger.Println("Failed to close consumer: ", err)
130 }
131 }
132
133 func getPartitions(c sarama.Consumer) ([]int32, error) {
134 if *partitions == "all" {
135 return c.Partitions(*topic)
136 }
137
138 tmp := strings.Split(*partitions, ",")
139 var pList []int32
140 for i := range tmp {
141 val, err := strconv.ParseInt(tmp[i], 10, 32)
142 if err != nil {
143 return nil, err
144 }
145 pList = append(pList, int32(val))
146 }
147
148 return pList, nil
149 }
150
151 func printErrorAndExit(code int, format string, values ...interface{}) {
152 fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
153 fmt.Fprintln(os.Stderr)
154 os.Exit(code)
155 }
156
157 func printUsageErrorAndExit(format string, values ...interface{}) {
158 fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
159 fmt.Fprintln(os.Stderr)
160 fmt.Fprintln(os.Stderr, "Available command line options:")
161 flag.PrintDefaults()
162 os.Exit(64)
163 }