Debian Codegraph @

tools/kafka-console-consumer/kafka-console-consumer.go

raw
1 package main
2
3 import (
4 "flag"
5 "fmt"
6 "log"
7 "os"
8 "os/signal"
9 "strconv"
10 "strings"
11 "sync"
12 "syscall"
13
14 "github.com/IBM/sarama"
15 "github.com/IBM/sarama/tools/tls"
16 )
17
18 var (
19 brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster")
20 topic = flag.String("topic", "", "REQUIRED: the topic to consume")
21 partitions = flag.String("partitions", "all", "The partitions to consume, can be 'all' or comma-separated numbers")
22 offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`")
23 verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging")
24 tlsEnabled = flag.Bool("tls-enabled", false, "Whether to enable TLS")
25 tlsSkipVerify = flag.Bool("tls-skip-verify", false, "Whether skip TLS server cert verification")
26 tlsClientCert = flag.String("tls-client-cert", "", "Client cert for client authentication (use with -tls-enabled and -tls-client-key)")
27 tlsClientKey = flag.String("tls-client-key", "", "Client key for client authentication (use with tls-enabled and -tls-client-cert)")
28
29 bufferSize = flag.Int("buffer-size", 256, "The buffer size of the message channel.")
30
31 logger = log.New(os.Stderr, "", log.LstdFlags)
32 )
33
34 func main() {
35 flag.Parse()
36
37 if *brokerList == "" {
38 printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.")
39 }
40
41 if *topic == "" {
42 printUsageErrorAndExit("-topic is required")
43 }
44
45 if *verbose {
46 sarama.Logger = logger
47 }
48
49 var initialOffset int64
50 switch *offset {
51 case "oldest":
52 initialOffset = sarama.OffsetOldest
53 case "newest":
54 initialOffset = sarama.OffsetNewest
55 default:
56 printUsageErrorAndExit("-offset should be `oldest` or `newest`")
57 }
58
59 config := sarama.NewConfig()
60 if *tlsEnabled {
61 tlsConfig, err := tls.NewConfig(*tlsClientCert, *tlsClientKey)
62 if err != nil {
63 printErrorAndExit(69, "Failed to create TLS config: %s", err)
64 }
65
66 config.Net.TLS.Enable = true
67 config.Net.TLS.Config = tlsConfig
68 config.Net.TLS.Config.InsecureSkipVerify = *tlsSkipVerify
69 }
70
71 c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), config)
72 if err != nil {
73 printErrorAndExit(69, "Failed to start consumer: %s", err)
74 }
75
76 partitionList, err := getPartitions(c)
77 if err != nil {
78 printErrorAndExit(69, "Failed to get the list of partitions: %s", err)
79 }
80
81 var (
82 messages = make(chan *sarama.ConsumerMessage, *bufferSize)
83 closing = make(chan struct{})
84 wg sync.WaitGroup
85 )
86
87 go func() {
88 signals := make(chan os.Signal, 1)
89 signal.Notify(signals, syscall.SIGTERM, os.Interrupt)
90 <-signals
91 logger.Println("Initiating shutdown of consumer...")
92 close(closing)
93 }()
94
95 for _, partition := range partitionList {
96 pc, err := c.ConsumePartition(*topic, partition, initialOffset)
97 if err != nil {
98 printErrorAndExit(69, "Failed to start consumer for partition %d: %s", partition, err)
99 }
100
101 go func(pc sarama.PartitionConsumer) {
102 <-closing
103 pc.AsyncClose()
104 }(pc)
105
106 wg.Add(1)
107 go func(pc sarama.PartitionConsumer) {
108 defer wg.Done()
109 for message := range pc.Messages() {
110 messages <- message
111 }
112 }(pc)
113 }
114
115 go func() {
116 for msg := range messages {
117 fmt.Printf("Partition:\t%d\n", msg.Partition)
118 fmt.Printf("Offset:\t%d\n", msg.Offset)
119 fmt.Printf("Key:\t%s\n", string(msg.Key))
120 fmt.Printf("Value:\t%s\n", string(msg.Value))
121 fmt.Println()
122 }
123 }()
124
125 wg.Wait()
126 logger.Println("Done consuming topic", *topic)
127 close(messages)
128
129 if err := c.Close(); err != nil {
130 logger.Println("Failed to close consumer: ", err)
131 }
132 }
133
134 func getPartitions(c sarama.Consumer) ([]int32, error) {
135 if *partitions == "all" {
136 return c.Partitions(*topic)
137 }
138
139 tmp := strings.Split(*partitions, ",")
140 var pList []int32
141 for i := range tmp {
142 val, err := strconv.ParseInt(tmp[i], 10, 32)
143 if err != nil {
144 return nil, err
145 }
146 pList = append(pList, int32(val))
147 }
148
149 return pList, nil
150 }
151
152 func printErrorAndExit(code int, format string, values ...interface{}) {
153 fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
154 fmt.Fprintln(os.Stderr)
155 os.Exit(code)
156 }
157
158 func printUsageErrorAndExit(format string, values ...interface{}) {
159 fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
160 fmt.Fprintln(os.Stderr)
161 fmt.Fprintln(os.Stderr, "Available command line options:")
162 flag.PrintDefaults()
163 os.Exit(64)
164 }