| 1 |
package main |
| 2 |
|
| 3 |
import ( |
| 4 |
"flag" |
| 5 |
"fmt" |
| 6 |
"log" |
| 7 |
"os" |
| 8 |
"os/signal" |
| 9 |
"strconv" |
| 10 |
"strings" |
| 11 |
"sync" |
| 12 |
|
| 13 |
"github.com/Shopify/sarama" |
| 14 |
"github.com/Shopify/sarama/tools/tls" |
| 15 |
) |
| 16 |
|
| 17 |
var ( |
| 18 |
brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster") |
| 19 |
topic = flag.String("topic", "", "REQUIRED: the topic to consume") |
| 20 |
partitions = flag.String("partitions", "all", "The partitions to consume, can be 'all' or comma-separated numbers") |
| 21 |
offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`") |
| 22 |
verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging") |
| 23 |
tlsEnabled = flag.Bool("tls-enabled", false, "Whether to enable TLS") |
| 24 |
tlsSkipVerify = flag.Bool("tls-skip-verify", false, "Whether skip TLS server cert verification") |
| 25 |
tlsClientCert = flag.String("tls-client-cert", "", "Client cert for client authentication (use with -tls-enabled and -tls-client-key)") |
| 26 |
tlsClientKey = flag.String("tls-client-key", "", "Client key for client authentication (use with tls-enabled and -tls-client-cert)") |
| 27 |
|
| 28 |
bufferSize = flag.Int("buffer-size", 256, "The buffer size of the message channel.") |
| 29 |
|
| 30 |
logger = log.New(os.Stderr, "", log.LstdFlags) |
| 31 |
) |
| 32 |
|
| 33 |
func main() { |
| 34 |
flag.Parse() |
| 35 |
|
| 36 |
if *brokerList == "" { |
| 37 |
printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.") |
| 38 |
} |
| 39 |
|
| 40 |
if *topic == "" { |
| 41 |
printUsageErrorAndExit("-topic is required") |
| 42 |
} |
| 43 |
|
| 44 |
if *verbose { |
| 45 |
sarama.Logger = logger |
| 46 |
} |
| 47 |
|
| 48 |
var initialOffset int64 |
| 49 |
switch *offset { |
| 50 |
case "oldest": |
| 51 |
initialOffset = sarama.OffsetOldest |
| 52 |
case "newest": |
| 53 |
initialOffset = sarama.OffsetNewest |
| 54 |
default: |
| 55 |
printUsageErrorAndExit("-offset should be `oldest` or `newest`") |
| 56 |
} |
| 57 |
|
| 58 |
config := sarama.NewConfig() |
| 59 |
if *tlsEnabled { |
| 60 |
tlsConfig, err := tls.NewConfig(*tlsClientCert, *tlsClientKey) |
| 61 |
if err != nil { |
| 62 |
printErrorAndExit(69, "Failed to create TLS config: %s", err) |
| 63 |
} |
| 64 |
|
| 65 |
config.Net.TLS.Enable = true |
| 66 |
config.Net.TLS.Config = tlsConfig |
| 67 |
config.Net.TLS.Config.InsecureSkipVerify = *tlsSkipVerify |
| 68 |
} |
| 69 |
|
| 70 |
c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), config) |
| 71 |
if err != nil { |
| 72 |
printErrorAndExit(69, "Failed to start consumer: %s", err) |
| 73 |
} |
| 74 |
|
| 75 |
partitionList, err := getPartitions(c) |
| 76 |
if err != nil { |
| 77 |
printErrorAndExit(69, "Failed to get the list of partitions: %s", err) |
| 78 |
} |
| 79 |
|
| 80 |
var ( |
| 81 |
messages = make(chan *sarama.ConsumerMessage, *bufferSize) |
| 82 |
closing = make(chan struct{}) |
| 83 |
wg sync.WaitGroup |
| 84 |
) |
| 85 |
|
| 86 |
go func() { |
| 87 |
signals := make(chan os.Signal, 1) |
| 88 |
signal.Notify(signals, os.Kill, os.Interrupt) |
| 89 |
<-signals |
| 90 |
logger.Println("Initiating shutdown of consumer...") |
| 91 |
close(closing) |
| 92 |
}() |
| 93 |
|
| 94 |
for _, partition := range partitionList { |
| 95 |
pc, err := c.ConsumePartition(*topic, partition, initialOffset) |
| 96 |
if err != nil { |
| 97 |
printErrorAndExit(69, "Failed to start consumer for partition %d: %s", partition, err) |
| 98 |
} |
| 99 |
|
| 100 |
go func(pc sarama.PartitionConsumer) { |
| 101 |
<-closing |
| 102 |
pc.AsyncClose() |
| 103 |
}(pc) |
| 104 |
|
| 105 |
wg.Add(1) |
| 106 |
go func(pc sarama.PartitionConsumer) { |
| 107 |
defer wg.Done() |
| 108 |
for message := range pc.Messages() { |
| 109 |
messages <- message |
| 110 |
} |
| 111 |
}(pc) |
| 112 |
} |
| 113 |
|
| 114 |
go func() { |
| 115 |
for msg := range messages { |
| 116 |
fmt.Printf("Partition:\t%d\n", msg.Partition) |
| 117 |
fmt.Printf("Offset:\t%d\n", msg.Offset) |
| 118 |
fmt.Printf("Key:\t%s\n", string(msg.Key)) |
| 119 |
fmt.Printf("Value:\t%s\n", string(msg.Value)) |
| 120 |
fmt.Println() |
| 121 |
} |
| 122 |
}() |
| 123 |
|
| 124 |
wg.Wait() |
| 125 |
logger.Println("Done consuming topic", *topic) |
| 126 |
close(messages) |
| 127 |
|
| 128 |
if err := c.Close(); err != nil { |
| 129 |
logger.Println("Failed to close consumer: ", err) |
| 130 |
} |
| 131 |
} |
| 132 |
|
| 133 |
func getPartitions(c sarama.Consumer) ([]int32, error) { |
| 134 |
if *partitions == "all" { |
| 135 |
return c.Partitions(*topic) |
| 136 |
} |
| 137 |
|
| 138 |
tmp := strings.Split(*partitions, ",") |
| 139 |
var pList []int32 |
| 140 |
for i := range tmp { |
| 141 |
val, err := strconv.ParseInt(tmp[i], 10, 32) |
| 142 |
if err != nil { |
| 143 |
return nil, err |
| 144 |
} |
| 145 |
pList = append(pList, int32(val)) |
| 146 |
} |
| 147 |
|
| 148 |
return pList, nil |
| 149 |
} |
| 150 |
|
| 151 |
func printErrorAndExit(code int, format string, values ...interface{}) { |
| 152 |
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) |
| 153 |
fmt.Fprintln(os.Stderr) |
| 154 |
os.Exit(code) |
| 155 |
} |
| 156 |
|
| 157 |
func printUsageErrorAndExit(format string, values ...interface{}) { |
| 158 |
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...)) |
| 159 |
fmt.Fprintln(os.Stderr) |
| 160 |
fmt.Fprintln(os.Stderr, "Available command line options:") |
| 161 |
flag.PrintDefaults() |
| 162 |
os.Exit(64) |
| 163 |
} |