Debian Codegraph @

pubsub/kafkapubsub/kafka_test.go

raw
1 // Copyright 2019 The Go Cloud Development Kit Authors
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 package kafkapubsub // import "gocloud.dev/pubsub/kafkapubsub"
16
17 // To run these tests against a real Kafka server, run localkafka.sh.
18 // See https://github.com/spotify/docker-kafka for more on the docker container
19 // that the script runs.
20
21 import (
22 "context"
23 "errors"
24 "fmt"
25 "math/rand"
26 "os"
27 "strings"
28 "sync/atomic"
29 "testing"
30 "time"
31
32 "github.com/IBM/sarama"
33 "github.com/google/go-cmp/cmp"
34 "github.com/google/go-cmp/cmp/cmpopts"
35 "gocloud.dev/internal/testing/setup"
36 "gocloud.dev/pubsub"
37 "gocloud.dev/pubsub/driver"
38 "gocloud.dev/pubsub/drivertest"
39 )
40
41 var (
42 localBrokerAddrs = []string{"localhost:9092"}
43 // Makes OpenSubscription wait ~forever until the subscriber has joined the
44 // ConsumerGroup. Messages sent to the topic before the subscriber has joined
45 // won't be received.
46 subscriptionOptions = &SubscriptionOptions{WaitForJoin: 24 * time.Hour}
47 )
48
49 type harness struct {
50 uniqueID int
51 numSubs uint32
52 numTopics uint32
53 }
54
55 func newHarness(ctx context.Context, t *testing.T) (drivertest.Harness, error) {
56 if !setup.HasDockerTestEnvironment() {
57 t.Skip("Skipping Kafka tests since the Kafka server is not available")
58 }
59 return &harness{uniqueID: rand.Int()}, nil
60 }
61
62 func createKafkaTopic(topicName string, partitions int32) (func(), error) {
63 // Create the topic.
64 config := MinimalConfig()
65 admin, err := sarama.NewClusterAdmin(localBrokerAddrs, config)
66 if err != nil {
67 return func() {}, err
68 }
69 close1 := func() { admin.Close() }
70
71 topicDetail := &sarama.TopicDetail{
72 NumPartitions: partitions,
73 ReplicationFactor: 1,
74 }
75 if err := admin.CreateTopic(topicName, topicDetail, false); err != nil {
76 return close1, err
77 }
78 close2 := func() {
79 admin.DeleteTopic(topicName)
80 close1()
81 }
82 return close2, nil
83 }
84
85 func (h *harness) CreateTopic(ctx context.Context, testName string) (driver.Topic, func(), error) {
86 topicName := fmt.Sprintf("%s-topic-%d-%d", sanitize(testName), h.uniqueID, atomic.AddUint32(&h.numTopics, 1))
87 cleanup, err := createKafkaTopic(topicName, 1)
88 if err != nil {
89 return nil, cleanup, err
90 }
91
92 // Open it.
93 dt, err := openTopic(localBrokerAddrs, MinimalConfig(), topicName, nil)
94 if err != nil {
95 return nil, cleanup, err
96 }
97 return dt, cleanup, nil
98 }
99
100 func (h *harness) MakeNonexistentTopic(ctx context.Context) (driver.Topic, error) {
101 return openTopic(localBrokerAddrs, MinimalConfig(), "nonexistent-topic", nil)
102 }
103
104 func (h *harness) CreateSubscription(ctx context.Context, dt driver.Topic, testName string) (driver.Subscription, func(), error) {
105 groupID := fmt.Sprintf("%s-sub-%d-%d", sanitize(testName), h.uniqueID, atomic.AddUint32(&h.numSubs, 1))
106 ds, err := openSubscription(localBrokerAddrs, MinimalConfig(), groupID, []string{dt.(*topic).topicName}, subscriptionOptions)
107 return ds, func() {}, err
108 }
109
110 func (h *harness) MakeNonexistentSubscription(ctx context.Context) (driver.Subscription, func(), error) {
111 ds, err := openSubscription(localBrokerAddrs, MinimalConfig(), "unused-group", []string{"nonexistent-topic"}, subscriptionOptions)
112 return ds, func() {}, err
113 }
114
115 func (h *harness) Close() {}
116
117 func (h *harness) MaxBatchSizes() (int, int) { return sendBatcherOpts.MaxBatchSize, 0 }
118
119 func (*harness) SupportsMultipleSubscriptions() bool { return true }
120
121 func TestConformance(t *testing.T) {
122 asTests := []drivertest.AsTest{asTest{}}
123 drivertest.RunConformanceTests(t, newHarness, asTests)
124 }
125
126 type asTest struct{}
127
128 func (asTest) Name() string {
129 return "kafka"
130 }
131
132 func (asTest) TopicCheck(topic *pubsub.Topic) error {
133 var sp sarama.SyncProducer
134 if !topic.As(&sp) {
135 return fmt.Errorf("cast failed for %T", sp)
136 }
137 return nil
138 }
139
140 func (asTest) SubscriptionCheck(sub *pubsub.Subscription) error {
141 var cg sarama.ConsumerGroup
142 if !sub.As(&cg) {
143 return fmt.Errorf("cast failed for %T", cg)
144 }
145 var cgs sarama.ConsumerGroupSession
146 if !sub.As(&cgs) {
147 return fmt.Errorf("cast failed for %T", cgs)
148 }
149 return nil
150 }
151
152 func (asTest) TopicErrorCheck(t *pubsub.Topic, err error) error {
153 var pe sarama.ProducerErrors
154 if !t.ErrorAs(err, &pe) {
155 return fmt.Errorf("failed to convert %v (%T)", err, err)
156 }
157 return nil
158 }
159
160 func (asTest) SubscriptionErrorCheck(s *pubsub.Subscription, err error) error {
161 var ke sarama.KError
162 if !s.ErrorAs(err, &ke) {
163 return fmt.Errorf("failed to convert %v (%T)", err, err)
164 }
165 return nil
166 }
167
168 func (asTest) MessageCheck(m *pubsub.Message) error {
169 var cm *sarama.ConsumerMessage
170 if !m.As(&cm) {
171 return fmt.Errorf("cast failed for %T", cm)
172 }
173 return nil
174 }
175
176 func (asTest) BeforeSend(as func(any) bool) error {
177 var pm *sarama.ProducerMessage
178 if !as(&pm) {
179 return fmt.Errorf("cast failed for %T", &pm)
180 }
181 return nil
182 }
183
184 func (asTest) AfterSend(as func(any) bool) error {
185 return nil
186 }
187
188 // TestKafkaKey tests sending/receiving a message with the Kafka message key set.
189 func TestKafkaKey(t *testing.T) {
190 if !setup.HasDockerTestEnvironment() {
191 t.Skip("Skipping Kafka tests since the Kafka server is not available")
192 }
193 const (
194 keyName = "kafkakey"
195 keyValue = "kafkakeyvalue"
196 )
197 uniqueID := rand.Int()
198 ctx := context.Background()
199
200 topicName := fmt.Sprintf("%s-topic-%d", sanitize(t.Name()), uniqueID)
201 topicCleanup, err := createKafkaTopic(topicName, 1)
202 defer topicCleanup()
203 if err != nil {
204 t.Fatal(err)
205 }
206 topic, err := OpenTopic(localBrokerAddrs, MinimalConfig(), topicName, &TopicOptions{KeyName: keyName})
207 if err != nil {
208 t.Fatal(err)
209 }
210 defer func() {
211 if err := topic.Shutdown(ctx); err != nil {
212 t.Error(err)
213 }
214 }()
215
216 groupID := fmt.Sprintf("%s-sub-%d", sanitize(t.Name()), uniqueID)
217 subOpts := *subscriptionOptions
218 subOpts.KeyName = keyName
219 sub, err := OpenSubscription(localBrokerAddrs, MinimalConfig(), groupID, []string{topicName}, &subOpts)
220 if err != nil {
221 t.Fatal(err)
222 }
223 defer func() {
224 if err := sub.Shutdown(ctx); err != nil {
225 t.Error(err)
226 }
227 }()
228
229 m := &pubsub.Message{
230 Metadata: map[string]string{
231 "foo": "bar",
232 keyName: keyValue,
233 },
234 Body: []byte("hello world"),
235 BeforeSend: func(as func(any) bool) error {
236 // Verify that the Key field was set correctly on the outgoing Kafka
237 // message.
238 var pm *sarama.ProducerMessage
239 if !as(&pm) {
240 return errors.New("failed to convert to ProducerMessage")
241 }
242 gotKeyBytes, err := pm.Key.Encode()
243 if err != nil {
244 return fmt.Errorf("failed to Encode Kafka Key: %v", err)
245 }
246 if gotKey := string(gotKeyBytes); gotKey != keyValue {
247 return errors.New("Kafka key wasn't set appropriately")
248 }
249 return nil
250 },
251 }
252 err = topic.Send(ctx, m)
253 if err != nil {
254 t.Fatal(err)
255 }
256
257 // The test will hang here if the message isn't available, so use a shorter timeout.
258 ctx2, cancel := context.WithTimeout(ctx, 30*time.Second)
259 defer cancel()
260 got, err := sub.Receive(ctx2)
261 if err != nil {
262 t.Fatal(err)
263 }
264 got.Ack()
265
266 m.BeforeSend = nil // don't expect this in the received message
267 m.LoggableID = keyValue
268 if diff := cmp.Diff(got, m, cmpopts.IgnoreUnexported(pubsub.Message{})); diff != "" {
269 t.Errorf("got\n%v\nwant\n%v\ndiff\n%v", got, m, diff)
270 }
271
272 // Verify that Key was set in the received Kafka message via As.
273 var cm *sarama.ConsumerMessage
274 if !got.As(&cm) {
275 t.Fatal("failed to get message As ConsumerMessage")
276 }
277 if gotKey := string(cm.Key); gotKey != keyValue {
278 t.Errorf("got key %q want %q", gotKey, keyValue)
279 }
280 }
281
282 // TestMultiplePartionsWithRebalancing tests use of a topic with multiple
283 // partitions, including the rebalancing that happens when a new consumer
284 // appears in the group.
285 func TestMultiplePartionsWithRebalancing(t *testing.T) {
286 if !setup.HasDockerTestEnvironment() {
287 t.Skip("Skipping Kafka tests since the Kafka server is not available")
288 }
289 const (
290 keyName = "kafkakey"
291 nMessages = 10
292 )
293 uniqueID := rand.Int()
294 ctx := context.Background()
295
296 // Create a topic with 10 partitions. Using 10 instead of just 2 because
297 // that also tests having multiple claims.
298 topicName := fmt.Sprintf("%s-topic-%d", sanitize(t.Name()), uniqueID)
299 topicCleanup, err := createKafkaTopic(topicName, 10)
300 defer topicCleanup()
301 if err != nil {
302 t.Fatal(err)
303 }
304 topic, err := OpenTopic(localBrokerAddrs, MinimalConfig(), topicName, &TopicOptions{KeyName: keyName})
305 if err != nil {
306 t.Fatal(err)
307 }
308 defer func() {
309 if err := topic.Shutdown(ctx); err != nil {
310 t.Error(err)
311 }
312 }()
313
314 // Open a subscription.
315 groupID := fmt.Sprintf("%s-sub-%d", sanitize(t.Name()), uniqueID)
316 subOpts := *subscriptionOptions
317 subOpts.KeyName = keyName
318 sub, err := OpenSubscription(localBrokerAddrs, MinimalConfig(), groupID, []string{topicName}, &subOpts)
319 if err != nil {
320 t.Fatal(err)
321 }
322 defer func() {
323 if err := sub.Shutdown(ctx); err != nil {
324 t.Error(err)
325 }
326 }()
327
328 // Send some messages.
329 send := func() {
330 for i := 0; i < nMessages; i++ {
331 m := &pubsub.Message{
332 Metadata: map[string]string{
333 keyName: fmt.Sprintf("key%d", i),
334 },
335 Body: []byte("hello world"),
336 }
337 if err := topic.Send(ctx, m); err != nil {
338 t.Fatal(err)
339 }
340 }
341 }
342 send()
343
344 // Receive the messages via the subscription.
345 got := make(chan int)
346 done := make(chan error)
347 read := func(ctx context.Context, subNum int, sub *pubsub.Subscription) {
348 for {
349 m, err := sub.Receive(ctx)
350 if err != nil {
351 if err == context.Canceled {
352 // Expected after all messages are received, no error.
353 done <- nil
354 } else {
355 done <- err
356 }
357 return
358 }
359 m.Ack()
360 got <- subNum
361 }
362 }
363 // The test will hang here if the messages aren't available, so use a shorter
364 // timeout.
365 ctx2, cancel := context.WithTimeout(ctx, 30*time.Second)
366 go read(ctx2, 0, sub)
367 for i := 0; i < nMessages; i++ {
368 select {
369 case <-got:
370 case err := <-done:
371 // Premature error.
372 if err != nil {
373 t.Fatal(err)
374 }
375 }
376 }
377 cancel()
378 if err := <-done; err != nil {
379 t.Fatal(err)
380 }
381
382 // Add another subscription to the same group. Kafka will rebalance the
383 // consumer group, causing the Cleanup/Setup/ConsumeClaim loop. Each of the
384 // two subscriptions should get claims for 50% of the partitions.
385 sub2, err := OpenSubscription(localBrokerAddrs, MinimalConfig(), groupID, []string{topicName}, &subOpts)
386 if err != nil {
387 t.Fatal(err)
388 }
389 defer func() {
390 if err := sub2.Shutdown(ctx); err != nil {
391 t.Error(err)
392 }
393 }()
394
395 // Send and receive some messages.
396 // Now both subscriptions should get some messages.
397 send()
398
399 // The test will hang here if the message isn't available, so use a shorter timeout.
400 ctx3, cancel := context.WithTimeout(ctx, 30*time.Second)
401 go read(ctx3, 0, sub)
402 go read(ctx3, 1, sub2)
403 counts := []int{0, 0}
404 for i := 0; i < nMessages; i++ {
405 select {
406 case sub := <-got:
407 counts[sub]++
408 case err := <-done:
409 // Premature error.
410 if err != nil {
411 t.Fatal(err)
412 }
413 }
414 }
415 cancel()
416 for i := 0; i < 2; i++ {
417 if err := <-done; err != nil {
418 t.Fatal(err)
419 }
420 }
421 if counts[0] == 0 || counts[1] == 0 {
422 t.Errorf("one of the partitioned subscriptions didn't get any messages: %v", counts)
423 }
424 }
425
426 func sanitize(testName string) string {
427 return strings.Replace(testName, "/", "_", -1)
428 }
429
430 func BenchmarkKafka(b *testing.B) {
431 ctx := context.Background()
432 uniqueID := rand.Int()
433
434 // Create the topic.
435 topicName := fmt.Sprintf("%s-topic-%d", b.Name(), uniqueID)
436 cleanup, err := createKafkaTopic(topicName, 1)
437 defer cleanup()
438 if err != nil {
439 b.Fatal(err)
440 }
441
442 topic, err := OpenTopic(localBrokerAddrs, MinimalConfig(), topicName, nil)
443 if err != nil {
444 b.Fatal(err)
445 }
446 defer topic.Shutdown(ctx)
447
448 groupID := fmt.Sprintf("%s-subscription-%d", b.Name(), uniqueID)
449 sub, err := OpenSubscription(localBrokerAddrs, MinimalConfig(), groupID, []string{topicName}, subscriptionOptions)
450 if err != nil {
451 b.Fatal(err)
452 }
453 defer sub.Shutdown(ctx)
454
455 drivertest.RunBenchmarks(b, topic, sub)
456 }
457
458 func fakeConnectionStringInEnv() func() {
459 oldEnvVal := os.Getenv("KAFKA_BROKERS")
460 os.Setenv("KAFKA_BROKERS", "localhost:10000")
461 return func() {
462 os.Setenv("KAFKA_BROKERS", oldEnvVal)
463 }
464 }
465
466 func TestOpenTopicFromURL(t *testing.T) {
467 cleanup := fakeConnectionStringInEnv()
468 defer cleanup()
469
470 tests := []struct {
471 URL string
472 WantErr bool
473 }{
474 // OK.
475 {"kafka://mytopic", false},
476 // OK, specifying key_name.
477 {"kafka://mytopic?key_name=x-partition-key", false},
478 // Invalid key_name value.
479 {"kafka://mytopic?key_name=", true},
480 // Invalid parameter.
481 {"kafka://mytopic?param=value", true},
482 }
483
484 ctx := context.Background()
485 for _, test := range tests {
486 topic, err := pubsub.OpenTopic(ctx, test.URL)
487 if err != nil && errors.Is(err, sarama.ErrOutOfBrokers) {
488 // Since we don't have a real kafka broker to talk to, we will always get an error when
489 // opening a topic. This test is checking specifically for query parameter usage, so
490 // we treat the "no brokers" error message as a nil error.
491 err = nil
492 }
493
494 if (err != nil) != test.WantErr {
495 t.Errorf("%s: got error %v, want error %v", test.URL, err, test.WantErr)
496 }
497 if topic != nil {
498 topic.Shutdown(ctx)
499 }
500 }
501 }
502
503 func TestOpenSubscriptionFromURL(t *testing.T) {
504 cleanup := fakeConnectionStringInEnv()
505 defer cleanup()
506
507 tests := []struct {
508 URL string
509 WantErr bool
510 }{
511 // OK.
512 {"kafka://mygroup?topic=mytopic", false},
513 // OK, specifying initial offset.
514 {"kafka://mygroup?topic=mytopic&offset=oldest", false},
515 {"kafka://mygroup?topic=mytopic&offset=newest", false},
516 // Invalid offset specified.
517 {"kafka://mygroup?topic=mytopic&offset=value", true},
518 // Invalid parameter.
519 {"kafka://mygroup?topic=mytopic&param=value", true},
520 }
521
522 ctx := context.Background()
523
524 for _, test := range tests {
525 sub, err := pubsub.OpenSubscription(ctx, test.URL)
526 if err != nil && errors.Is(err, sarama.ErrOutOfBrokers) {
527 // Since we don't have a real kafka broker to talk to, we will always get an error when
528 // opening a subscription. This test is checking specifically for query parameter usage, so
529 // we treat the "no brokers" error message as a nil error.
530 err = nil
531 }
532
533 if (err != nil) != test.WantErr {
534 t.Errorf("%s: got error %v, want error %v", test.URL, err, test.WantErr)
535 }
536 if sub != nil {
537 sub.Shutdown(ctx)
538 }
539 }
540 }