Debian Codegraph @

pubsub/v2/internal/longtest/endtoend_test.go

raw
1 // Copyright 2025 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 package longtest_test
16
17 import (
18 "context"
19 "errors"
20 "fmt"
21 "log"
22 "math/rand"
23 "strconv"
24 "strings"
25 "sync"
26 "testing"
27 "time"
28
29 "cloud.google.com/go/internal/testutil"
30 "cloud.google.com/go/pubsub"
31 "google.golang.org/api/iterator"
32 "google.golang.org/api/option"
33 )
34
35 const (
36 timeout = time.Minute * 10
37 ackDeadline = time.Second * 10
38 nMessages = 1e4
39 acceptableDupPercentage = 1
40 numAcceptableDups = int(nMessages * acceptableDupPercentage / 100)
41 resourcePrefix = "endtoend"
42 )
43
44 // The end-to-end pumps many messages into a topic and tests that they are all
45 // delivered to each subscription for the topic. It also tests that messages
46 // are not unexpectedly redelivered.
47 func TestEndToEnd_Dupes(t *testing.T) {
48 t.Skip("https://github.com/googleapis/google-cloud-go/issues/1752")
49
50 ctx, cancel := context.WithTimeout(context.Background(), timeout)
51 defer cancel()
52 client, topic, cleanup := prepareEndToEndTest(ctx, t)
53 defer cleanup()
54 subPrefix := fmt.Sprintf("%s-%d", resourcePrefix, time.Now().UnixNano())
55
56 // Two subscriptions to the same topic.
57 var err error
58 var subs [2]*pubsub.Subscription
59 for i := 0; i < len(subs); i++ {
60 subs[i], err = client.CreateSubscription(ctx, fmt.Sprintf("%s-%d", subPrefix, i), pubsub.SubscriptionConfig{
61 Topic: topic,
62 AckDeadline: ackDeadline,
63 })
64 if err != nil {
65 t.Fatalf("CreateSub error: %v", err)
66 }
67 defer subs[i].Delete(ctx)
68 }
69
70 err = publish(ctx, topic, nMessages)
71 topic.Stop()
72 if err != nil {
73 t.Fatalf("publish: %v", err)
74 }
75
76 // recv provides an indication that messages are still arriving.
77 recv := make(chan struct{})
78 // We have two subscriptions to our topic.
79 // Each subscription will get a copy of each published message.
80 var wg sync.WaitGroup
81 cctx, cancel := context.WithTimeout(ctx, timeout)
82 defer cancel()
83
84 consumers := []*consumer{
85 {
86 counts: make(map[string]int),
87 recv: recv,
88 durations: []time.Duration{time.Hour},
89 done: make(chan struct{}),
90 },
91 {
92 counts: make(map[string]int),
93 recv: recv,
94 durations: []time.Duration{ackDeadline, ackDeadline, ackDeadline / 2, ackDeadline / 2, time.Hour},
95 done: make(chan struct{}),
96 },
97 }
98 for i, con := range consumers {
99 con := con
100 sub := subs[i]
101 wg.Add(1)
102 go func() {
103 defer wg.Done()
104 con.consume(ctx, t, sub)
105 }()
106 }
107 // Wait for a while after the last message before declaring quiescence.
108 // We wait a multiple of the ack deadline, for two reasons:
109 // 1. To detect if messages are redelivered after having their ack
110 // deadline extended.
111 // 2. To wait for redelivery of messages that were en route when a Receive
112 // is canceled. This can take considerably longer than the ack deadline.
113 quiescenceDur := ackDeadline * 6
114 quiescenceTimer := time.NewTimer(quiescenceDur)
115
116 loop:
117 for {
118 select {
119 case <-recv:
120 // Reset timer so we wait quiescenceDur after the last message.
121 // See https://godoc.org/time#Timer.Reset for why the Stop
122 // and channel drain are necessary.
123 if !quiescenceTimer.Stop() {
124 <-quiescenceTimer.C
125 }
126 quiescenceTimer.Reset(quiescenceDur)
127
128 case <-quiescenceTimer.C:
129 cancel()
130 log.Println("quiesced")
131 break loop
132
133 case <-cctx.Done():
134 t.Fatal("timed out")
135 }
136 }
137 wg.Wait()
138 close(recv)
139 for i, con := range consumers {
140 var numDups int
141 var zeroes int
142 for _, v := range con.counts {
143 if v == 0 {
144 zeroes++
145 }
146 numDups += v - 1
147 }
148
149 if zeroes > 0 {
150 t.Errorf("Consumer %d: %d messages never arrived", i, zeroes)
151 } else if numDups > numAcceptableDups {
152 t.Errorf("Consumer %d: Willing to accept %d dups (%v%% duplicated of %d messages), but got %d", i, numAcceptableDups, acceptableDupPercentage, int(nMessages), numDups)
153 }
154 }
155
156 for i, con := range consumers {
157 select {
158 case <-con.done:
159 case <-time.After(15 * time.Second):
160 t.Fatalf("timed out waiting for consumer %d to finish", i)
161 }
162 }
163 }
164
165 func TestEndToEnd_LongProcessingTime(t *testing.T) {
166 ctx, cancel := context.WithTimeout(context.Background(), timeout)
167 defer cancel()
168 client, topic, cleanup := prepareEndToEndTest(ctx, t)
169 defer cleanup()
170 subPrefix := fmt.Sprintf("%s-%d", resourcePrefix, time.Now().UnixNano())
171
172 // Two subscriptions to the same topic.
173 sub, err := client.CreateSubscription(ctx, subPrefix+"-00", pubsub.SubscriptionConfig{
174 Topic: topic,
175 AckDeadline: ackDeadline,
176 })
177 if err != nil {
178 t.Fatalf("CreateSub error: %v", err)
179 }
180 defer sub.Delete(ctx)
181
182 // Tests the issue found in https://github.com/googleapis/google-cloud-go/issues/1247.
183 sub.ReceiveSettings.Synchronous = true
184 sub.ReceiveSettings.MaxOutstandingMessages = 500
185
186 err = publish(ctx, topic, 500)
187 topic.Stop()
188 if err != nil {
189 t.Fatalf("publish: %v", err)
190 }
191
192 // recv provides an indication that messages are still arriving.
193 recv := make(chan struct{})
194 consumer := consumer{
195 counts: make(map[string]int),
196 recv: recv,
197 durations: []time.Duration{time.Hour},
198 processingDelay: func() time.Duration {
199 return time.Duration(1+rand.Int63n(120)) * time.Second
200 },
201 done: make(chan struct{}),
202 }
203 go consumer.consume(ctx, t, sub)
204 // Wait for a while after the last message before declaring quiescence.
205 // We wait a multiple of the ack deadline, for two reasons:
206 // 1. To detect if messages are redelivered after having their ack
207 // deadline extended.
208 // 2. To wait for redelivery of messages that were en route when a Receive
209 // is canceled. This can take considerably longer than the ack deadline.
210 quiescenceDur := 12 * ackDeadline
211 quiescenceTimer := time.NewTimer(quiescenceDur)
212 loop:
213 for {
214 select {
215 case <-recv:
216 // Reset timer so we wait quiescenceDur after the last message.
217 // See https://godoc.org/time#Timer.Reset for why the Stop
218 // and channel drain are necessary.
219 if !quiescenceTimer.Stop() {
220 <-quiescenceTimer.C
221 }
222 quiescenceTimer.Reset(quiescenceDur)
223
224 case <-quiescenceTimer.C:
225 cancel()
226 log.Println("quiesced")
227 break loop
228
229 case <-ctx.Done():
230 t.Fatal("timed out")
231 }
232 }
233 close(recv)
234 var numDups int
235 var zeroes int
236 for _, v := range consumer.counts {
237 if v == 0 {
238 zeroes++
239 }
240 numDups += v - 1
241 }
242
243 if zeroes > 0 {
244 t.Errorf("%d messages never arrived", zeroes)
245 } else if numDups > numAcceptableDups {
246 t.Errorf("Willing to accept %d dups (%v duplicated of %d messages), but got %d", numAcceptableDups, acceptableDupPercentage, int(nMessages), numDups)
247 }
248
249 select {
250 case <-consumer.done:
251 case <-time.After(15 * time.Second):
252 t.Fatal("timed out waiting for consumer to finish")
253 }
254 }
255
256 // publish publishes n messages to topic.
257 func publish(ctx context.Context, topic *pubsub.Topic, n int) error {
258 var rs []*pubsub.PublishResult
259 for i := 0; i < n; i++ {
260 m := &pubsub.Message{Data: []byte(fmt.Sprintf("msg %d", i))}
261 rs = append(rs, topic.Publish(ctx, m))
262 }
263 for _, r := range rs {
264 _, err := r.Get(ctx)
265 if err != nil {
266 return err
267 }
268 }
269 return nil
270 }
271
272 // consumer consumes messages according to its configuration.
273 type consumer struct {
274 // A consumer will spin out a Receive for each duration, which will be
275 // canceled after each duration and the next one spun up. For example, if
276 // there are 5 3 second durations, then there will be 5 3 second Receives.
277 durations []time.Duration
278
279 // A value is sent to recv each time process is called.
280 recv chan struct{}
281
282 // How long to wait for before acking.
283 processingDelay func() time.Duration
284
285 mu sync.Mutex
286 counts map[string]int // msgID: recvdAmt
287 totalRecvd int
288
289 // Done consuming.
290 done chan struct{}
291 }
292
293 // consume reads messages from a subscription, and keeps track of what it receives in mc.
294 // After consume returns, the caller should wait on wg to ensure that no more updates to mc will be made.
295 func (c *consumer) consume(ctx context.Context, t *testing.T, sub *pubsub.Subscription) {
296 defer close(c.done)
297 for _, dur := range c.durations {
298 ctx2, cancel := context.WithTimeout(ctx, dur)
299 defer cancel()
300 id := sub.String()[len(sub.String())-1:]
301 t.Logf("%s: start receive", id)
302 prev := c.totalRecvd
303 err := sub.Receive(ctx2, c.process)
304 t.Logf("%s: end receive; read %d", id, c.totalRecvd-prev)
305 if err != nil && !errors.Is(err, context.Canceled) {
306 panic(err)
307 }
308 select {
309 case <-ctx.Done():
310 return
311 default:
312 }
313 }
314 }
315
316 // process handles a message and records it in mc.
317 func (c *consumer) process(_ context.Context, m *pubsub.Message) {
318 c.mu.Lock()
319 c.counts[m.ID]++
320 c.totalRecvd++
321 c.mu.Unlock()
322 c.recv <- struct{}{}
323
324 var delay time.Duration
325 if c.processingDelay == nil {
326 delay = time.Duration(rand.Intn(int(ackDeadline * 3)))
327 } else {
328 delay = c.processingDelay()
329 }
330
331 // Simulate time taken to process m, while continuing to process more messages.
332 // Some messages will need to have their ack deadline extended due to this delay.
333 time.AfterFunc(delay, func() {
334 m.Ack()
335 })
336 }
337
338 // Remember to call cleanup!
339 func prepareEndToEndTest(ctx context.Context, t *testing.T) (*pubsub.Client, *pubsub.Topic, func()) {
340 if testing.Short() {
341 t.Skip("Integration tests skipped in short mode")
342 }
343 ts := testutil.TokenSource(ctx, pubsub.ScopePubSub, pubsub.ScopeCloudPlatform)
344 if ts == nil {
345 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
346 }
347
348 now := time.Now()
349 topicName := fmt.Sprintf("%s-%d", resourcePrefix, now.UnixNano())
350
351 client, err := pubsub.NewClient(ctx, testutil.ProjID(), option.WithTokenSource(ts))
352 if err != nil {
353 t.Fatalf("Creating client error: %v", err)
354 }
355
356 // Don't stop the test if cleanup failed.
357 if err := cleanupSubscription(ctx, client); err != nil {
358 t.Logf("Pre-test subscription cleanup failed: %v", err)
359 }
360 if err := cleanupTopic(ctx, client); err != nil {
361 t.Logf("Pre-test topic cleanup failed: %v", err)
362 }
363
364 var topic *pubsub.Topic
365 if topic, err = client.CreateTopic(ctx, topicName); err != nil {
366 t.Fatalf("CreateTopic error: %v", err)
367 }
368
369 return client, topic, func() {
370 topic.Delete(ctx)
371 client.Close()
372 }
373 }
374
375 // cleanupTopic deletes stale testing topics.
376 func cleanupTopic(ctx context.Context, client *pubsub.Client) error {
377 if testing.Short() {
378 return nil // Don't clean up in short mode.
379 }
380 // Delete topics which were created a while ago.
381 const expireAge = 24 * time.Hour
382
383 it := client.Topics(ctx)
384 for {
385 t, err := it.Next()
386 if errors.Is(err, iterator.Done) {
387 break
388 }
389 if err != nil {
390 return err
391 }
392 // Take timestamp from id.
393 tID := t.ID()
394 p := strings.Split(tID, "-")
395
396 // Only delete resources created from the endtoend test.
397 // Otherwise, this will affect other tests running midflight.
398 if p[0] == resourcePrefix {
399 tCreated := p[len(p)-1]
400 timestamp, err := strconv.ParseInt(tCreated, 10, 64)
401 if err != nil {
402 continue
403 }
404 timeTCreated := time.Unix(0, timestamp)
405 if time.Since(timeTCreated) > expireAge {
406 log.Printf("deleting topic %q", tID)
407 if err := t.Delete(ctx); err != nil {
408 return fmt.Errorf("Delete topic: %v: %v", t.String(), err)
409 }
410 }
411 }
412 }
413 return nil
414 }
415
416 // cleanupSubscription deletes stale testing subscriptions.
417 func cleanupSubscription(ctx context.Context, client *pubsub.Client) error {
418 if testing.Short() {
419 return nil // Don't clean up in short mode.
420 }
421 // Delete subscriptions which were created a while ago.
422 const expireAge = 24 * time.Hour
423
424 it := client.Subscriptions(ctx)
425 for {
426 s, err := it.Next()
427 if errors.Is(err, iterator.Done) {
428 break
429 }
430 if err != nil {
431 return err
432 }
433 sID := s.ID()
434 p := strings.Split(sID, "-")
435
436 // Only delete resources created from the endtoend test.
437 // Otherwise, this will affect other tests running midflight.
438 if p[0] == resourcePrefix {
439 sCreated := p[len(p)-2]
440 timestamp, err := strconv.ParseInt(sCreated, 10, 64)
441 if err != nil {
442 continue
443 }
444 timeSCreated := time.Unix(0, timestamp)
445 if time.Since(timeSCreated) > expireAge {
446 log.Printf("deleting subscription %q", sID)
447 if err := s.Delete(ctx); err != nil {
448 return fmt.Errorf("Delete subscription: %v: %v", s.String(), err)
449 }
450 }
451 }
452 }
453 return nil
454 }