Debian Codegraph @

pkg/storage/cacher/cache_watcher_test.go

raw
1 /*
2 Copyright 2023 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package cacher
18
19 import (
20 "context"
21 "fmt"
22 "reflect"
23 "sync"
24 "testing"
25 "time"
26
27 "github.com/google/go-cmp/cmp"
28 v1 "k8s.io/api/core/v1"
29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30 "k8s.io/apimachinery/pkg/fields"
31 "k8s.io/apimachinery/pkg/labels"
32 "k8s.io/apimachinery/pkg/runtime/schema"
33 "k8s.io/apimachinery/pkg/util/wait"
34 "k8s.io/apimachinery/pkg/watch"
35 "k8s.io/apiserver/pkg/storage"
36 utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
37 "k8s.io/client-go/tools/cache"
38 testingclock "k8s.io/utils/clock/testing"
39 )
40
41 // verifies the cacheWatcher.process goroutine is properly cleaned up even if
42 // the writes to cacheWatcher.result channel is blocked.
43 func TestCacheWatcherCleanupNotBlockedByResult(t *testing.T) {
44 var lock sync.RWMutex
45 var w *cacheWatcher
46 count := 0
47 filter := func(string, labels.Set, fields.Set) bool { return true }
48 forget := func(drainWatcher bool) {
49 lock.Lock()
50 defer lock.Unlock()
51 count++
52 // forget() has to stop the watcher, as only stopping the watcher
53 // triggers stopping the process() goroutine which we are in the
54 // end waiting for in this test.
55 w.setDrainInputBufferLocked(drainWatcher)
56 w.stopLocked()
57 }
58 initEvents := []*watchCacheEvent{
59 {Object: &v1.Pod{}},
60 {Object: &v1.Pod{}},
61 }
62 // set the size of the buffer of w.result to 0, so that the writes to
63 // w.result is blocked.
64 w = newCacheWatcher(0, filter, forget, storage.APIObjectVersioner{}, time.Now(), false, schema.GroupResource{Resource: "pods"}, "")
65 go w.processInterval(context.Background(), intervalFromEvents(initEvents), 0)
66 w.Stop()
67 if err := wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) {
68 lock.RLock()
69 defer lock.RUnlock()
70 return count == 2, nil
71 }); err != nil {
72 t.Fatalf("expected forget() to be called twice, because sendWatchCacheEvent should not be blocked by the result channel: %v", err)
73 }
74 }
75
76 func TestCacheWatcherHandlesFiltering(t *testing.T) {
77 filter := func(_ string, _ labels.Set, field fields.Set) bool {
78 return field["spec.nodeName"] == "host"
79 }
80 forget := func(bool) {}
81
82 testCases := []struct {
83 events []*watchCacheEvent
84 expected []watch.Event
85 }{
86 // properly handle starting with the filter, then being deleted, then re-added
87 {
88 events: []*watchCacheEvent{
89 {
90 Type: watch.Added,
91 Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}},
92 ObjFields: fields.Set{"spec.nodeName": "host"},
93 ResourceVersion: 1,
94 },
95 {
96 Type: watch.Modified,
97 PrevObject: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}},
98 PrevObjFields: fields.Set{"spec.nodeName": "host"},
99 Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}},
100 ObjFields: fields.Set{"spec.nodeName": ""},
101 ResourceVersion: 2,
102 },
103 {
104 Type: watch.Modified,
105 PrevObject: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}},
106 PrevObjFields: fields.Set{"spec.nodeName": ""},
107 Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}},
108 ObjFields: fields.Set{"spec.nodeName": "host"},
109 ResourceVersion: 3,
110 },
111 },
112 expected: []watch.Event{
113 {Type: watch.Added, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}}},
114 {Type: watch.Deleted, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}}},
115 {Type: watch.Added, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}}},
116 },
117 },
118 // properly handle ignoring changes prior to the filter, then getting added, then deleted
119 {
120 events: []*watchCacheEvent{
121 {
122 Type: watch.Added,
123 Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}},
124 ObjFields: fields.Set{"spec.nodeName": ""},
125 ResourceVersion: 1,
126 },
127 {
128 Type: watch.Modified,
129 PrevObject: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}},
130 PrevObjFields: fields.Set{"spec.nodeName": ""},
131 Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}},
132 ObjFields: fields.Set{"spec.nodeName": ""},
133 ResourceVersion: 2,
134 },
135 {
136 Type: watch.Modified,
137 PrevObject: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}},
138 PrevObjFields: fields.Set{"spec.nodeName": ""},
139 Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}},
140 ObjFields: fields.Set{"spec.nodeName": "host"},
141 ResourceVersion: 3,
142 },
143 {
144 Type: watch.Modified,
145 PrevObject: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}},
146 PrevObjFields: fields.Set{"spec.nodeName": "host"},
147 Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "4"}},
148 ObjFields: fields.Set{"spec.nodeName": "host"},
149 ResourceVersion: 4,
150 },
151 {
152 Type: watch.Modified,
153 PrevObject: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "4"}},
154 PrevObjFields: fields.Set{"spec.nodeName": "host"},
155 Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "5"}},
156 ObjFields: fields.Set{"spec.nodeName": ""},
157 ResourceVersion: 5,
158 },
159 {
160 Type: watch.Modified,
161 PrevObject: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "5"}},
162 PrevObjFields: fields.Set{"spec.nodeName": ""},
163 Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "6"}},
164 ObjFields: fields.Set{"spec.nodeName": ""},
165 ResourceVersion: 6,
166 },
167 },
168 expected: []watch.Event{
169 {Type: watch.Added, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}}},
170 {Type: watch.Modified, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "4"}}},
171 {Type: watch.Deleted, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "5"}}},
172 },
173 },
174 }
175
176 TestCase:
177 for i, testCase := range testCases {
178 // set the size of the buffer of w.result to 0, so that the writes to
179 // w.result is blocked.
180 for j := range testCase.events {
181 testCase.events[j].ResourceVersion = uint64(j) + 1
182 }
183
184 w := newCacheWatcher(0, filter, forget, storage.APIObjectVersioner{}, time.Now(), false, schema.GroupResource{Resource: "pods"}, "")
185 go w.processInterval(context.Background(), intervalFromEvents(testCase.events), 0)
186
187 ch := w.ResultChan()
188 for j, event := range testCase.expected {
189 e := <-ch
190 if !reflect.DeepEqual(event, e) {
191 t.Errorf("%d: unexpected event %d: %s", i, j, cmp.Diff(event, e))
192 break TestCase
193 }
194 }
195 select {
196 case obj, ok := <-ch:
197 t.Errorf("%d: unexpected excess event: %#v %t", i, obj, ok)
198 break TestCase
199 default:
200 }
201 w.setDrainInputBufferLocked(false)
202 w.stopLocked()
203 }
204 }
205
206 func TestCacheWatcherStoppedInAnotherGoroutine(t *testing.T) {
207 var w *cacheWatcher
208 done := make(chan struct{})
209 filter := func(string, labels.Set, fields.Set) bool { return true }
210 forget := func(drainWatcher bool) {
211 w.setDrainInputBufferLocked(drainWatcher)
212 w.stopLocked()
213 done <- struct{}{}
214 }
215
216 maxRetriesToProduceTheRaceCondition := 1000
217 // Simulating the timer is fired and stopped concurrently by set time
218 // timeout to zero and run the Stop goroutine concurrently.
219 // May sure that the watch will not be blocked on Stop.
220 for i := 0; i < maxRetriesToProduceTheRaceCondition; i++ {
221 w = newCacheWatcher(0, filter, forget, storage.APIObjectVersioner{}, time.Now(), false, schema.GroupResource{Resource: "pods"}, "")
222 go w.Stop()
223 select {
224 case <-done:
225 case <-time.After(time.Second):
226 t.Fatal("stop is blocked when the timer is fired concurrently")
227 }
228 }
229
230 deadline := time.Now().Add(time.Hour)
231 // After that, verifies the cacheWatcher.process goroutine works correctly.
232 for i := 0; i < maxRetriesToProduceTheRaceCondition; i++ {
233 w = newCacheWatcher(2, filter, emptyFunc, storage.APIObjectVersioner{}, deadline, false, schema.GroupResource{Resource: "pods"}, "")
234 w.input <- &watchCacheEvent{Object: &v1.Pod{}, ResourceVersion: uint64(i + 1)}
235 ctx, cancel := context.WithDeadline(context.Background(), deadline)
236 defer cancel()
237 go w.processInterval(ctx, intervalFromEvents(nil), 0)
238 select {
239 case <-w.ResultChan():
240 case <-time.After(time.Second):
241 t.Fatal("expected received a event on ResultChan")
242 }
243 w.setDrainInputBufferLocked(false)
244 w.stopLocked()
245 }
246 }
247
248 func TestCacheWatcherStoppedOnDestroy(t *testing.T) {
249 backingStorage := &dummyStorage{}
250 cacher, _, err := newTestCacher(backingStorage)
251 if err != nil {
252 t.Fatalf("Couldn't create cacher: %v", err)
253 }
254 defer cacher.Stop()
255
256 // Wait until cacher is initialized.
257 if err := cacher.ready.wait(context.Background()); err != nil {
258 t.Fatalf("unexpected error waiting for the cache to be ready")
259 }
260
261 w, err := cacher.Watch(context.Background(), "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything})
262 if err != nil {
263 t.Fatalf("Failed to create watch: %v", err)
264 }
265
266 watchClosed := make(chan struct{})
267 go func() {
268 defer close(watchClosed)
269 for event := range w.ResultChan() {
270 switch event.Type {
271 case watch.Added, watch.Modified, watch.Deleted:
272 // ok
273 default:
274 t.Errorf("unexpected event %#v", event)
275 }
276 }
277 }()
278
279 cacher.Stop()
280
281 select {
282 case <-watchClosed:
283 case <-time.After(wait.ForeverTestTimeout):
284 t.Errorf("timed out waiting for watch to close")
285 }
286
287 }
288
289 func TestResourceVersionAfterInitEvents(t *testing.T) {
290 const numObjects = 10
291 store := cache.NewIndexer(storeElementKey, storeElementIndexers(nil))
292
293 for i := 0; i < numObjects; i++ {
294 elem := makeTestStoreElement(makeTestPod(fmt.Sprintf("pod-%d", i), uint64(i)))
295 store.Add(elem)
296 }
297
298 wci, err := newCacheIntervalFromStore(numObjects, store, "", false)
299 if err != nil {
300 t.Fatal(err)
301 }
302
303 filter := func(_ string, _ labels.Set, _ fields.Set) bool { return true }
304 forget := func(_ bool) {}
305 deadline := time.Now().Add(time.Minute)
306 w := newCacheWatcher(numObjects+1, filter, forget, storage.APIObjectVersioner{}, deadline, true, schema.GroupResource{Resource: "pods"}, "")
307
308 // Simulate a situation when the last event will that was already in
309 // the state, wasn't yet processed by cacher and will be delivered
310 // via channel again.
311 event := &watchCacheEvent{
312 Type: watch.Added,
313 Object: makeTestPod(fmt.Sprintf("pod-%d", numObjects-1), uint64(numObjects-1)),
314 ResourceVersion: uint64(numObjects - 1),
315 }
316 if !w.add(event, time.NewTimer(time.Second)) {
317 t.Fatalf("failed to add event")
318 }
319 w.stopLocked()
320
321 wg := sync.WaitGroup{}
322 wg.Add(1)
323 go func() {
324 defer wg.Done()
325 w.processInterval(context.Background(), wci, uint64(numObjects-1))
326 }()
327
328 // We expect all init events to be delivered.
329 for i := 0; i < numObjects; i++ {
330 <-w.ResultChan()
331 }
332 // We don't expect any other event to be delivered and thus
333 // the ResultChan to be closed.
334 result, ok := <-w.ResultChan()
335 if ok {
336 t.Errorf("unexpected event: %#v", result)
337 }
338
339 wg.Wait()
340 }
341
342 func TestTimeBucketWatchersBasic(t *testing.T) {
343 filter := func(_ string, _ labels.Set, _ fields.Set) bool {
344 return true
345 }
346 forget := func(bool) {}
347
348 newWatcher := func(deadline time.Time) *cacheWatcher {
349 w := newCacheWatcher(0, filter, forget, storage.APIObjectVersioner{}, deadline, true, schema.GroupResource{Resource: "pods"}, "")
350 w.setBookmarkAfterResourceVersion(0)
351 return w
352 }
353
354 clock := testingclock.NewFakeClock(time.Now())
355 watchers := newTimeBucketWatchers(clock, defaultBookmarkFrequency)
356 now := clock.Now()
357 watchers.addWatcherThreadUnsafe(newWatcher(now.Add(10 * time.Second)))
358 watchers.addWatcherThreadUnsafe(newWatcher(now.Add(20 * time.Second)))
359 watchers.addWatcherThreadUnsafe(newWatcher(now.Add(20 * time.Second)))
360
361 if len(watchers.watchersBuckets) != 2 {
362 t.Errorf("unexpected bucket size: %#v", watchers.watchersBuckets)
363 }
364 watchers0 := watchers.popExpiredWatchersThreadUnsafe()
365 if len(watchers0) != 0 {
366 t.Errorf("unexpected bucket size: %#v", watchers0)
367 }
368
369 clock.Step(10 * time.Second)
370 watchers1 := watchers.popExpiredWatchersThreadUnsafe()
371 if len(watchers1) != 1 || len(watchers1[0]) != 1 {
372 t.Errorf("unexpected bucket size: %v", watchers1)
373 }
374 watchers1 = watchers.popExpiredWatchersThreadUnsafe()
375 if len(watchers1) != 0 {
376 t.Errorf("unexpected bucket size: %#v", watchers1)
377 }
378
379 clock.Step(12 * time.Second)
380 watchers2 := watchers.popExpiredWatchersThreadUnsafe()
381 if len(watchers2) != 1 || len(watchers2[0]) != 2 {
382 t.Errorf("unexpected bucket size: %#v", watchers2)
383 }
384 }
385
386 func makeWatchCacheEvent(rv uint64) *watchCacheEvent {
387 return &watchCacheEvent{
388 Type: watch.Added,
389 Object: &v1.Pod{
390 ObjectMeta: metav1.ObjectMeta{
391 Name: fmt.Sprintf("pod-%d", rv),
392 ResourceVersion: fmt.Sprintf("%d", rv),
393 },
394 },
395 ResourceVersion: rv,
396 }
397 }
398
399 // TestCacheWatcherDraining verifies the cacheWatcher.process goroutine is properly cleaned up when draining was requested
400 func TestCacheWatcherDraining(t *testing.T) {
401 var lock sync.RWMutex
402 var w *cacheWatcher
403 count := 0
404 filter := func(string, labels.Set, fields.Set) bool { return true }
405 forget := func(drainWatcher bool) {
406 lock.Lock()
407 defer lock.Unlock()
408 count++
409 w.setDrainInputBufferLocked(drainWatcher)
410 w.stopLocked()
411 }
412 initEvents := []*watchCacheEvent{
413 makeWatchCacheEvent(5),
414 makeWatchCacheEvent(6),
415 }
416 w = newCacheWatcher(1, filter, forget, storage.APIObjectVersioner{}, time.Now(), true, schema.GroupResource{Resource: "pods"}, "")
417 go w.processInterval(context.Background(), intervalFromEvents(initEvents), 1)
418 if !w.add(makeWatchCacheEvent(7), time.NewTimer(1*time.Second)) {
419 t.Fatal("failed adding an even to the watcher")
420 }
421 forget(true) // drain the watcher
422
423 eventCount := 0
424 for range w.ResultChan() {
425 eventCount++
426 }
427 if eventCount != 3 {
428 t.Errorf("Unexpected number of objects received: %d, expected: 3", eventCount)
429 }
430 if err := wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) {
431 lock.RLock()
432 defer lock.RUnlock()
433 return count == 2, nil
434 }); err != nil {
435 t.Fatalf("expected forget() to be called twice, because processInterval should call Stop(): %v", err)
436 }
437 }
438
439 // TestCacheWatcherDrainingRequestedButNotDrained verifies the cacheWatcher.process goroutine is properly cleaned up when draining was requested
440 // but the client never actually get any data
441 func TestCacheWatcherDrainingRequestedButNotDrained(t *testing.T) {
442 var lock sync.RWMutex
443 var w *cacheWatcher
444 count := 0
445 filter := func(string, labels.Set, fields.Set) bool { return true }
446 forget := func(drainWatcher bool) {
447 lock.Lock()
448 defer lock.Unlock()
449 count++
450 w.setDrainInputBufferLocked(drainWatcher)
451 w.stopLocked()
452 }
453 initEvents := []*watchCacheEvent{
454 makeWatchCacheEvent(5),
455 makeWatchCacheEvent(6),
456 }
457 w = newCacheWatcher(1, filter, forget, storage.APIObjectVersioner{}, time.Now(), true, schema.GroupResource{Resource: "pods"}, "")
458 go w.processInterval(context.Background(), intervalFromEvents(initEvents), 1)
459 if !w.add(makeWatchCacheEvent(7), time.NewTimer(1*time.Second)) {
460 t.Fatal("failed adding an even to the watcher")
461 }
462 forget(true) // drain the watcher
463 w.Stop() // client disconnected, timeout expired or ctx was actually closed
464 if err := wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) {
465 lock.RLock()
466 defer lock.RUnlock()
467 return count == 3, nil
468 }); err != nil {
469 t.Fatalf("expected forget() to be called three times, because processInterval should call Stop(): %v", err)
470 }
471 }
472
473 // TestCacheWatcherDrainingNoBookmarkAfterResourceVersionReceived verifies if the watcher will be stopped
474 // when adding an item times out and the bookmarkAfterResourceVersion hasn't been received
475 func TestCacheWatcherDrainingNoBookmarkAfterResourceVersionReceived(t *testing.T) {
476 var lock sync.RWMutex
477 var w *cacheWatcher
478 count := 0
479 filter := func(string, labels.Set, fields.Set) bool { return true }
480 forget := func(drainWatcher bool) {
481 lock.Lock()
482 defer lock.Unlock()
483 if drainWatcher {
484 t.Fatalf("didn't expect drainWatcher to be set to true")
485 }
486 count++
487 w.setDrainInputBufferLocked(drainWatcher)
488 w.stopLocked()
489 }
490 initEvents := []*watchCacheEvent{
491 {Object: &v1.Pod{}},
492 {Object: &v1.Pod{}},
493 }
494 w = newCacheWatcher(0, filter, forget, storage.APIObjectVersioner{}, time.Now(), true, schema.GroupResource{Resource: "pods"}, "")
495 w.setBookmarkAfterResourceVersion(10)
496 go w.processInterval(context.Background(), intervalFromEvents(initEvents), 0)
497
498 // get an event so that
499 // we know the w.processInterval
500 // has been scheduled, and
501 // it will be blocked on
502 // sending the other event
503 // to the result chan
504 <-w.ResultChan()
505
506 // now, once we know, the processInterval
507 // is waiting add another event that will time out
508 // and start the cleanup process
509 if w.add(&watchCacheEvent{Object: &v1.Pod{}}, time.NewTimer(10*time.Millisecond)) {
510 t.Fatal("expected the add method to fail")
511 }
512 if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(_ context.Context) (bool, error) {
513 lock.RLock()
514 defer lock.RUnlock()
515 return count == 2, nil
516 }); err != nil {
517 t.Fatalf("expected forget() to be called twice, first call from w.add() and then from w.Stop() called from w.processInterval(): %v", err)
518 }
519
520 if !w.stopped {
521 t.Fatal("expected the watcher to be stopped but it wasn't")
522 }
523 }
524
525 // TestCacheWatcherDrainingNoBookmarkAfterResourceVersionSent checks if the watcher's input
526 // channel is drained if the bookmarkAfterResourceVersion was received but not sent
527 func TestCacheWatcherDrainingNoBookmarkAfterResourceVersionSent(t *testing.T) {
528 makePod := func(rv uint64) *v1.Pod {
529 return &v1.Pod{
530 ObjectMeta: metav1.ObjectMeta{
531 Name: fmt.Sprintf("pod-%d", rv),
532 Namespace: "ns",
533 ResourceVersion: fmt.Sprintf("%d", rv),
534 Annotations: map[string]string{},
535 },
536 }
537 }
538 var lock sync.RWMutex
539 var w *cacheWatcher
540 watchInitializationSignal := utilflowcontrol.NewInitializationSignal()
541 ctx := utilflowcontrol.WithInitializationSignal(context.Background(), watchInitializationSignal)
542 count := 0
543 filter := func(string, labels.Set, fields.Set) bool { return true }
544 forget := func(drainWatcher bool) {
545 lock.Lock()
546 defer lock.Unlock()
547 count++
548 w.setDrainInputBufferLocked(drainWatcher)
549 w.stopLocked()
550 }
551 initEvents := []*watchCacheEvent{{Object: makePod(1)}, {Object: makePod(2)}}
552 w = newCacheWatcher(2, filter, forget, storage.APIObjectVersioner{}, time.Now(), true, schema.GroupResource{Resource: "pods"}, "")
553 w.setBookmarkAfterResourceVersion(10)
554 go w.processInterval(ctx, intervalFromEvents(initEvents), 0)
555 watchInitializationSignal.Wait()
556
557 // note that we can add three events even though the chanSize is two because
558 // one event has been popped off from the input chan
559 if !w.add(&watchCacheEvent{Object: makePod(5), ResourceVersion: 5}, time.NewTimer(1*time.Second)) {
560 t.Fatal("failed adding an even to the watcher")
561 }
562 if !w.nonblockingAdd(&watchCacheEvent{Type: watch.Bookmark, ResourceVersion: 10, Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "10"}}}) {
563 t.Fatal("failed adding an even to the watcher")
564 }
565 if !w.add(&watchCacheEvent{Object: makePod(15), ResourceVersion: 15}, time.NewTimer(1*time.Second)) {
566 t.Fatal("failed adding an even to the watcher")
567 }
568 if w.add(&watchCacheEvent{Object: makePod(20), ResourceVersion: 20}, time.NewTimer(1*time.Second)) {
569 t.Fatal("expected the add method to fail")
570 }
571 if err := wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) {
572 lock.RLock()
573 defer lock.RUnlock()
574 return count == 1, nil
575 }); err != nil {
576 t.Fatalf("expected forget() to be called once, just from the w.add() method: %v", err)
577 }
578
579 if !w.stopped {
580 t.Fatal("expected the watcher to be stopped but it wasn't")
581 }
582 verifyEvents(t, w, []watch.Event{
583 {Type: watch.Added, Object: makePod(1)},
584 {Type: watch.Added, Object: makePod(2)},
585 {Type: watch.Added, Object: makePod(5)},
586 {Type: watch.Bookmark, Object: &v1.Pod{
587 ObjectMeta: metav1.ObjectMeta{
588 ResourceVersion: "10",
589 Annotations: map[string]string{metav1.InitialEventsAnnotationKey: "true"},
590 },
591 }},
592 {Type: watch.Added, Object: makePod(15)},
593 }, true)
594
595 if err := wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) {
596 lock.RLock()
597 defer lock.RUnlock()
598 return count == 2, nil
599 }); err != nil {
600 t.Fatalf("expected forget() to be called twice, the second call is from w.Stop() method called from w.processInterval(): %v", err)
601 }
602 }
603
604 func TestBookmarkAfterResourceVersionWatchers(t *testing.T) {
605 newWatcher := func(id string, deadline time.Time) *cacheWatcher {
606 w := newCacheWatcher(0, func(_ string, _ labels.Set, _ fields.Set) bool { return true }, func(bool) {}, storage.APIObjectVersioner{}, deadline, true, schema.GroupResource{Resource: "pods"}, id)
607 w.setBookmarkAfterResourceVersion(10)
608 return w
609 }
610
611 clock := testingclock.NewFakeClock(time.Now())
612 target := newTimeBucketWatchers(clock, defaultBookmarkFrequency)
613 if !target.addWatcherThreadUnsafe(newWatcher("1", clock.Now().Add(2*time.Minute))) {
614 t.Fatal("failed adding an even to the watcher")
615 }
616
617 // the watcher is immediately expired (it's waiting for bookmark, so it is scheduled immediately)
618 ret := target.popExpiredWatchersThreadUnsafe()
619 if len(ret) != 1 || len(ret[0]) != 1 {
620 t.Fatalf("expected only one watcher to be expired")
621 }
622 if !target.addWatcherThreadUnsafe(ret[0][0]) {
623 t.Fatal("failed adding an even to the watcher")
624 }
625
626 // after one second time the watcher is still expired
627 clock.Step(1 * time.Second)
628 ret = target.popExpiredWatchersThreadUnsafe()
629 if len(ret) != 1 || len(ret[0]) != 1 {
630 t.Fatalf("expected only one watcher to be expired")
631 }
632 if !target.addWatcherThreadUnsafe(ret[0][0]) {
633 t.Fatal("failed adding an even to the watcher")
634 }
635
636 // after 29 seconds the watcher is still expired
637 clock.Step(29 * time.Second)
638 ret = target.popExpiredWatchersThreadUnsafe()
639 if len(ret) != 1 || len(ret[0]) != 1 {
640 t.Fatalf("expected only one watcher to be expired")
641 }
642
643 // after confirming the watcher is not expired immediately
644 ret[0][0].markBookmarkAfterRvAsReceived(&watchCacheEvent{Type: watch.Bookmark, ResourceVersion: 10, Object: &v1.Pod{}})
645 if !target.addWatcherThreadUnsafe(ret[0][0]) {
646 t.Fatal("failed adding an even to the watcher")
647 }
648 clock.Step(30 * time.Second)
649 ret = target.popExpiredWatchersThreadUnsafe()
650 if len(ret) != 0 {
651 t.Fatalf("didn't expect any watchers to be expired")
652 }
653
654 clock.Step(30 * time.Second)
655 ret = target.popExpiredWatchersThreadUnsafe()
656 if len(ret) != 1 || len(ret[0]) != 1 {
657 t.Fatalf("expected only one watcher to be expired")
658 }
659 }