Debian Codegraph @

pkg/storage/cacher/watch_cache_test.go

raw
1 /*
2 Copyright 2014 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package cacher
18
19 import (
20 "context"
21 "fmt"
22 "strconv"
23 "strings"
24 "testing"
25 "time"
26
27 "github.com/stretchr/testify/assert"
28 "github.com/stretchr/testify/require"
29
30 v1 "k8s.io/api/core/v1"
31 apiequality "k8s.io/apimachinery/pkg/api/equality"
32 "k8s.io/apimachinery/pkg/api/errors"
33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34 "k8s.io/apimachinery/pkg/fields"
35 "k8s.io/apimachinery/pkg/labels"
36 "k8s.io/apimachinery/pkg/runtime"
37 "k8s.io/apimachinery/pkg/runtime/schema"
38 "k8s.io/apimachinery/pkg/util/wait"
39 "k8s.io/apimachinery/pkg/watch"
40 "k8s.io/apiserver/pkg/features"
41 "k8s.io/apiserver/pkg/storage"
42 "k8s.io/apiserver/pkg/storage/cacher/metrics"
43 "k8s.io/apiserver/pkg/storage/cacher/progress"
44 utilfeature "k8s.io/apiserver/pkg/util/feature"
45 "k8s.io/client-go/tools/cache"
46 featuregatetesting "k8s.io/component-base/featuregate/testing"
47 k8smetrics "k8s.io/component-base/metrics"
48 "k8s.io/component-base/metrics/testutil"
49 "k8s.io/utils/clock"
50 testingclock "k8s.io/utils/clock/testing"
51 )
52
53 func makeTestPod(name string, resourceVersion uint64) *v1.Pod {
54 return makeTestPodDetails(name, resourceVersion, "some-node", map[string]string{"k8s-app": "my-app"})
55 }
56
57 func makeTestPodDetails(name string, resourceVersion uint64, nodeName string, labels map[string]string) *v1.Pod {
58 return &v1.Pod{
59 ObjectMeta: metav1.ObjectMeta{
60 Namespace: "ns",
61 Name: name,
62 ResourceVersion: strconv.FormatUint(resourceVersion, 10),
63 Labels: labels,
64 },
65 Spec: v1.PodSpec{
66 NodeName: nodeName,
67 },
68 }
69 }
70
71 func makeTestStoreElement(pod *v1.Pod) *storeElement {
72 return &storeElement{
73 Key: "prefix/ns/" + pod.Name,
74 Object: pod,
75 Labels: labels.Set(pod.Labels),
76 Fields: fields.Set{"spec.nodeName": pod.Spec.NodeName},
77 }
78 }
79
80 type testWatchCache struct {
81 *watchCache
82
83 bookmarkRevision chan int64
84 stopCh chan struct{}
85 }
86
87 func (w *testWatchCache) getAllEventsSince(resourceVersion uint64, opts storage.ListOptions) ([]*watchCacheEvent, error) {
88 cacheInterval, err := w.getCacheIntervalForEvents(resourceVersion, opts)
89 if err != nil {
90 return nil, err
91 }
92
93 result := []*watchCacheEvent{}
94 for {
95 event, err := cacheInterval.Next()
96 if err != nil {
97 return nil, err
98 }
99 if event == nil {
100 break
101 }
102 result = append(result, event)
103 }
104
105 return result, nil
106 }
107
108 func (w *testWatchCache) getCacheIntervalForEvents(resourceVersion uint64, opts storage.ListOptions) (*watchCacheInterval, error) {
109 w.RLock()
110 defer w.RUnlock()
111
112 return w.getAllEventsSinceLocked(resourceVersion, "", opts)
113 }
114
115 // newTestWatchCache just adds a fake clock.
116 func newTestWatchCache(capacity int, eventFreshDuration time.Duration, indexers *cache.Indexers) *testWatchCache {
117 keyFunc := func(obj runtime.Object) (string, error) {
118 return storage.NamespaceKeyFunc("prefix", obj)
119 }
120 getAttrsFunc := func(obj runtime.Object) (labels.Set, fields.Set, error) {
121 pod, ok := obj.(*v1.Pod)
122 if !ok {
123 return nil, nil, fmt.Errorf("not a pod")
124 }
125 return labels.Set(pod.Labels), fields.Set{"spec.nodeName": pod.Spec.NodeName}, nil
126 }
127 versioner := storage.APIObjectVersioner{}
128 mockHandler := func(*watchCacheEvent) {}
129 wc := &testWatchCache{}
130 wc.bookmarkRevision = make(chan int64, 1)
131 wc.stopCh = make(chan struct{})
132 pr := progress.NewConditionalProgressRequester(wc.RequestWatchProgress, &immediateTickerFactory{}, nil)
133 go pr.Run(wc.stopCh)
134 wc.watchCache = newWatchCache(keyFunc, mockHandler, getAttrsFunc, versioner, indexers, testingclock.NewFakeClock(time.Now()), eventFreshDuration, schema.GroupResource{Resource: "pods"}, pr)
135 // To preserve behavior of tests that assume a given capacity,
136 // resize it to th expected size.
137 wc.capacity = capacity
138 wc.cache = make([]*watchCacheEvent, capacity)
139 wc.lowerBoundCapacity = min(capacity, defaultLowerBoundCapacity)
140 wc.upperBoundCapacity = max(capacity, defaultUpperBoundCapacity)
141
142 return wc
143 }
144
145 type immediateTickerFactory struct{}
146
147 func (t *immediateTickerFactory) NewTimer(d time.Duration) clock.Timer {
148 timer := immediateTicker{
149 c: make(chan time.Time),
150 }
151 timer.Reset(d)
152 return &timer
153 }
154
155 type immediateTicker struct {
156 c chan time.Time
157 }
158
159 func (t *immediateTicker) Reset(d time.Duration) (active bool) {
160 select {
161 case <-t.c:
162 active = true
163 default:
164 }
165 go func() {
166 t.c <- time.Now()
167 }()
168 return active
169 }
170
171 func (t *immediateTicker) C() <-chan time.Time {
172 return t.c
173 }
174
175 func (t *immediateTicker) Stop() bool {
176 select {
177 case <-t.c:
178 return true
179 default:
180 return false
181 }
182 }
183
184 func (w *testWatchCache) RequestWatchProgress(ctx context.Context) error {
185 go func() {
186 select {
187 case rev := <-w.bookmarkRevision:
188 w.UpdateResourceVersion(fmt.Sprintf("%d", rev))
189 case <-ctx.Done():
190 return
191 }
192 }()
193 return nil
194 }
195
196 func (w *testWatchCache) Stop() {
197 close(w.stopCh)
198 }
199
200 func TestWatchCacheBasic(t *testing.T) {
201 store := newTestWatchCache(2, DefaultEventFreshDuration, &cache.Indexers{})
202 defer store.Stop()
203
204 // Test Add/Update/Delete.
205 pod1 := makeTestPod("pod", 1)
206 if err := store.Add(pod1); err != nil {
207 t.Errorf("unexpected error: %v", err)
208 }
209 if item, ok, _ := store.Get(pod1); !ok {
210 t.Errorf("didn't find pod")
211 } else {
212 expected := makeTestStoreElement(makeTestPod("pod", 1))
213 if !apiequality.Semantic.DeepEqual(expected, item) {
214 t.Errorf("expected %v, got %v", expected, item)
215 }
216 }
217 pod2 := makeTestPod("pod", 2)
218 if err := store.Update(pod2); err != nil {
219 t.Errorf("unexpected error: %v", err)
220 }
221 if item, ok, _ := store.Get(pod2); !ok {
222 t.Errorf("didn't find pod")
223 } else {
224 expected := makeTestStoreElement(makeTestPod("pod", 2))
225 if !apiequality.Semantic.DeepEqual(expected, item) {
226 t.Errorf("expected %v, got %v", expected, item)
227 }
228 }
229 pod3 := makeTestPod("pod", 3)
230 if err := store.Delete(pod3); err != nil {
231 t.Errorf("unexpected error: %v", err)
232 }
233 if _, ok, _ := store.Get(pod3); ok {
234 t.Errorf("found pod")
235 }
236
237 // Test List.
238 store.Add(makeTestPod("pod1", 4))
239 store.Add(makeTestPod("pod2", 5))
240 store.Add(makeTestPod("pod3", 6))
241 {
242 expected := map[string]storeElement{
243 "prefix/ns/pod1": *makeTestStoreElement(makeTestPod("pod1", 4)),
244 "prefix/ns/pod2": *makeTestStoreElement(makeTestPod("pod2", 5)),
245 "prefix/ns/pod3": *makeTestStoreElement(makeTestPod("pod3", 6)),
246 }
247 items := make(map[string]storeElement)
248 for _, item := range store.List() {
249 elem := item.(*storeElement)
250 items[elem.Key] = *elem
251 }
252 if !apiequality.Semantic.DeepEqual(expected, items) {
253 t.Errorf("expected %v, got %v", expected, items)
254 }
255 }
256
257 // Test Replace.
258 store.Replace([]interface{}{
259 makeTestPod("pod4", 7),
260 makeTestPod("pod5", 8),
261 }, "8")
262 {
263 expected := map[string]storeElement{
264 "prefix/ns/pod4": *makeTestStoreElement(makeTestPod("pod4", 7)),
265 "prefix/ns/pod5": *makeTestStoreElement(makeTestPod("pod5", 8)),
266 }
267 items := make(map[string]storeElement)
268 for _, item := range store.List() {
269 elem := item.(*storeElement)
270 items[elem.Key] = *elem
271 }
272 if !apiequality.Semantic.DeepEqual(expected, items) {
273 t.Errorf("expected %v, got %v", expected, items)
274 }
275 }
276 }
277
278 func TestEvents(t *testing.T) {
279 store := newTestWatchCache(5, DefaultEventFreshDuration, &cache.Indexers{})
280 defer store.Stop()
281
282 // no dynamic-size cache to fit old tests.
283 store.lowerBoundCapacity = 5
284 store.upperBoundCapacity = 5
285
286 store.Add(makeTestPod("pod", 3))
287
288 // Test for Added event.
289 {
290 _, err := store.getAllEventsSince(1, storage.ListOptions{Predicate: storage.Everything})
291 if err == nil {
292 t.Errorf("expected error too old")
293 }
294 if _, ok := err.(*errors.StatusError); !ok {
295 t.Errorf("expected error to be of type StatusError")
296 }
297 }
298 {
299 result, err := store.getAllEventsSince(2, storage.ListOptions{Predicate: storage.Everything})
300 if err != nil {
301 t.Errorf("unexpected error: %v", err)
302 }
303 if len(result) != 1 {
304 t.Fatalf("unexpected events: %v", result)
305 }
306 if result[0].Type != watch.Added {
307 t.Errorf("unexpected event type: %v", result[0].Type)
308 }
309 pod := makeTestPod("pod", uint64(3))
310 if !apiequality.Semantic.DeepEqual(pod, result[0].Object) {
311 t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod)
312 }
313 if result[0].PrevObject != nil {
314 t.Errorf("unexpected item: %v", result[0].PrevObject)
315 }
316 }
317
318 store.Update(makeTestPod("pod", 4))
319 store.Update(makeTestPod("pod", 5))
320
321 // Test with not full cache.
322 {
323 _, err := store.getAllEventsSince(1, storage.ListOptions{Predicate: storage.Everything})
324 if err == nil {
325 t.Errorf("expected error too old")
326 }
327 }
328 {
329 result, err := store.getAllEventsSince(3, storage.ListOptions{Predicate: storage.Everything})
330 if err != nil {
331 t.Errorf("unexpected error: %v", err)
332 }
333 if len(result) != 2 {
334 t.Fatalf("unexpected events: %v", result)
335 }
336 for i := 0; i < 2; i++ {
337 if result[i].Type != watch.Modified {
338 t.Errorf("unexpected event type: %v", result[i].Type)
339 }
340 pod := makeTestPod("pod", uint64(i+4))
341 if !apiequality.Semantic.DeepEqual(pod, result[i].Object) {
342 t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod)
343 }
344 prevPod := makeTestPod("pod", uint64(i+3))
345 if !apiequality.Semantic.DeepEqual(prevPod, result[i].PrevObject) {
346 t.Errorf("unexpected item: %v, expected: %v", result[i].PrevObject, prevPod)
347 }
348 }
349 }
350
351 for i := 6; i < 10; i++ {
352 store.Update(makeTestPod("pod", uint64(i)))
353 }
354
355 // Test with full cache - there should be elements from 5 to 9.
356 {
357 _, err := store.getAllEventsSince(3, storage.ListOptions{Predicate: storage.Everything})
358 if err == nil {
359 t.Errorf("expected error too old")
360 }
361 }
362 {
363 result, err := store.getAllEventsSince(4, storage.ListOptions{Predicate: storage.Everything})
364 if err != nil {
365 t.Errorf("unexpected error: %v", err)
366 }
367 if len(result) != 5 {
368 t.Fatalf("unexpected events: %v", result)
369 }
370 for i := 0; i < 5; i++ {
371 pod := makeTestPod("pod", uint64(i+5))
372 if !apiequality.Semantic.DeepEqual(pod, result[i].Object) {
373 t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod)
374 }
375 }
376 }
377
378 // Test for delete event.
379 store.Delete(makeTestPod("pod", uint64(10)))
380
381 {
382 result, err := store.getAllEventsSince(9, storage.ListOptions{Predicate: storage.Everything})
383 if err != nil {
384 t.Errorf("unexpected error: %v", err)
385 }
386 if len(result) != 1 {
387 t.Fatalf("unexpected events: %v", result)
388 }
389 if result[0].Type != watch.Deleted {
390 t.Errorf("unexpected event type: %v", result[0].Type)
391 }
392 pod := makeTestPod("pod", uint64(10))
393 if !apiequality.Semantic.DeepEqual(pod, result[0].Object) {
394 t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod)
395 }
396 prevPod := makeTestPod("pod", uint64(9))
397 if !apiequality.Semantic.DeepEqual(prevPod, result[0].PrevObject) {
398 t.Errorf("unexpected item: %v, expected: %v", result[0].PrevObject, prevPod)
399 }
400 }
401 }
402
403 func TestMarker(t *testing.T) {
404 store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{})
405 defer store.Stop()
406
407 // First thing that is called when propagated from storage is Replace.
408 store.Replace([]interface{}{
409 makeTestPod("pod1", 5),
410 makeTestPod("pod2", 9),
411 }, "9")
412
413 _, err := store.getAllEventsSince(8, storage.ListOptions{Predicate: storage.Everything})
414 if err == nil || !strings.Contains(err.Error(), "too old resource version") {
415 t.Errorf("unexpected error: %v", err)
416 }
417 // Getting events from 8 should return no events,
418 // even though there is a marker there.
419 result, err := store.getAllEventsSince(9, storage.ListOptions{Predicate: storage.Everything})
420 if err != nil {
421 t.Fatalf("unexpected error: %v", err)
422 }
423 if len(result) != 0 {
424 t.Errorf("unexpected result: %#v, expected no events", result)
425 }
426
427 pod := makeTestPod("pods", 12)
428 store.Add(pod)
429 // Getting events from 8 should still work and return one event.
430 result, err = store.getAllEventsSince(9, storage.ListOptions{Predicate: storage.Everything})
431 if err != nil {
432 t.Fatalf("unexpected error: %v", err)
433 }
434 if len(result) != 1 || !apiequality.Semantic.DeepEqual(result[0].Object, pod) {
435 t.Errorf("unexpected result: %#v, expected %v", result, pod)
436 }
437 }
438
439 func TestWaitUntilFreshAndList(t *testing.T) {
440 ctx := context.Background()
441 store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{
442 "l:label": func(obj interface{}) ([]string, error) {
443 pod, ok := obj.(*v1.Pod)
444 if !ok {
445 return nil, fmt.Errorf("not a pod %#v", obj)
446 }
447 if value, ok := pod.Labels["label"]; ok {
448 return []string{value}, nil
449 }
450 return nil, nil
451 },
452 "f:spec.nodeName": func(obj interface{}) ([]string, error) {
453 pod, ok := obj.(*v1.Pod)
454 if !ok {
455 return nil, fmt.Errorf("not a pod %#v", obj)
456 }
457 return []string{pod.Spec.NodeName}, nil
458 },
459 })
460 defer store.Stop()
461 // In background, update the store.
462 go func() {
463 store.Add(makeTestPodDetails("pod1", 2, "node1", map[string]string{"label": "value1"}))
464 store.Add(makeTestPodDetails("pod2", 3, "node1", map[string]string{"label": "value1"}))
465 store.Add(makeTestPodDetails("pod3", 5, "node2", map[string]string{"label": "value2"}))
466 }()
467
468 // list by empty MatchValues.
469 resp, indexUsed, err := store.WaitUntilFreshAndList(ctx, 5, "prefix/", storage.ListOptions{Predicate: storage.Everything})
470 if err != nil {
471 t.Fatalf("unexpected error: %v", err)
472 }
473 if resp.ResourceVersion != 5 {
474 t.Errorf("unexpected resourceVersion: %v, expected: 5", resp.ResourceVersion)
475 }
476 if len(resp.Items) != 3 {
477 t.Errorf("unexpected list returned: %#v", resp)
478 }
479 if indexUsed != "" {
480 t.Errorf("Used index %q but expected none to be used", indexUsed)
481 }
482
483 // list by label index.
484 resp, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", storage.ListOptions{Predicate: storage.SelectionPredicate{
485 Label: labels.SelectorFromSet(map[string]string{
486 "label": "value1",
487 }),
488 Field: fields.SelectorFromSet(map[string]string{
489 "spec.nodeName": "node2",
490 }),
491 IndexLabels: []string{"label"},
492 }})
493 if err != nil {
494 t.Fatalf("unexpected error: %v", err)
495 }
496 if resp.ResourceVersion != 5 {
497 t.Errorf("unexpected resourceVersion: %v, expected: 5", resp.ResourceVersion)
498 }
499 if len(resp.Items) != 2 {
500 t.Errorf("unexpected list returned: %#v", resp)
501 }
502 if indexUsed != "l:label" {
503 t.Errorf("Used index %q but expected %q", indexUsed, "l:label")
504 }
505
506 // list with spec.nodeName index.
507 resp, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", storage.ListOptions{Predicate: storage.SelectionPredicate{
508 Label: labels.SelectorFromSet(map[string]string{
509 "not-exist-label": "whatever",
510 }),
511 Field: fields.SelectorFromSet(map[string]string{
512 "spec.nodeName": "node2",
513 }),
514 IndexFields: []string{"spec.nodeName"},
515 }})
516 if err != nil {
517 t.Fatalf("unexpected error: %v", err)
518 }
519 if resp.ResourceVersion != 5 {
520 t.Errorf("unexpected resourceVersion: %v, expected: 5", resp.ResourceVersion)
521 }
522 if len(resp.Items) != 1 {
523 t.Errorf("unexpected list returned: %#v", resp)
524 }
525 if indexUsed != "f:spec.nodeName" {
526 t.Errorf("Used index %q but expected %q", indexUsed, "f:spec.nodeName")
527 }
528
529 // list with index not exists.
530 resp, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", storage.ListOptions{Predicate: storage.SelectionPredicate{
531 Label: labels.SelectorFromSet(map[string]string{
532 "not-exist-label": "whatever",
533 }),
534 Field: fields.Everything(),
535 IndexLabels: []string{"label"},
536 }})
537 if err != nil {
538 t.Fatalf("unexpected error: %v", err)
539 }
540 if resp.ResourceVersion != 5 {
541 t.Errorf("unexpected resourceVersion: %v, expected: 5", resp.ResourceVersion)
542 }
543 if len(resp.Items) != 3 {
544 t.Errorf("unexpected list returned: %#v", resp)
545 }
546 if indexUsed != "" {
547 t.Errorf("Used index %q but expected none to be used", indexUsed)
548 }
549 }
550
551 func TestWaitUntilFreshAndListFromCache(t *testing.T) {
552 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true)
553 forceRequestWatchProgressSupport(t)
554 ctx := context.Background()
555 store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{})
556 defer store.Stop()
557 // In background, update the store.
558 go func() {
559 store.Add(makeTestPod("pod1", 2))
560 store.bookmarkRevision <- 3
561 }()
562
563 // list from future revision. Requires watch cache to request bookmark to get it.
564 resp, indexUsed, err := store.WaitUntilFreshAndList(ctx, 3, "prefix/", storage.ListOptions{Predicate: storage.Everything})
565 if err != nil {
566 t.Fatalf("unexpected error: %v", err)
567 }
568 if resp.ResourceVersion != 3 {
569 t.Errorf("unexpected resourceVersion: %v, expected: 6", resp.ResourceVersion)
570 }
571 if len(resp.Items) != 1 {
572 t.Errorf("unexpected list returned: %#v", resp)
573 }
574 if indexUsed != "" {
575 t.Errorf("Used index %q but expected none to be used", indexUsed)
576 }
577 }
578
579 func TestWaitUntilFreshAndGet(t *testing.T) {
580 ctx := context.Background()
581 store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{})
582 defer store.Stop()
583
584 // In background, update the store.
585 go func() {
586 store.Add(makeTestPod("foo", 2))
587 store.Add(makeTestPod("bar", 5))
588 }()
589
590 obj, exists, resourceVersion, err := store.WaitUntilFreshAndGet(ctx, 5, "prefix/ns/bar")
591 if err != nil {
592 t.Fatalf("unexpected error: %v", err)
593 }
594 if resourceVersion != 5 {
595 t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion)
596 }
597 if !exists {
598 t.Fatalf("no results returned: %#v", obj)
599 }
600 expected := makeTestStoreElement(makeTestPod("bar", 5))
601 if !apiequality.Semantic.DeepEqual(expected, obj) {
602 t.Errorf("expected %v, got %v", expected, obj)
603 }
604 }
605
606 func TestWaitUntilFreshAndListTimeout(t *testing.T) {
607 tcs := []struct {
608 name string
609 ConsistentListFromCache bool
610 }{
611 {
612 name: "FromStorage",
613 ConsistentListFromCache: false,
614 },
615 {
616 name: "FromCache",
617 ConsistentListFromCache: true,
618 },
619 }
620 for _, tc := range tcs {
621 t.Run(tc.name, func(t *testing.T) {
622 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, tc.ConsistentListFromCache)
623 ctx := context.Background()
624 store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{})
625 defer store.Stop()
626 fc := store.clock.(*testingclock.FakeClock)
627
628 // In background, step clock after the below call starts the timer.
629 go func() {
630 for !fc.HasWaiters() {
631 time.Sleep(time.Millisecond)
632 }
633 store.Add(makeTestPod("foo", 2))
634 store.bookmarkRevision <- 3
635 fc.Step(blockTimeout)
636
637 // Add an object to make sure the test would
638 // eventually fail instead of just waiting
639 // forever.
640 time.Sleep(30 * time.Second)
641 store.Add(makeTestPod("bar", 4))
642 }()
643
644 _, _, err := store.WaitUntilFreshAndList(ctx, 4, "", storage.ListOptions{Predicate: storage.Everything})
645 if !errors.IsTimeout(err) {
646 t.Errorf("expected timeout error but got: %v", err)
647 }
648 if !storage.IsTooLargeResourceVersion(err) {
649 t.Errorf("expected 'Too large resource version' cause in error but got: %v", err)
650 }
651 })
652 }
653 }
654
655 type testLW struct {
656 ListFunc func(options metav1.ListOptions) (runtime.Object, error)
657 WatchFunc func(options metav1.ListOptions) (watch.Interface, error)
658 }
659
660 func (t *testLW) List(options metav1.ListOptions) (runtime.Object, error) {
661 return t.ListFunc(options)
662 }
663 func (t *testLW) Watch(options metav1.ListOptions) (watch.Interface, error) {
664 return t.WatchFunc(options)
665 }
666
667 func TestReflectorForWatchCache(t *testing.T) {
668 ctx := context.Background()
669 store := newTestWatchCache(5, DefaultEventFreshDuration, &cache.Indexers{})
670 defer store.Stop()
671
672 {
673 resp, _, err := store.WaitUntilFreshAndList(ctx, 0, "", storage.ListOptions{Predicate: storage.Everything})
674 if err != nil {
675 t.Fatalf("unexpected error: %v", err)
676 }
677 if resp.ResourceVersion != 0 {
678 t.Errorf("unexpected resource version: %d", resp.ResourceVersion)
679 }
680 }
681
682 lw := &testLW{
683 WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
684 fw := watch.NewFake()
685 go fw.Stop()
686 return fw, nil
687 },
688 ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
689 return &v1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "10"}}, nil
690 },
691 }
692 r := cache.NewReflector(lw, &v1.Pod{}, store, 0)
693 r.ListAndWatch(wait.NeverStop)
694
695 {
696 resp, _, err := store.WaitUntilFreshAndList(ctx, 10, "", storage.ListOptions{Predicate: storage.Everything})
697 if err != nil {
698 t.Fatalf("unexpected error: %v", err)
699 }
700 if resp.ResourceVersion != 10 {
701 t.Errorf("unexpected resource version: %d", resp.ResourceVersion)
702 }
703 }
704 }
705
706 func TestDynamicCache(t *testing.T) {
707 tests := []struct {
708 name string
709 eventCount int
710 cacheCapacity int
711 startIndex int
712 // interval is time duration between adjacent events.
713 lowerBoundCapacity int
714 upperBoundCapacity int
715 interval time.Duration
716 expectCapacity int
717 expectStartIndex int
718 }{
719 {
720 name: "[capacity not equals 4*n] events inside DefaultEventFreshDuration cause cache expanding",
721 eventCount: 5,
722 cacheCapacity: 5,
723 lowerBoundCapacity: 5 / 2,
724 upperBoundCapacity: 5 * 2,
725 interval: DefaultEventFreshDuration / 6,
726 expectCapacity: 10,
727 expectStartIndex: 0,
728 },
729 {
730 name: "[capacity not equals 4*n] events outside DefaultEventFreshDuration without change cache capacity",
731 eventCount: 5,
732 cacheCapacity: 5,
733 lowerBoundCapacity: 5 / 2,
734 upperBoundCapacity: 5 * 2,
735 interval: DefaultEventFreshDuration / 4,
736 expectCapacity: 5,
737 expectStartIndex: 0,
738 },
739 {
740 name: "[capacity not equals 4*n] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking",
741 eventCount: 5,
742 cacheCapacity: 5,
743 lowerBoundCapacity: 5 / 2,
744 upperBoundCapacity: 5 * 2,
745 interval: DefaultEventFreshDuration + time.Second,
746 expectCapacity: 2,
747 expectStartIndex: 3,
748 },
749 {
750 name: "[capacity not equals 4*n] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking with given lowerBoundCapacity",
751 eventCount: 5,
752 cacheCapacity: 5,
753 lowerBoundCapacity: 3,
754 upperBoundCapacity: 5 * 2,
755 interval: DefaultEventFreshDuration + time.Second,
756 expectCapacity: 3,
757 expectStartIndex: 2,
758 },
759 {
760 name: "[capacity not equals 4*n] events inside DefaultEventFreshDuration cause cache expanding with given upperBoundCapacity",
761 eventCount: 5,
762 cacheCapacity: 5,
763 lowerBoundCapacity: 5 / 2,
764 upperBoundCapacity: 8,
765 interval: DefaultEventFreshDuration / 6,
766 expectCapacity: 8,
767 expectStartIndex: 0,
768 },
769 {
770 name: "[capacity not equals 4*n] [startIndex not equal 0] events inside DefaultEventFreshDuration cause cache expanding",
771 eventCount: 5,
772 cacheCapacity: 5,
773 startIndex: 3,
774 lowerBoundCapacity: 5 / 2,
775 upperBoundCapacity: 5 * 2,
776 interval: DefaultEventFreshDuration / 6,
777 expectCapacity: 10,
778 expectStartIndex: 3,
779 },
780 {
781 name: "[capacity not equals 4*n] [startIndex not equal 0] events outside DefaultEventFreshDuration without change cache capacity",
782 eventCount: 5,
783 cacheCapacity: 5,
784 startIndex: 3,
785 lowerBoundCapacity: 5 / 2,
786 upperBoundCapacity: 5 * 2,
787 interval: DefaultEventFreshDuration / 4,
788 expectCapacity: 5,
789 expectStartIndex: 3,
790 },
791 {
792 name: "[capacity not equals 4*n] [startIndex not equal 0] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking",
793 eventCount: 5,
794 cacheCapacity: 5,
795 startIndex: 3,
796 lowerBoundCapacity: 5 / 2,
797 upperBoundCapacity: 5 * 2,
798 interval: DefaultEventFreshDuration + time.Second,
799 expectCapacity: 2,
800 expectStartIndex: 6,
801 },
802 {
803 name: "[capacity not equals 4*n] [startIndex not equal 0] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking with given lowerBoundCapacity",
804 eventCount: 5,
805 cacheCapacity: 5,
806 startIndex: 3,
807 lowerBoundCapacity: 3,
808 upperBoundCapacity: 5 * 2,
809 interval: DefaultEventFreshDuration + time.Second,
810 expectCapacity: 3,
811 expectStartIndex: 5,
812 },
813 {
814 name: "[capacity not equals 4*n] [startIndex not equal 0] events inside DefaultEventFreshDuration cause cache expanding with given upperBoundCapacity",
815 eventCount: 5,
816 cacheCapacity: 5,
817 startIndex: 3,
818 lowerBoundCapacity: 5 / 2,
819 upperBoundCapacity: 8,
820 interval: DefaultEventFreshDuration / 6,
821 expectCapacity: 8,
822 expectStartIndex: 3,
823 },
824 {
825 name: "[capacity equals 4*n] events inside DefaultEventFreshDuration cause cache expanding",
826 eventCount: 8,
827 cacheCapacity: 8,
828 lowerBoundCapacity: 8 / 2,
829 upperBoundCapacity: 8 * 2,
830 interval: DefaultEventFreshDuration / 9,
831 expectCapacity: 16,
832 expectStartIndex: 0,
833 },
834 {
835 name: "[capacity equals 4*n] events outside DefaultEventFreshDuration without change cache capacity",
836 eventCount: 8,
837 cacheCapacity: 8,
838 lowerBoundCapacity: 8 / 2,
839 upperBoundCapacity: 8 * 2,
840 interval: DefaultEventFreshDuration / 8,
841 expectCapacity: 8,
842 expectStartIndex: 0,
843 },
844 {
845 name: "[capacity equals 4*n] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking",
846 eventCount: 8,
847 cacheCapacity: 8,
848 lowerBoundCapacity: 8 / 2,
849 upperBoundCapacity: 8 * 2,
850 interval: DefaultEventFreshDuration/2 + time.Second,
851 expectCapacity: 4,
852 expectStartIndex: 4,
853 },
854 {
855 name: "[capacity equals 4*n] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking with given lowerBoundCapacity",
856 eventCount: 8,
857 cacheCapacity: 8,
858 lowerBoundCapacity: 7,
859 upperBoundCapacity: 8 * 2,
860 interval: DefaultEventFreshDuration/2 + time.Second,
861 expectCapacity: 7,
862 expectStartIndex: 1,
863 },
864 {
865 name: "[capacity equals 4*n] events inside DefaultEventFreshDuration cause cache expanding with given upperBoundCapacity",
866 eventCount: 8,
867 cacheCapacity: 8,
868 lowerBoundCapacity: 8 / 2,
869 upperBoundCapacity: 10,
870 interval: DefaultEventFreshDuration / 9,
871 expectCapacity: 10,
872 expectStartIndex: 0,
873 },
874 {
875 name: "[capacity equals 4*n] [startIndex not equal 0] events inside DefaultEventFreshDuration cause cache expanding",
876 eventCount: 8,
877 cacheCapacity: 8,
878 startIndex: 3,
879 lowerBoundCapacity: 8 / 2,
880 upperBoundCapacity: 8 * 2,
881 interval: DefaultEventFreshDuration / 9,
882 expectCapacity: 16,
883 expectStartIndex: 3,
884 },
885 {
886 name: "[capacity equals 4*n] [startIndex not equal 0] events outside DefaultEventFreshDuration without change cache capacity",
887 eventCount: 8,
888 cacheCapacity: 8,
889 startIndex: 3,
890 lowerBoundCapacity: 8 / 2,
891 upperBoundCapacity: 8 * 2,
892 interval: DefaultEventFreshDuration / 8,
893 expectCapacity: 8,
894 expectStartIndex: 3,
895 },
896 {
897 name: "[capacity equals 4*n] [startIndex not equal 0] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking",
898 eventCount: 8,
899 cacheCapacity: 8,
900 startIndex: 3,
901 lowerBoundCapacity: 8 / 2,
902 upperBoundCapacity: 8 * 2,
903 interval: DefaultEventFreshDuration/2 + time.Second,
904 expectCapacity: 4,
905 expectStartIndex: 7,
906 },
907 {
908 name: "[capacity equals 4*n] [startIndex not equal 0] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking with given lowerBoundCapacity",
909 eventCount: 8,
910 cacheCapacity: 8,
911 startIndex: 3,
912 lowerBoundCapacity: 7,
913 upperBoundCapacity: 8 * 2,
914 interval: DefaultEventFreshDuration/2 + time.Second,
915 expectCapacity: 7,
916 expectStartIndex: 4,
917 },
918 {
919 name: "[capacity equals 4*n] [startIndex not equal 0] events inside DefaultEventFreshDuration cause cache expanding with given upperBoundCapacity",
920 eventCount: 8,
921 cacheCapacity: 8,
922 startIndex: 3,
923 lowerBoundCapacity: 8 / 2,
924 upperBoundCapacity: 10,
925 interval: DefaultEventFreshDuration / 9,
926 expectCapacity: 10,
927 expectStartIndex: 3,
928 },
929 }
930
931 for _, test := range tests {
932 t.Run(test.name, func(t *testing.T) {
933 store := newTestWatchCache(test.cacheCapacity, DefaultEventFreshDuration, &cache.Indexers{})
934 defer store.Stop()
935 store.cache = make([]*watchCacheEvent, test.cacheCapacity)
936 store.startIndex = test.startIndex
937 store.lowerBoundCapacity = test.lowerBoundCapacity
938 store.upperBoundCapacity = test.upperBoundCapacity
939 loadEventWithDuration(store, test.eventCount, test.interval)
940 nextInterval := store.clock.Now().Add(time.Duration(test.interval.Nanoseconds() * int64(test.eventCount)))
941 store.resizeCacheLocked(nextInterval)
942 if store.capacity != test.expectCapacity {
943 t.Errorf("expect capacity %d, but get %d", test.expectCapacity, store.capacity)
944 }
945
946 // check cache's startIndex, endIndex and all elements.
947 if store.startIndex != test.expectStartIndex {
948 t.Errorf("expect startIndex %d, but get %d", test.expectStartIndex, store.startIndex)
949 }
950 if store.endIndex != test.startIndex+test.eventCount {
951 t.Errorf("expect endIndex %d get %d", test.startIndex+test.eventCount, store.endIndex)
952 }
953 if !checkCacheElements(store) {
954 t.Errorf("some elements locations in cache is wrong")
955 }
956 })
957 }
958 }
959
960 func loadEventWithDuration(cache *testWatchCache, count int, interval time.Duration) {
961 for i := 0; i < count; i++ {
962 event := &watchCacheEvent{
963 Key: fmt.Sprintf("event-%d", i+cache.startIndex),
964 RecordTime: cache.clock.Now().Add(time.Duration(interval.Nanoseconds() * int64(i))),
965 }
966 cache.cache[(i+cache.startIndex)%cache.capacity] = event
967 }
968 cache.endIndex = cache.startIndex + count
969 }
970
971 func checkCacheElements(cache *testWatchCache) bool {
972 for i := cache.startIndex; i < cache.endIndex; i++ {
973 location := i % cache.capacity
974 if cache.cache[location].Key != fmt.Sprintf("event-%d", i) {
975 return false
976 }
977 }
978 return true
979 }
980
981 func TestCacheIncreaseDoesNotBreakWatch(t *testing.T) {
982 store := newTestWatchCache(2, DefaultEventFreshDuration, &cache.Indexers{})
983 defer store.Stop()
984
985 now := store.clock.Now()
986 addEvent := func(key string, rv uint64, t time.Time) {
987 event := &watchCacheEvent{
988 Key: key,
989 ResourceVersion: rv,
990 RecordTime: t,
991 }
992 store.updateCache(event)
993 }
994
995 // Initial LIST comes from the moment of RV=10.
996 store.Replace(nil, "10")
997
998 addEvent("key1", 20, now)
999
1000 // Force "key1" to rotate our of cache.
1001 later := now.Add(2 * DefaultEventFreshDuration)
1002 addEvent("key2", 30, later)
1003 addEvent("key3", 40, later)
1004
1005 // Force cache resize.
1006 addEvent("key4", 50, later.Add(time.Second))
1007
1008 _, err := store.getAllEventsSince(15, storage.ListOptions{Predicate: storage.Everything})
1009 if err == nil || !strings.Contains(err.Error(), "too old resource version") {
1010 t.Errorf("unexpected error: %v", err)
1011 }
1012 }
1013
1014 func TestSuggestedWatchChannelSize(t *testing.T) {
1015 testCases := []struct {
1016 name string
1017 capacity int
1018 indexExists bool
1019 triggerUsed bool
1020 eventsFreshDuration time.Duration
1021 expected int
1022 }{
1023 {
1024 name: "capacity=100, indexExists, triggerUsed",
1025 capacity: 100,
1026 indexExists: true,
1027 triggerUsed: true,
1028 eventsFreshDuration: DefaultEventFreshDuration,
1029 expected: 10,
1030 },
1031 {
1032 name: "capacity=100, indexExists, !triggerUsed",
1033 capacity: 100,
1034 indexExists: true,
1035 triggerUsed: false,
1036 eventsFreshDuration: DefaultEventFreshDuration,
1037 expected: 10,
1038 },
1039 {
1040 name: "capacity=100, !indexExists",
1041 capacity: 100,
1042 indexExists: false,
1043 triggerUsed: false,
1044 eventsFreshDuration: DefaultEventFreshDuration,
1045 expected: 10,
1046 },
1047 {
1048 name: "capacity=750, indexExists, triggerUsed",
1049 capacity: 750,
1050 indexExists: true,
1051 triggerUsed: true,
1052 eventsFreshDuration: DefaultEventFreshDuration,
1053 expected: 10,
1054 },
1055 {
1056 name: "capacity=750, indexExists, !triggerUsed",
1057 capacity: 750,
1058 indexExists: true,
1059 triggerUsed: false,
1060 eventsFreshDuration: DefaultEventFreshDuration,
1061 expected: 10,
1062 },
1063 {
1064 name: "capacity=750, !indexExists",
1065 capacity: 750,
1066 indexExists: false,
1067 triggerUsed: false,
1068 eventsFreshDuration: DefaultEventFreshDuration,
1069 expected: 10,
1070 },
1071 {
1072 name: "capacity=7500, indexExists, triggerUsed",
1073 capacity: 7500,
1074 indexExists: true,
1075 triggerUsed: true,
1076 eventsFreshDuration: DefaultEventFreshDuration,
1077 expected: 10,
1078 },
1079 {
1080 name: "capacity=7500, indexExists, !triggerUsed",
1081 capacity: 7500,
1082 indexExists: true,
1083 triggerUsed: false,
1084 eventsFreshDuration: DefaultEventFreshDuration,
1085 expected: 100,
1086 },
1087 {
1088 name: "capacity=7500, indexExists, !triggerUsed, eventsFreshDuration=2m30s",
1089 capacity: 7500,
1090 indexExists: true,
1091 triggerUsed: false,
1092 eventsFreshDuration: 2 * DefaultEventFreshDuration,
1093 expected: 50,
1094 },
1095 {
1096 name: "capacity=7500, !indexExists",
1097 capacity: 7500,
1098 indexExists: false,
1099 triggerUsed: false,
1100 eventsFreshDuration: DefaultEventFreshDuration,
1101 expected: 100,
1102 },
1103 {
1104 name: "capacity=7500, !indexExists, eventsFreshDuration=2m30s",
1105 capacity: 7500,
1106 indexExists: false,
1107 triggerUsed: false,
1108 eventsFreshDuration: 2 * DefaultEventFreshDuration,
1109 expected: 50,
1110 },
1111 {
1112 name: "capacity=75000, indexExists, triggerUsed",
1113 capacity: 75000,
1114 indexExists: true,
1115 triggerUsed: true,
1116 eventsFreshDuration: DefaultEventFreshDuration,
1117 expected: 10,
1118 },
1119 {
1120 name: "capacity=75000, indexExists, !triggerUsed",
1121 capacity: 75000,
1122 indexExists: true,
1123 triggerUsed: false,
1124 eventsFreshDuration: DefaultEventFreshDuration,
1125 expected: 1000,
1126 },
1127 {
1128 name: "capacity=75000, indexExists, !triggerUsed, eventsFreshDuration=2m30s",
1129 capacity: 75000,
1130 indexExists: true,
1131 triggerUsed: false,
1132 eventsFreshDuration: 2 * DefaultEventFreshDuration,
1133 expected: 500,
1134 },
1135 {
1136 name: "capacity=75000, !indexExists",
1137 capacity: 75000,
1138 indexExists: false,
1139 triggerUsed: false,
1140 eventsFreshDuration: DefaultEventFreshDuration,
1141 expected: 100,
1142 },
1143 {
1144 name: "capacity=750000, indexExists, triggerUsed",
1145 capacity: 750000,
1146 indexExists: true,
1147 triggerUsed: true,
1148 eventsFreshDuration: DefaultEventFreshDuration,
1149 expected: 10,
1150 },
1151 {
1152 name: "capacity=750000, indexExists, !triggerUsed",
1153 capacity: 750000,
1154 indexExists: true,
1155 triggerUsed: false,
1156 eventsFreshDuration: DefaultEventFreshDuration,
1157 expected: 1000,
1158 },
1159 {
1160 name: "capacity=750000, !indexExists",
1161 capacity: 750000,
1162 indexExists: false,
1163 triggerUsed: false,
1164 eventsFreshDuration: DefaultEventFreshDuration,
1165 expected: 100,
1166 },
1167 }
1168
1169 for _, test := range testCases {
1170 t.Run(test.name, func(t *testing.T) {
1171 store := newTestWatchCache(test.capacity, test.eventsFreshDuration, &cache.Indexers{})
1172 defer store.Stop()
1173 got := store.suggestedWatchChannelSize(test.indexExists, test.triggerUsed)
1174 if got != test.expected {
1175 t.Errorf("unexpected channel size got: %v, expected: %v", got, test.expected)
1176 }
1177 })
1178 }
1179 }
1180
1181 func TestCapacityUpperBound(t *testing.T) {
1182 testCases := []struct {
1183 name string
1184 eventFreshDuration time.Duration
1185 expected int
1186 }{
1187 {
1188 name: "default eventFreshDuration",
1189 eventFreshDuration: DefaultEventFreshDuration, // 75s
1190 expected: defaultUpperBoundCapacity, // 100 * 1024
1191 },
1192 {
1193 name: "lower eventFreshDuration, capacity limit unchanged",
1194 eventFreshDuration: 45 * time.Second, // 45s
1195 expected: defaultUpperBoundCapacity, // 100 * 1024
1196 },
1197 {
1198 name: "higher eventFreshDuration, capacity limit scaled up",
1199 eventFreshDuration: 4 * DefaultEventFreshDuration, // 4 * 75s
1200 expected: 4 * defaultUpperBoundCapacity, // 4 * 100 * 1024
1201 },
1202 {
1203 name: "higher eventFreshDuration, capacity limit scaled and rounded up",
1204 eventFreshDuration: 3 * DefaultEventFreshDuration, // 3 * 75s
1205 expected: 4 * defaultUpperBoundCapacity, // 4 * 100 * 1024
1206 },
1207 {
1208 name: "higher eventFreshDuration, capacity limit scaled up and capped",
1209 eventFreshDuration: DefaultEventFreshDuration << 20, // 2^20 * 75s
1210 expected: defaultUpperBoundCapacity << 14, // 2^14 * 100 * 1024
1211 },
1212 }
1213
1214 for _, test := range testCases {
1215 t.Run(test.name, func(t *testing.T) {
1216 capacity := capacityUpperBound(test.eventFreshDuration)
1217 if test.expected != capacity {
1218 t.Errorf("expected %v, got %v", test.expected, capacity)
1219 }
1220 })
1221 }
1222 }
1223
1224 func BenchmarkWatchCache_updateCache(b *testing.B) {
1225 store := newTestWatchCache(defaultUpperBoundCapacity, DefaultEventFreshDuration, &cache.Indexers{})
1226 defer store.Stop()
1227 store.cache = store.cache[:0]
1228 store.upperBoundCapacity = defaultUpperBoundCapacity
1229 loadEventWithDuration(store, defaultUpperBoundCapacity, 0)
1230 add := &watchCacheEvent{
1231 Key: fmt.Sprintf("event-%d", defaultUpperBoundCapacity),
1232 RecordTime: store.clock.Now(),
1233 }
1234 b.ResetTimer()
1235 for i := 0; i < b.N; i++ {
1236 store.updateCache(add)
1237 }
1238 }
1239
1240 func TestHistogramCacheReadWait(t *testing.T) {
1241 registry := k8smetrics.NewKubeRegistry()
1242 if err := registry.Register(metrics.WatchCacheReadWait); err != nil {
1243 t.Errorf("unexpected error: %v", err)
1244 }
1245 ctx := context.Background()
1246 testedMetrics := "apiserver_watch_cache_read_wait_seconds"
1247 store := newTestWatchCache(2, DefaultEventFreshDuration, &cache.Indexers{})
1248 defer store.Stop()
1249
1250 // In background, update the store.
1251 go func() {
1252 if err := store.Add(makeTestPod("foo", 2)); err != nil {
1253 t.Errorf("unexpected error: %v", err)
1254 }
1255 if err := store.Add(makeTestPod("bar", 5)); err != nil {
1256 t.Errorf("unexpected error: %v", err)
1257 }
1258 }()
1259
1260 testCases := []struct {
1261 desc string
1262 resourceVersion uint64
1263 want string
1264 }{
1265 {
1266 desc: "resourceVersion is non-zero",
1267 resourceVersion: 5,
1268 want: `
1269 # HELP apiserver_watch_cache_read_wait_seconds [ALPHA] Histogram of time spent waiting for a watch cache to become fresh.
1270 # TYPE apiserver_watch_cache_read_wait_seconds histogram
1271 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.005"} 1
1272 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.025"} 1
1273 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.05"} 1
1274 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.1"} 1
1275 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.2"} 1
1276 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.4"} 1
1277 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.6"} 1
1278 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.8"} 1
1279 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1"} 1
1280 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.25"} 1
1281 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.5"} 1
1282 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="2"} 1
1283 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="3"} 1
1284 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="+Inf"} 1
1285 apiserver_watch_cache_read_wait_seconds_sum{resource="pods"} 0
1286 apiserver_watch_cache_read_wait_seconds_count{resource="pods"} 1
1287 `,
1288 },
1289 {
1290 desc: "resourceVersion is 0",
1291 resourceVersion: 0,
1292 want: ``,
1293 },
1294 }
1295
1296 for _, test := range testCases {
1297 t.Run(test.desc, func(t *testing.T) {
1298 defer registry.Reset()
1299 if _, _, _, err := store.WaitUntilFreshAndGet(ctx, test.resourceVersion, "prefix/ns/bar"); err != nil {
1300 t.Errorf("unexpected error: %v", err)
1301 }
1302 if err := testutil.GatherAndCompare(registry, strings.NewReader(test.want), testedMetrics); err != nil {
1303 t.Errorf("unexpected error: %v", err)
1304 }
1305 })
1306 }
1307 }
1308
1309 func TestCacheSnapshots(t *testing.T) {
1310 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ListFromCacheSnapshot, true)
1311
1312 store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{})
1313 defer store.Stop()
1314 store.upperBoundCapacity = 3
1315 store.lowerBoundCapacity = 1
1316 clock := store.clock.(*testingclock.FakeClock)
1317
1318 _, found := store.snapshots.GetLessOrEqual(100)
1319 assert.False(t, found, "Expected empty cache to not include any snapshots")
1320
1321 t.Log("Test cache on rev 100")
1322 require.NoError(t, store.Add(makeTestPod("foo", 100)))
1323 require.NoError(t, store.Update(makeTestPod("foo", 200)))
1324 clock.Step(time.Second)
1325 require.NoError(t, store.Delete(makeTestPod("foo", 300)))
1326
1327 t.Log("Test cache on rev 100")
1328 _, found = store.snapshots.GetLessOrEqual(99)
1329 assert.False(t, found, "Expected store to not include rev 99")
1330 lister, found := store.snapshots.GetLessOrEqual(100)
1331 assert.True(t, found, "Expected store to not include rev 100")
1332 elements := lister.ListPrefix("", "")
1333 assert.Len(t, elements, 1)
1334 assert.Equal(t, makeTestPod("foo", 100), elements[0].(*storeElement).Object)
1335
1336 t.Log("Overflow cache to remove rev 100")
1337 require.NoError(t, store.Add(makeTestPod("foo", 400)))
1338 _, found = store.snapshots.GetLessOrEqual(100)
1339 assert.False(t, found, "Expected overfilled cache to delete oldest rev 100")
1340
1341 t.Log("Test cache on rev 200")
1342 lister, found = store.snapshots.GetLessOrEqual(200)
1343 assert.True(t, found, "Expected store to still keep rev 200")
1344 elements = lister.ListPrefix("", "")
1345 assert.Len(t, elements, 1)
1346 assert.Equal(t, makeTestPod("foo", 200), elements[0].(*storeElement).Object)
1347
1348 t.Log("Test cache on rev 300")
1349 lister, found = store.snapshots.GetLessOrEqual(300)
1350 assert.True(t, found, "Expected store to still keep rev 300")
1351 elements = lister.ListPrefix("", "")
1352 assert.Empty(t, elements)
1353
1354 t.Log("Test cache on rev 400")
1355 lister, found = store.snapshots.GetLessOrEqual(400)
1356 assert.True(t, found, "Expected store to still keep rev 400")
1357 elements = lister.ListPrefix("", "")
1358 assert.Len(t, elements, 1)
1359 assert.Equal(t, makeTestPod("foo", 400), elements[0].(*storeElement).Object)
1360
1361 t.Log("Add event outside the event fresh window to force cache capacity downsize")
1362 assert.Equal(t, 3, store.capacity)
1363 clock.Step(DefaultEventFreshDuration + 1)
1364 require.NoError(t, store.Update(makeTestPod("foo", 500)))
1365 assert.Equal(t, 1, store.capacity)
1366 assert.Equal(t, 1, store.snapshots.Len())
1367 _, found = store.snapshots.GetLessOrEqual(499)
1368 assert.False(t, found, "Expected overfilled cache to delete events below 500")
1369
1370 t.Log("Test cache on rev 500")
1371 lister, found = store.snapshots.GetLessOrEqual(500)
1372 assert.True(t, found, "Expected store to still keep rev 500")
1373 elements = lister.ListPrefix("", "")
1374 assert.Len(t, elements, 1)
1375 assert.Equal(t, makeTestPod("foo", 500), elements[0].(*storeElement).Object)
1376
1377 t.Log("Add event to force capacity upsize")
1378 require.NoError(t, store.Update(makeTestPod("foo", 600)))
1379 assert.Equal(t, 2, store.capacity)
1380 assert.Equal(t, 2, store.snapshots.Len())
1381
1382 t.Log("Test cache on rev 600")
1383 lister, found = store.snapshots.GetLessOrEqual(600)
1384 assert.True(t, found, "Expected replace to be snapshotted")
1385 elements = lister.ListPrefix("", "")
1386 assert.Len(t, elements, 1)
1387 assert.Equal(t, makeTestPod("foo", 600), elements[0].(*storeElement).Object)
1388
1389 t.Log("Replace cache to remove history")
1390 _, found = store.snapshots.GetLessOrEqual(500)
1391 assert.True(t, found, "Confirm that cache stores history before replace")
1392 err := store.Replace([]interface{}{makeTestPod("foo", 600)}, "700")
1393 require.NoError(t, err)
1394 _, found = store.snapshots.GetLessOrEqual(500)
1395 assert.False(t, found, "Expected replace to remove history")
1396 _, found = store.snapshots.GetLessOrEqual(600)
1397 assert.False(t, found, "Expected replace to remove history")
1398
1399 t.Log("Test cache on rev 700")
1400 lister, found = store.snapshots.GetLessOrEqual(700)
1401 assert.True(t, found, "Expected replace to be snapshotted")
1402 elements = lister.ListPrefix("", "")
1403 assert.Len(t, elements, 1)
1404 assert.Equal(t, makeTestPod("foo", 600), elements[0].(*storeElement).Object)
1405 }