Debian Codegraph @

pkg/storage/cacher/cacher_whitebox_test.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package cacher
18
19 import (
20 "context"
21 "crypto/rand"
22 "errors"
23 "fmt"
24 "reflect"
25 goruntime "runtime"
26 "strconv"
27 "strings"
28 "sync"
29 "testing"
30 "time"
31
32 "github.com/stretchr/testify/require"
33 "google.golang.org/grpc/metadata"
34 "k8s.io/apimachinery/pkg/apis/meta/internalversion"
35 "k8s.io/apimachinery/pkg/apis/meta/internalversion/validation"
36
37 apiequality "k8s.io/apimachinery/pkg/api/equality"
38 apierrors "k8s.io/apimachinery/pkg/api/errors"
39 "k8s.io/apimachinery/pkg/api/meta"
40 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
41 "k8s.io/apimachinery/pkg/fields"
42 "k8s.io/apimachinery/pkg/labels"
43 "k8s.io/apimachinery/pkg/runtime"
44 "k8s.io/apimachinery/pkg/runtime/schema"
45 "k8s.io/apimachinery/pkg/util/wait"
46 "k8s.io/apimachinery/pkg/watch"
47 "k8s.io/apiserver/pkg/apis/example"
48 examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
49 "k8s.io/apiserver/pkg/features"
50 "k8s.io/apiserver/pkg/storage"
51 "k8s.io/apiserver/pkg/storage/cacher/delegator"
52 "k8s.io/apiserver/pkg/storage/cacher/metrics"
53 etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
54 etcdfeature "k8s.io/apiserver/pkg/storage/feature"
55 storagetesting "k8s.io/apiserver/pkg/storage/testing"
56 utilfeature "k8s.io/apiserver/pkg/util/feature"
57 featuregatetesting "k8s.io/component-base/featuregate/testing"
58 k8smetrics "k8s.io/component-base/metrics"
59 "k8s.io/component-base/metrics/testutil"
60 "k8s.io/utils/clock"
61 testingclock "k8s.io/utils/clock/testing"
62 "k8s.io/utils/pointer"
63 )
64
65 func newTestCacherWithoutSyncing(s storage.Interface, c clock.WithTicker) (*Cacher, storage.Versioner, error) {
66 prefix := "pods"
67 config := Config{
68 Storage: s,
69 Versioner: storage.APIObjectVersioner{},
70 GroupResource: schema.GroupResource{Resource: "pods"},
71 EventsHistoryWindow: DefaultEventFreshDuration,
72 ResourcePrefix: prefix,
73 KeyFunc: func(obj runtime.Object) (string, error) { return storage.NamespaceKeyFunc(prefix, obj) },
74 GetAttrsFunc: func(obj runtime.Object) (labels.Set, fields.Set, error) {
75 pod, ok := obj.(*example.Pod)
76 if !ok {
77 return storage.DefaultNamespaceScopedAttr(obj)
78 }
79 labelsSet, fieldsSet, err := storage.DefaultNamespaceScopedAttr(obj)
80 if err != nil {
81 return nil, nil, err
82 }
83 fieldsSet["spec.nodeName"] = pod.Spec.NodeName
84 return labelsSet, fieldsSet, nil
85 },
86 NewFunc: func() runtime.Object { return &example.Pod{} },
87 NewListFunc: func() runtime.Object { return &example.PodList{} },
88 Codec: codecs.LegacyCodec(examplev1.SchemeGroupVersion),
89 Clock: c,
90 }
91 cacher, err := NewCacherFromConfig(config)
92
93 return cacher, storage.APIObjectVersioner{}, err
94 }
95
96 func newTestCacher(s storage.Interface) (*Cacher, storage.Versioner, error) {
97 cacher, versioner, err := newTestCacherWithoutSyncing(s, clock.RealClock{})
98 if err != nil {
99 return nil, versioner, err
100 }
101
102 if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
103 // The tests assume that Get/GetList/Watch calls shouldn't fail.
104 // However, 429 error can now be returned if watchcache is under initialization.
105 // To avoid rewriting all tests, we wait for watcache to initialize.
106 if err := cacher.Wait(context.Background()); err != nil {
107 return nil, storage.APIObjectVersioner{}, err
108 }
109 }
110 return cacher, versioner, nil
111 }
112
113 type dummyStorage struct {
114 sync.RWMutex
115 err error
116 getListFn func(_ context.Context, _ string, _ storage.ListOptions, listObj runtime.Object) error
117 getRVFn func(_ context.Context) (uint64, error)
118 watchFn func(_ context.Context, _ string, _ storage.ListOptions) (watch.Interface, error)
119
120 // use getRequestWatchProgressCounter when reading
121 // the value of the counter
122 requestWatchProgressCounter int
123 }
124
125 func (d *dummyStorage) RequestWatchProgress(ctx context.Context) error {
126 d.Lock()
127 defer d.Unlock()
128 d.requestWatchProgressCounter++
129 return nil
130 }
131
132 func (d *dummyStorage) getRequestWatchProgressCounter() int {
133 d.RLock()
134 defer d.RUnlock()
135 return d.requestWatchProgressCounter
136 }
137
138 type dummyWatch struct {
139 ch chan watch.Event
140 }
141
142 func (w *dummyWatch) ResultChan() <-chan watch.Event {
143 return w.ch
144 }
145
146 func (w *dummyWatch) Stop() {
147 close(w.ch)
148 }
149
150 func newDummyWatch() watch.Interface {
151 return &dummyWatch{
152 ch: make(chan watch.Event),
153 }
154 }
155
156 func (d *dummyStorage) Versioner() storage.Versioner { return nil }
157 func (d *dummyStorage) Create(_ context.Context, _ string, _, _ runtime.Object, _ uint64) error {
158 return fmt.Errorf("unimplemented")
159 }
160 func (d *dummyStorage) Delete(_ context.Context, _ string, _ runtime.Object, _ *storage.Preconditions, _ storage.ValidateObjectFunc, _ runtime.Object, _ storage.DeleteOptions) error {
161 return fmt.Errorf("unimplemented")
162 }
163 func (d *dummyStorage) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
164 if d.watchFn != nil {
165 return d.watchFn(ctx, key, opts)
166 }
167 d.RLock()
168 defer d.RUnlock()
169
170 return newDummyWatch(), d.err
171 }
172 func (d *dummyStorage) Get(_ context.Context, _ string, _ storage.GetOptions, _ runtime.Object) error {
173 d.RLock()
174 defer d.RUnlock()
175
176 return d.err
177 }
178 func (d *dummyStorage) GetList(ctx context.Context, resPrefix string, opts storage.ListOptions, listObj runtime.Object) error {
179 if d.getListFn != nil {
180 return d.getListFn(ctx, resPrefix, opts, listObj)
181 }
182 d.RLock()
183 defer d.RUnlock()
184 podList := listObj.(*example.PodList)
185 podList.ListMeta = metav1.ListMeta{ResourceVersion: "100"}
186 return d.err
187 }
188 func (d *dummyStorage) GuaranteedUpdate(_ context.Context, _ string, _ runtime.Object, _ bool, _ *storage.Preconditions, _ storage.UpdateFunc, _ runtime.Object) error {
189 return fmt.Errorf("unimplemented")
190 }
191 func (d *dummyStorage) Count(_ string) (int64, error) {
192 return 0, fmt.Errorf("unimplemented")
193 }
194 func (d *dummyStorage) ReadinessCheck() error {
195 return nil
196 }
197 func (d *dummyStorage) injectError(err error) {
198 d.Lock()
199 defer d.Unlock()
200
201 d.err = err
202 }
203
204 func (d *dummyStorage) GetCurrentResourceVersion(ctx context.Context) (uint64, error) {
205 if d.getRVFn != nil {
206 return d.getRVFn(ctx)
207 }
208 return 100, nil
209 }
210
211 type dummyCacher struct {
212 dummyStorage
213 ready bool
214 }
215
216 func (d *dummyCacher) Ready() bool {
217 return d.ready
218 }
219
220 func TestShouldDelegateList(t *testing.T) {
221 type opts struct {
222 Recursive bool
223 ResourceVersion string
224 ResourceVersionMatch metav1.ResourceVersionMatch
225 Limit int64
226 Continue string
227 }
228 toInternalOpts := func(opt opts) *internalversion.ListOptions {
229 return &internalversion.ListOptions{
230 ResourceVersion: opt.ResourceVersion,
231 ResourceVersionMatch: opt.ResourceVersionMatch,
232 Limit: opt.Limit,
233 Continue: opt.Continue,
234 }
235 }
236
237 toStorageOpts := func(opt opts) storage.ListOptions {
238 return storage.ListOptions{
239 Recursive: opt.Recursive,
240 ResourceVersion: opt.ResourceVersion,
241 ResourceVersionMatch: opt.ResourceVersionMatch,
242 Predicate: storage.SelectionPredicate{
243 Continue: opt.Continue,
244 Limit: opt.Limit,
245 },
246 }
247 }
248 oldRV := "80"
249 cacheRV := "100"
250 etcdRV := "120"
251
252 keyPrefix := "/pods/"
253 continueOnOldRV, err := storage.EncodeContinue(keyPrefix+"foo", keyPrefix, int64(mustAtoi(oldRV)))
254 if err != nil {
255 t.Fatalf("Unexpected error: %v", err)
256 }
257 continueOnCacheRV, err := storage.EncodeContinue(keyPrefix+"foo", keyPrefix, int64(mustAtoi(cacheRV)))
258 if err != nil {
259 t.Fatalf("Unexpected error: %v", err)
260 }
261 // Continue from different apiserver that is forward in RVs
262 continueOnEtcdRV, err := storage.EncodeContinue(keyPrefix+"foo", keyPrefix, int64(mustAtoi(etcdRV)))
263 if err != nil {
264 t.Fatalf("Unexpected error: %v", err)
265 }
266
267 continueOnNegativeRV, err := storage.EncodeContinue(keyPrefix+"foo", keyPrefix, -1)
268 if err != nil {
269 t.Fatalf("Unexpected error: %v", err)
270 }
271 testCases := map[opts]bool{}
272 testCases[opts{}] = true
273 testCases[opts{Limit: 100}] = true
274 testCases[opts{ResourceVersion: "0"}] = false
275 testCases[opts{ResourceVersion: "0", Limit: 100}] = false
276 testCases[opts{ResourceVersion: "0", ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan}] = false
277 testCases[opts{ResourceVersion: "0", ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan, Limit: 100}] = false
278 // Continue
279 for _, continueToken := range []string{continueOnOldRV, continueOnCacheRV, continueOnEtcdRV, continueOnNegativeRV} {
280 testCases[opts{Continue: continueToken}] = true
281 testCases[opts{Limit: 100, Continue: continueToken}] = true
282 testCases[opts{ResourceVersion: "0", Continue: continueToken}] = true
283 testCases[opts{ResourceVersion: "0", Limit: 100, Continue: continueToken}] = true
284 }
285 // With RV
286 for _, rv := range []string{oldRV, cacheRV, etcdRV} {
287 testCases[opts{ResourceVersion: rv}] = false
288 testCases[opts{ResourceVersion: rv, Limit: 100}] = true
289 testCases[opts{ResourceVersion: rv, ResourceVersionMatch: metav1.ResourceVersionMatchExact}] = true
290 testCases[opts{ResourceVersion: rv, ResourceVersionMatch: metav1.ResourceVersionMatchExact, Limit: 100}] = true
291 testCases[opts{ResourceVersion: rv, ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan}] = false
292 testCases[opts{ResourceVersion: rv, ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan, Limit: 100}] = false
293 }
294
295 // Bypass for most requests doesn't depend on Recursive
296 for opts, expectBypass := range testCases {
297 opts.Recursive = true
298 testCases[opts] = expectBypass
299 }
300 // Continue is ignored on non recursive LIST
301 for _, rv := range []string{oldRV, etcdRV} {
302 for _, continueToken := range []string{continueOnOldRV, continueOnCacheRV, continueOnEtcdRV, continueOnNegativeRV} {
303 testCases[opts{ResourceVersion: rv, Continue: continueToken}] = true
304 testCases[opts{ResourceVersion: rv, Continue: continueToken, Limit: 100}] = true
305 }
306 }
307
308 for _, rv := range []string{"", "0", oldRV, etcdRV} {
309 for _, match := range []metav1.ResourceVersionMatch{"", metav1.ResourceVersionMatchExact, metav1.ResourceVersionMatchNotOlderThan} {
310 for _, continueKey := range []string{"", continueOnOldRV, continueOnCacheRV, continueOnEtcdRV, continueOnNegativeRV} {
311 for _, limit := range []int64{0, 100} {
312 for _, recursive := range []bool{true, false} {
313 opt := opts{
314 Recursive: recursive,
315 ResourceVersion: rv,
316 ResourceVersionMatch: match,
317 Limit: limit,
318 Continue: continueKey,
319 }
320 if errs := validation.ValidateListOptions(toInternalOpts(opt), false); len(errs) != 0 {
321 continue
322 }
323 if _, _, err = storage.ValidateListOptions(keyPrefix, storage.APIObjectVersioner{}, toStorageOpts(opt)); err != nil {
324 continue
325 }
326 _, found := testCases[opt]
327 if !found {
328 t.Errorf("Test case not covered, but passes validation: %+v", opt)
329 }
330 }
331
332 }
333 }
334 }
335 }
336
337 for opt := range testCases {
338 if errs := validation.ValidateListOptions(toInternalOpts(opt), false); len(errs) != 0 {
339 t.Errorf("Invalid LIST request that should not be tested %+v", opt)
340 continue
341 }
342 if _, _, err = storage.ValidateListOptions(keyPrefix, storage.APIObjectVersioner{}, toStorageOpts(opt)); err != nil {
343 t.Errorf("Invalid LIST request that should not be tested %+v", opt)
344 continue
345 }
346 }
347
348 runTestCases := func(t *testing.T, snapshotAvailable bool, testcases map[opts]bool, overrides ...map[opts]bool) {
349 for opt, expectBypass := range testCases {
350 for _, override := range overrides {
351 if bypass, ok := override[opt]; ok {
352 expectBypass = bypass
353 }
354 }
355 backingStorage := &dummyStorage{}
356 backingStorage.getListFn = func(_ context.Context, _ string, _ storage.ListOptions, listObj runtime.Object) error {
357 podList := listObj.(*example.PodList)
358 podList.ListMeta = metav1.ListMeta{ResourceVersion: cacheRV}
359 return nil
360 }
361 cacher, _, err := newTestCacher(backingStorage)
362 if err != nil {
363 t.Fatalf("Couldn't create cacher: %v", err)
364 }
365 defer cacher.Stop()
366 if snapshotAvailable {
367 cacher.watchCache.snapshots.Add(uint64(mustAtoi(oldRV)), fakeOrderedLister{})
368 }
369 result, err := delegator.ShouldDelegateList(toStorageOpts(opt), cacher)
370 if err != nil {
371 t.Fatal(err)
372 }
373 if result.ShouldDelegate != expectBypass {
374 t.Errorf("Unexpected bypass result for List request with options %+v, bypass expected: %v, got: %v", opt, expectBypass, result.ShouldDelegate)
375 }
376 }
377 }
378
379 // Exacts and continue on current cache RV.
380 listFromSnapshotEnabledOverrides := map[opts]bool{}
381 listFromSnapshotEnabledOverrides[opts{Recursive: true, ResourceVersion: cacheRV, Limit: 100}] = false
382 listFromSnapshotEnabledOverrides[opts{Recursive: true, ResourceVersion: cacheRV, ResourceVersionMatch: metav1.ResourceVersionMatchExact}] = false
383 listFromSnapshotEnabledOverrides[opts{Recursive: true, ResourceVersion: cacheRV, Limit: 100, ResourceVersionMatch: metav1.ResourceVersionMatchExact}] = false
384 listFromSnapshotEnabledOverrides[opts{Recursive: true, ResourceVersion: "0", Limit: 100, Continue: continueOnCacheRV}] = false
385 listFromSnapshotEnabledOverrides[opts{Recursive: true, ResourceVersion: "0", Continue: continueOnCacheRV}] = false
386 listFromSnapshotEnabledOverrides[opts{Recursive: true, Limit: 100, Continue: continueOnCacheRV}] = false
387 listFromSnapshotEnabledOverrides[opts{Recursive: true, Continue: continueOnCacheRV}] = false
388
389 // Exacts and continue RV with a snapshot.
390 snapshotAvailableOverrides := map[opts]bool{}
391 snapshotAvailableOverrides[opts{Recursive: true, Continue: continueOnOldRV}] = false
392 snapshotAvailableOverrides[opts{Recursive: true, Limit: 100, Continue: continueOnOldRV}] = false
393 snapshotAvailableOverrides[opts{Recursive: true, ResourceVersion: "0", Continue: continueOnOldRV}] = false
394 snapshotAvailableOverrides[opts{Recursive: true, ResourceVersion: "0", Limit: 100, Continue: continueOnOldRV}] = false
395 snapshotAvailableOverrides[opts{Recursive: true, ResourceVersion: oldRV, Limit: 100}] = false
396 snapshotAvailableOverrides[opts{Recursive: true, ResourceVersion: oldRV, ResourceVersionMatch: metav1.ResourceVersionMatchExact}] = false
397 snapshotAvailableOverrides[opts{Recursive: true, ResourceVersion: oldRV, ResourceVersionMatch: metav1.ResourceVersionMatchExact, Limit: 100}] = false
398
399 t.Run("ConsistentListFromCache=false", func(t *testing.T) {
400 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, false)
401 t.Run("ListFromCacheSnapshot=false", func(t *testing.T) {
402 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ListFromCacheSnapshot, false)
403 runTestCases(t, false, testCases)
404 })
405
406 t.Run("ListFromCacheSnapshot=true", func(t *testing.T) {
407 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ListFromCacheSnapshot, true)
408 t.Run("SnapshotAvailable=false", func(t *testing.T) {
409 runTestCases(t, false, testCases, listFromSnapshotEnabledOverrides)
410 })
411 t.Run("SnapshotAvailable=true", func(t *testing.T) {
412 runTestCases(t, true, testCases, listFromSnapshotEnabledOverrides, snapshotAvailableOverrides)
413 })
414 })
415 })
416 t.Run("ConsistentListFromCache=true", func(t *testing.T) {
417 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true)
418 // TODO(p0lyn0mial): the following tests assume that etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress)
419 // evaluates to true. Otherwise the cache will be bypassed and the test will fail.
420 //
421 // If you were to run only TestGetListCacheBypass you would see that the test fail.
422 // However in CI all test are run and there must be a test(s) that properly
423 // initialize the storage layer so that the mentioned method evaluates to true
424 forceRequestWatchProgressSupport(t)
425
426 consistentListFromCacheOverrides := map[opts]bool{}
427 for _, recursive := range []bool{true, false} {
428 consistentListFromCacheOverrides[opts{Recursive: recursive}] = false
429 consistentListFromCacheOverrides[opts{Limit: 100, Recursive: recursive}] = false
430 }
431
432 t.Run("ListFromCacheSnapshot=false", func(t *testing.T) {
433 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ListFromCacheSnapshot, false)
434 runTestCases(t, false, testCases, consistentListFromCacheOverrides)
435 })
436 t.Run("ListFromCacheSnapshot=true", func(t *testing.T) {
437 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ListFromCacheSnapshot, true)
438
439 consistentReadWithSnapshotOverrides := map[opts]bool{}
440 // Continues with negative RV are same as consistent read.
441 consistentReadWithSnapshotOverrides[opts{Recursive: true, Limit: 0, Continue: continueOnNegativeRV}] = false
442 consistentReadWithSnapshotOverrides[opts{Recursive: true, Limit: 100, Continue: continueOnNegativeRV}] = false
443 consistentReadWithSnapshotOverrides[opts{Recursive: true, ResourceVersion: "0", Limit: 0, Continue: continueOnNegativeRV}] = false
444 consistentReadWithSnapshotOverrides[opts{Recursive: true, ResourceVersion: "0", Limit: 100, Continue: continueOnNegativeRV}] = false
445 // Exact on RV not yet observed by cache
446 consistentReadWithSnapshotOverrides[opts{Recursive: true, ResourceVersion: etcdRV, Limit: 100}] = false
447 consistentReadWithSnapshotOverrides[opts{Recursive: true, ResourceVersion: etcdRV, ResourceVersionMatch: metav1.ResourceVersionMatchExact}] = false
448 consistentReadWithSnapshotOverrides[opts{Recursive: true, ResourceVersion: etcdRV, ResourceVersionMatch: metav1.ResourceVersionMatchExact, Limit: 100}] = false
449 consistentReadWithSnapshotOverrides[opts{Recursive: true, Continue: continueOnEtcdRV}] = false
450 consistentReadWithSnapshotOverrides[opts{Recursive: true, ResourceVersion: "0", Continue: continueOnEtcdRV}] = false
451 consistentReadWithSnapshotOverrides[opts{Recursive: true, Continue: continueOnEtcdRV, Limit: 100}] = false
452 consistentReadWithSnapshotOverrides[opts{Recursive: true, ResourceVersion: "0", Continue: continueOnEtcdRV, Limit: 100}] = false
453 t.Run("SnapshotAvailable=false", func(t *testing.T) {
454 runTestCases(t, false, testCases, listFromSnapshotEnabledOverrides, consistentListFromCacheOverrides, consistentReadWithSnapshotOverrides)
455 })
456 t.Run("SnapshotAvailable=true", func(t *testing.T) {
457 runTestCases(t, true, testCases, listFromSnapshotEnabledOverrides, consistentListFromCacheOverrides, consistentReadWithSnapshotOverrides, snapshotAvailableOverrides)
458 })
459 })
460 })
461 }
462
463 func mustAtoi(s string) int {
464 value, err := strconv.Atoi(s)
465 if err != nil {
466 panic(err)
467 }
468 return value
469 }
470
471 func TestConsistentReadFallback(t *testing.T) {
472 tcs := []struct {
473 name string
474 consistentReadsEnabled bool
475 watchCacheRV string
476 storageRV string
477 fallbackError bool
478
479 expectError bool
480 expectRV string
481 expectBlock bool
482 expectRequestsToStorage int
483 expectMetric string
484 }{
485 {
486 name: "Success",
487 consistentReadsEnabled: true,
488 watchCacheRV: "42",
489 storageRV: "42",
490 expectRV: "42",
491 expectRequestsToStorage: 1,
492 expectMetric: `
493 # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache.
494 # TYPE apiserver_watch_cache_consistent_read_total counter
495 apiserver_watch_cache_consistent_read_total{fallback="false", resource="pods", success="true"} 1
496 `,
497 },
498 {
499 name: "Fallback",
500 consistentReadsEnabled: true,
501 watchCacheRV: "2",
502 storageRV: "42",
503 expectRV: "42",
504 expectBlock: true,
505 expectRequestsToStorage: 2,
506 expectMetric: `
507 # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache.
508 # TYPE apiserver_watch_cache_consistent_read_total counter
509 apiserver_watch_cache_consistent_read_total{fallback="true", resource="pods", success="true"} 1
510 `,
511 },
512 {
513 name: "Fallback Failure",
514 consistentReadsEnabled: true,
515 watchCacheRV: "2",
516 storageRV: "42",
517 fallbackError: true,
518 expectError: true,
519 expectBlock: true,
520 expectRequestsToStorage: 2,
521 expectMetric: `
522 # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache.
523 # TYPE apiserver_watch_cache_consistent_read_total counter
524 apiserver_watch_cache_consistent_read_total{fallback="true", resource="pods", success="false"} 1
525 `,
526 },
527 {
528 name: "Disabled",
529 watchCacheRV: "2",
530 storageRV: "42",
531 expectRV: "42",
532 expectRequestsToStorage: 1,
533 expectMetric: ``,
534 },
535 }
536 for _, tc := range tcs {
537 t.Run(tc.name, func(t *testing.T) {
538 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, tc.consistentReadsEnabled)
539 if tc.consistentReadsEnabled {
540 forceRequestWatchProgressSupport(t)
541 }
542
543 registry := k8smetrics.NewKubeRegistry()
544 metrics.ConsistentReadTotal.Reset()
545 if err := registry.Register(metrics.ConsistentReadTotal); err != nil {
546 t.Errorf("unexpected error: %v", err)
547 }
548 backingStorage := &dummyStorage{}
549 backingStorage.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
550 podList := listObj.(*example.PodList)
551 podList.ResourceVersion = tc.watchCacheRV
552 return nil
553 }
554 c := testingclock.NewFakeClock(time.Now())
555 cacher, _, err := newTestCacherWithoutSyncing(backingStorage, c)
556 if err != nil {
557 t.Fatalf("Couldn't create cacher: %v", err)
558 }
559 defer cacher.Stop()
560 delegator := NewCacheDelegator(cacher, backingStorage)
561 defer delegator.Stop()
562 if err := cacher.ready.wait(context.Background()); err != nil {
563 t.Fatalf("unexpected error waiting for the cache to be ready")
564 }
565
566 if fmt.Sprintf("%d", cacher.watchCache.resourceVersion) != tc.watchCacheRV {
567 t.Fatalf("Expected watch cache RV to equal watchCacheRV, got: %d, want: %s", cacher.watchCache.resourceVersion, tc.watchCacheRV)
568 }
569 requestToStorageCount := 0
570 backingStorage.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
571 requestToStorageCount += 1
572 podList := listObj.(*example.PodList)
573 if key == cacher.resourcePrefix {
574 podList.ResourceVersion = tc.storageRV
575 return nil
576 }
577 if tc.fallbackError {
578 return errDummy
579 }
580 podList.ResourceVersion = tc.storageRV
581 return nil
582 }
583 backingStorage.getRVFn = func(_ context.Context) (uint64, error) {
584 requestToStorageCount += 1
585 rv, err := strconv.Atoi(tc.storageRV)
586 if err != nil {
587 t.Fatalf("failed to parse RV: %s", err)
588 }
589 return uint64(rv), nil
590 }
591 result := &example.PodList{}
592
593 ctx, clockStepCancelFn := context.WithTimeout(context.TODO(), time.Minute)
594 defer clockStepCancelFn()
595 if tc.expectBlock {
596 go func(ctx context.Context) {
597 for {
598 select {
599 case <-ctx.Done():
600 return
601 default:
602 if c.HasWaiters() {
603 c.Step(blockTimeout)
604 }
605 time.Sleep(time.Millisecond)
606 }
607 }
608 }(ctx)
609 }
610
611 start := cacher.clock.Now()
612 err = delegator.GetList(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: ""}, result)
613 clockStepCancelFn()
614 duration := cacher.clock.Since(start)
615 if (err != nil) != tc.expectError {
616 t.Fatalf("Unexpected error err: %v", err)
617 }
618 if result.ResourceVersion != tc.expectRV {
619 t.Fatalf("Unexpected List response RV, got: %q, want: %q", result.ResourceVersion, tc.expectRV)
620 }
621 if requestToStorageCount != tc.expectRequestsToStorage {
622 t.Fatalf("Unexpected number of requests to storage, got: %d, want: %d", requestToStorageCount, tc.expectRequestsToStorage)
623 }
624 blocked := duration >= blockTimeout
625 if blocked != tc.expectBlock {
626 t.Fatalf("Unexpected block, got: %v, want: %v", blocked, tc.expectBlock)
627 }
628
629 if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectMetric), "apiserver_watch_cache_consistent_read_total"); err != nil {
630 t.Errorf("unexpected error: %v", err)
631 }
632 })
633 }
634 }
635
636 func TestMatchExactResourceVersionFallback(t *testing.T) {
637 tcs := []struct {
638 name string
639 snapshotsAvailable []bool
640
641 expectStoreRequests int
642 expectSnapshotRequests int
643 }{
644 {
645 name: "Disabled",
646 snapshotsAvailable: []bool{false, false},
647 expectStoreRequests: 2,
648 expectSnapshotRequests: 1,
649 },
650 {
651 name: "Enabled",
652 snapshotsAvailable: []bool{true, true},
653 expectStoreRequests: 1,
654 expectSnapshotRequests: 2,
655 },
656 {
657 name: "Fallback",
658 snapshotsAvailable: []bool{true, false},
659 expectSnapshotRequests: 2,
660 expectStoreRequests: 2,
661 },
662 }
663 for _, tc := range tcs {
664 t.Run(tc.name, func(t *testing.T) {
665 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ListFromCacheSnapshot, true)
666 backingStorage := &dummyStorage{}
667 expectStoreRequests := 0
668 backingStorage.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
669 expectStoreRequests++
670 podList := listObj.(*example.PodList)
671 switch opts.ResourceVersionMatch {
672 case "":
673 podList.ResourceVersion = "42"
674 case metav1.ResourceVersionMatchExact:
675 podList.ResourceVersion = opts.ResourceVersion
676 }
677 return nil
678 }
679 cacher, _, err := newTestCacherWithoutSyncing(backingStorage, clock.RealClock{})
680 if err != nil {
681 t.Fatalf("Couldn't create cacher: %v", err)
682 }
683 defer cacher.Stop()
684 snapshotRequestCount := 0
685 cacher.watchCache.RWMutex.Lock()
686 cacher.watchCache.snapshots = &fakeSnapshotter{
687 getLessOrEqual: func(rv uint64) (orderedLister, bool) {
688 snapshotAvailable := tc.snapshotsAvailable[snapshotRequestCount]
689 snapshotRequestCount++
690 if snapshotAvailable {
691 return fakeOrderedLister{}, true
692 } else {
693 return nil, false
694 }
695 },
696 }
697 cacher.watchCache.RWMutex.Unlock()
698 if err := cacher.ready.wait(context.Background()); err != nil {
699 t.Fatalf("unexpected error waiting for the cache to be ready")
700 }
701 delegator := NewCacheDelegator(cacher, backingStorage)
702
703 result := &example.PodList{}
704 err = delegator.GetList(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "20", ResourceVersionMatch: metav1.ResourceVersionMatchExact, Recursive: true}, result)
705 if err != nil {
706 t.Fatalf("Unexpected error: %v", err)
707 }
708 if result.ResourceVersion != "20" {
709 t.Fatalf("Unexpected List response RV, got: %q, want: %d", result.ResourceVersion, 20)
710 }
711 if expectStoreRequests != tc.expectStoreRequests {
712 t.Fatalf("Unexpected number of requests to storage, got: %d, want: %d", expectStoreRequests, tc.expectStoreRequests)
713 }
714 if snapshotRequestCount != tc.expectSnapshotRequests {
715 t.Fatalf("Unexpected number of requests to snapshots, got: %d, want: %d", snapshotRequestCount, tc.expectSnapshotRequests)
716 }
717
718 })
719 }
720 }
721
722 func TestGetListNonRecursiveCacheBypass(t *testing.T) {
723 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, false)
724 backingStorage := &dummyStorage{}
725 cacher, _, err := newTestCacher(backingStorage)
726 if err != nil {
727 t.Fatalf("Couldn't create cacher: %v", err)
728 }
729 defer cacher.Stop()
730 delegator := NewCacheDelegator(cacher, backingStorage)
731 defer delegator.Stop()
732
733 pred := storage.SelectionPredicate{
734 Limit: 500,
735 }
736 result := &example.PodList{}
737
738 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
739 if err := cacher.ready.wait(context.Background()); err != nil {
740 t.Fatalf("unexpected error waiting for the cache to be ready")
741 }
742 }
743
744 // Inject error to underlying layer and check if cacher is not bypassed.
745 backingStorage.injectError(errDummy)
746 err = delegator.GetList(context.TODO(), "pods/ns", storage.ListOptions{
747 ResourceVersion: "0",
748 Predicate: pred,
749 }, result)
750 if err != nil {
751 t.Errorf("GetList with Limit and RV=0 should be served from cache: %v", err)
752 }
753
754 err = delegator.GetList(context.TODO(), "pods/ns", storage.ListOptions{
755 ResourceVersion: "",
756 Predicate: pred,
757 }, result)
758 if !errors.Is(err, errDummy) {
759 t.Errorf("GetList with Limit without RV=0 should bypass cacher: %v", err)
760 }
761 }
762
763 func TestGetListNonRecursiveCacheWithConsistentListFromCache(t *testing.T) {
764 // Set feature gates once at the beginning since we only care about ConsistentListFromCache=true and ListFromCacheSnapshot=false
765 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true)
766 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ListFromCacheSnapshot, false)
767 forceRequestWatchProgressSupport(t)
768
769 tests := []struct {
770 name string
771 consistentListFromCache bool
772 expectGetListCallCount int
773 expectGetCurrentRV bool
774 injectRVError bool
775 expectedError error
776 }{
777 {
778 name: "ConsistentListFromCache enabled - served from cache",
779 consistentListFromCache: true,
780 expectGetListCallCount: 1,
781 expectGetCurrentRV: true,
782 injectRVError: false,
783 expectedError: nil,
784 },
785 }
786
787 for _, tc := range tests {
788 t.Run(tc.name, func(t *testing.T) {
789 var getListCount, getCurrentRVCount int
790 backingStorage := &dummyStorage{}
791
792 backingStorage.getListFn = func(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
793 getListCount++
794 if tc.injectRVError {
795 return errDummy
796 }
797 podList := listObj.(*example.PodList)
798 podList.ListMeta = metav1.ListMeta{ResourceVersion: "100"}
799 return nil
800 }
801
802 backingStorage.getRVFn = func(ctx context.Context) (uint64, error) {
803 getCurrentRVCount++
804 rv := uint64(100)
805 err := error(nil)
806 if tc.injectRVError {
807 err = errDummy
808 return 0, err
809 }
810 return rv, nil
811 }
812
813 cacher, v, err := newTestCacher(backingStorage)
814 if err != nil {
815 t.Fatalf("Couldn't create cacher: %v", err)
816 }
817 defer cacher.Stop()
818
819 // Wait for cacher to be ready before injecting errors
820 if err := cacher.ready.wait(context.Background()); err != nil {
821 t.Fatalf("unexpected error waiting for the cache to be ready: %v", err)
822 }
823 delegator := NewCacheDelegator(cacher, backingStorage)
824 defer delegator.Stop()
825
826 // Setup test object
827 key := "pods/ns"
828 input := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "ns"}}
829 if err := v.UpdateObject(input, 100); err != nil {
830 t.Fatalf("Unexpected error: %v", err)
831 }
832
833 // Put object into the store
834 if err := cacher.watchCache.Add(input); err != nil {
835 t.Fatalf("Unexpected error: %v", err)
836 }
837
838 pred := storage.SelectionPredicate{
839 Label: labels.Everything(),
840 Field: fields.Everything(),
841 Limit: 500,
842 }
843 result := &example.PodList{}
844
845 // Make the list call with empty RV - delegator will get current RV and use it
846 err = delegator.GetList(context.TODO(), key, storage.ListOptions{
847 ResourceVersion: "",
848 Predicate: pred,
849 Recursive: true,
850 }, result)
851
852 // Verify error matches expectation
853 if !errors.Is(err, tc.expectedError) {
854 t.Errorf("Expected error %v, got: %v", tc.expectedError, err)
855 }
856
857 // Verify the correct storage method was called
858 if getListCount != tc.expectGetListCallCount {
859 t.Errorf("Expected GetList to be called %d times, but it was called %d times", tc.expectGetListCallCount, getListCount)
860 }
861 if tc.expectGetCurrentRV && getCurrentRVCount == 0 {
862 t.Error("Expected GetCurrentResourceVersion to be called, but it wasn't")
863 }
864 if !tc.expectGetCurrentRV && getCurrentRVCount > 0 {
865 t.Errorf("Expected GetCurrentResourceVersion not to be called, but it was called %d times", getCurrentRVCount)
866 }
867
868 // For successful cache reads, verify the resource version
869 if err == nil {
870 resultRV, err := cacher.versioner.ParseResourceVersion(result.ResourceVersion)
871 if err != nil {
872 t.Fatalf("Failed to parse result resource version: %v", err)
873 }
874 expectedRV := uint64(100)
875 if resultRV != expectedRV {
876 t.Errorf("Expected RV %d but got %d", expectedRV, resultRV)
877 }
878 }
879 })
880 }
881 }
882 func TestGetCacheBypass(t *testing.T) {
883 backingStorage := &dummyStorage{}
884 cacher, _, err := newTestCacher(backingStorage)
885 if err != nil {
886 t.Fatalf("Couldn't create cacher: %v", err)
887 }
888 defer cacher.Stop()
889 delegator := NewCacheDelegator(cacher, backingStorage)
890 defer delegator.Stop()
891
892 result := &example.Pod{}
893
894 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
895 if err := cacher.ready.wait(context.Background()); err != nil {
896 t.Fatalf("unexpected error waiting for the cache to be ready")
897 }
898 }
899
900 // Inject error to underlying layer and check if cacher is not bypassed.
901 backingStorage.injectError(errDummy)
902 err = delegator.Get(context.TODO(), "pods/ns/pod-0", storage.GetOptions{
903 IgnoreNotFound: true,
904 ResourceVersion: "0",
905 }, result)
906 if err != nil {
907 t.Errorf("Get with RV=0 should be served from cache: %v", err)
908 }
909
910 err = delegator.Get(context.TODO(), "pods/ns/pod-0", storage.GetOptions{
911 IgnoreNotFound: true,
912 ResourceVersion: "",
913 }, result)
914 if !errors.Is(err, errDummy) {
915 t.Errorf("Get without RV=0 should bypass cacher: %v", err)
916 }
917 }
918
919 func TestWatchCacheBypass(t *testing.T) {
920 backingStorage := &dummyStorage{}
921 cacher, _, err := newTestCacher(backingStorage)
922 if err != nil {
923 t.Fatalf("Couldn't create cacher: %v", err)
924 }
925 defer cacher.Stop()
926 delegator := NewCacheDelegator(cacher, backingStorage)
927 defer delegator.Stop()
928
929 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
930 if err := cacher.ready.wait(context.Background()); err != nil {
931 t.Fatalf("unexpected error waiting for the cache to be ready")
932 }
933 }
934
935 _, err = delegator.Watch(context.TODO(), "pod/ns", storage.ListOptions{
936 ResourceVersion: "0",
937 Predicate: storage.Everything,
938 })
939 if err != nil {
940 t.Errorf("Watch with RV=0 should be served from cache: %v", err)
941 }
942
943 _, err = delegator.Watch(context.TODO(), "pod/ns", storage.ListOptions{
944 ResourceVersion: "",
945 Predicate: storage.Everything,
946 })
947 if err != nil {
948 t.Errorf("Watch without RV=0 should be served from cache: %v", err)
949 }
950 }
951
952 func TestTooManyRequestsNotReturned(t *testing.T) {
953 // Ensure that with ResilientWatchCacheInitialization feature disabled, we don't return 429
954 // errors when watchcache is not initialized.
955 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ResilientWatchCacheInitialization, false)
956
957 dummyErr := fmt.Errorf("dummy")
958 backingStorage := &dummyStorage{err: dummyErr}
959 cacher, _, err := newTestCacherWithoutSyncing(backingStorage, clock.RealClock{})
960 if err != nil {
961 t.Fatalf("Couldn't create cacher: %v", err)
962 }
963 defer cacher.Stop()
964 delegator := NewCacheDelegator(cacher, backingStorage)
965 defer delegator.Stop()
966
967 opts := storage.ListOptions{
968 ResourceVersion: "0",
969 Predicate: storage.Everything,
970 }
971
972 // Cancel the request so that it doesn't hang forever.
973 listCtx, listCancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
974 defer listCancel()
975
976 result := &example.PodList{}
977 err = delegator.GetList(listCtx, "/pods/ns", opts, result)
978 if err != nil && apierrors.IsTooManyRequests(err) {
979 t.Errorf("Unexpected 429 error without ResilientWatchCacheInitialization feature for List")
980 }
981
982 watchCtx, watchCancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
983 defer watchCancel()
984
985 _, err = delegator.Watch(watchCtx, "/pods/ns", opts)
986 if err != nil && apierrors.IsTooManyRequests(err) {
987 t.Errorf("Unexpected 429 error without ResilientWatchCacheInitialization feature for Watch")
988 }
989 }
990
991 func TestEmptyWatchEventCache(t *testing.T) {
992 server, etcdStorage := newEtcdTestStorage(t, etcd3testing.PathPrefix())
993 defer server.Terminate(t)
994
995 // add a few objects
996 v := storage.APIObjectVersioner{}
997 lastRV := uint64(0)
998 for i := 0; i < 5; i++ {
999 pod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("foo-%d", i), Namespace: "test-ns"}}
1000 out := &example.Pod{}
1001 key := computePodKey(pod)
1002 if err := etcdStorage.Create(context.Background(), key, pod, out, 0); err != nil {
1003 t.Fatalf("Create failed: %v", err)
1004 }
1005 var err error
1006 if lastRV, err = v.ParseResourceVersion(out.ResourceVersion); err != nil {
1007 t.Fatalf("Unexpected error: %v", err)
1008 }
1009 }
1010
1011 cacher, _, err := newTestCacher(etcdStorage)
1012 if err != nil {
1013 t.Fatalf("Couldn't create cacher: %v", err)
1014 }
1015 defer cacher.Stop()
1016
1017 // Given that cacher is always initialized from the "current" version of etcd,
1018 // we now have a cacher with an empty cache of watch events and a resourceVersion of rv.
1019 // It should support establishing watches from rv and higher, but not older.
1020
1021 expectedResourceExpiredError := apierrors.NewResourceExpired("").ErrStatus
1022 tests := []struct {
1023 name string
1024 resourceVersion uint64
1025 expectedEvent *watch.Event
1026 }{
1027 {
1028 name: "RV-1",
1029 resourceVersion: lastRV - 1,
1030 expectedEvent: &watch.Event{Type: watch.Error, Object: &expectedResourceExpiredError},
1031 },
1032 {
1033 name: "RV",
1034 resourceVersion: lastRV,
1035 },
1036 {
1037 name: "RV+1",
1038 resourceVersion: lastRV + 1,
1039 },
1040 }
1041
1042 for _, tt := range tests {
1043 t.Run(tt.name, func(t *testing.T) {
1044 opts := storage.ListOptions{
1045 ResourceVersion: strconv.Itoa(int(tt.resourceVersion)),
1046 Predicate: storage.Everything,
1047 }
1048 watcher, err := cacher.Watch(context.Background(), "/pods/test-ns", opts)
1049 if err != nil {
1050 t.Fatalf("Failed to create watch: %v", err)
1051 }
1052 defer watcher.Stop()
1053 select {
1054 case event := <-watcher.ResultChan():
1055 if tt.expectedEvent == nil {
1056 t.Errorf("Unexpected event: type=%#v, object=%#v", event.Type, event.Object)
1057 break
1058 }
1059 if e, a := tt.expectedEvent.Type, event.Type; e != a {
1060 t.Errorf("Expected: %s, got: %s", e, a)
1061 }
1062 if e, a := tt.expectedEvent.Object, event.Object; !apiequality.Semantic.DeepDerivative(e, a) {
1063 t.Errorf("Expected: %#v, got: %#v", e, a)
1064 }
1065 case <-time.After(1 * time.Second):
1066 // the watch was established otherwise
1067 // we would be blocking on cache.Watch(...)
1068 // in addition to that, the tests are serial in nature,
1069 // meaning that there is no other actor
1070 // that could add items to the database,
1071 // which could result in receiving new items.
1072 // given that waiting 1s seems okay
1073 if tt.expectedEvent != nil {
1074 t.Errorf("Failed to get an event")
1075 }
1076 // watch remained established successfully
1077 }
1078 })
1079 }
1080 }
1081
1082 func TestWatchNotHangingOnStartupFailure(t *testing.T) {
1083 // Configure cacher so that it can't initialize, because of
1084 // constantly failing lists to the underlying storage.
1085 dummyErr := fmt.Errorf("dummy")
1086 backingStorage := &dummyStorage{err: dummyErr}
1087 cacher, _, err := newTestCacherWithoutSyncing(backingStorage, testingclock.NewFakeClock(time.Now()))
1088 if err != nil {
1089 t.Fatalf("Couldn't create cacher: %v", err)
1090 }
1091 defer cacher.Stop()
1092
1093 ctx, cancel := context.WithCancel(context.Background())
1094 // Cancel the watch after some time to check if it will properly
1095 // terminate instead of hanging forever.
1096 go func() {
1097 defer cancel()
1098 cacher.clock.Sleep(1 * time.Second)
1099 }()
1100
1101 // Watch hangs waiting on watchcache being initialized.
1102 // Ensure that it terminates when its context is cancelled
1103 // (e.g. the request is terminated for whatever reason).
1104 _, err = cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "0"})
1105 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1106 if err == nil || err.Error() != apierrors.NewServiceUnavailable(context.Canceled.Error()).Error() {
1107 t.Errorf("Unexpected error: %#v", err)
1108 }
1109 } else {
1110 if err == nil || err.Error() != apierrors.NewTooManyRequests("storage is (re)initializing", 1).Error() {
1111 t.Errorf("Unexpected error: %#v", err)
1112 }
1113 }
1114 }
1115
1116 func TestWatcherNotGoingBackInTime(t *testing.T) {
1117 backingStorage := &dummyStorage{}
1118 cacher, v, err := newTestCacher(backingStorage)
1119 if err != nil {
1120 t.Fatalf("Couldn't create cacher: %v", err)
1121 }
1122 defer cacher.Stop()
1123
1124 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1125 if err := cacher.ready.wait(context.Background()); err != nil {
1126 t.Fatalf("unexpected error waiting for the cache to be ready")
1127 }
1128 }
1129
1130 // Ensure there is some budget for slowing down processing.
1131 cacher.dispatchTimeoutBudget.returnUnused(100 * time.Millisecond)
1132
1133 makePod := func(i int) *examplev1.Pod {
1134 return &examplev1.Pod{
1135 ObjectMeta: metav1.ObjectMeta{
1136 Name: fmt.Sprintf("pod-%d", 1000+i),
1137 Namespace: "ns",
1138 ResourceVersion: fmt.Sprintf("%d", 1000+i),
1139 },
1140 }
1141 }
1142 if err := cacher.watchCache.Add(makePod(0)); err != nil {
1143 t.Errorf("error: %v", err)
1144 }
1145
1146 totalPods := 100
1147
1148 // Create watcher that will be slowing down reading.
1149 w1, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{
1150 ResourceVersion: "999",
1151 Predicate: storage.Everything,
1152 })
1153 if err != nil {
1154 t.Fatalf("Failed to create watch: %v", err)
1155 }
1156 defer w1.Stop()
1157 go func() {
1158 a := 0
1159 for range w1.ResultChan() {
1160 time.Sleep(time.Millisecond)
1161 a++
1162 if a == 100 {
1163 break
1164 }
1165 }
1166 }()
1167
1168 // Now push a ton of object to cache.
1169 for i := 1; i < totalPods; i++ {
1170 cacher.watchCache.Add(makePod(i))
1171 }
1172
1173 // Create fast watcher and ensure it will get each object exactly once.
1174 w2, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: storage.Everything})
1175 if err != nil {
1176 t.Fatalf("Failed to create watch: %v", err)
1177 }
1178 defer w2.Stop()
1179
1180 shouldContinue := true
1181 currentRV := uint64(0)
1182 for shouldContinue {
1183 select {
1184 case event, ok := <-w2.ResultChan():
1185 if !ok {
1186 shouldContinue = false
1187 break
1188 }
1189 rv, err := v.ParseResourceVersion(event.Object.(metaRuntimeInterface).GetResourceVersion())
1190 if err != nil {
1191 t.Errorf("unexpected parsing error: %v", err)
1192 } else {
1193 if rv < currentRV {
1194 t.Errorf("watcher going back in time")
1195 }
1196 currentRV = rv
1197 }
1198 case <-time.After(time.Second):
1199 w2.Stop()
1200 }
1201 }
1202 }
1203
1204 func TestCacherDontAcceptRequestsStopped(t *testing.T) {
1205 backingStorage := &dummyStorage{}
1206 cacher, _, err := newTestCacher(backingStorage)
1207 if err != nil {
1208 t.Fatalf("Couldn't create cacher: %v", err)
1209 }
1210 delegator := NewCacheDelegator(cacher, backingStorage)
1211 defer delegator.Stop()
1212
1213 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1214 if err := cacher.ready.wait(context.Background()); err != nil {
1215 t.Fatalf("unexpected error waiting for the cache to be ready")
1216 }
1217 }
1218
1219 w, err := delegator.Watch(context.Background(), "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything})
1220 if err != nil {
1221 t.Fatalf("Failed to create watch: %v", err)
1222 }
1223
1224 watchClosed := make(chan struct{})
1225 go func() {
1226 defer close(watchClosed)
1227 for event := range w.ResultChan() {
1228 switch event.Type {
1229 case watch.Added, watch.Modified, watch.Deleted:
1230 // ok
1231 default:
1232 t.Errorf("unexpected event %#v", event)
1233 }
1234 }
1235 }()
1236
1237 cacher.Stop()
1238
1239 _, err = delegator.Watch(context.Background(), "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything})
1240 if err == nil {
1241 t.Fatalf("Success to create Watch: %v", err)
1242 }
1243
1244 result := &example.Pod{}
1245 err = delegator.Get(context.TODO(), "pods/ns/pod-0", storage.GetOptions{
1246 IgnoreNotFound: true,
1247 ResourceVersion: "1",
1248 }, result)
1249 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1250 if err == nil {
1251 t.Fatalf("Success to create Get: %v", err)
1252 }
1253 } else {
1254 if err != nil {
1255 t.Fatalf("Failed to get object: %v:", err)
1256 }
1257 }
1258
1259 listResult := &example.PodList{}
1260 err = delegator.GetList(context.TODO(), "pods/ns", storage.ListOptions{
1261 ResourceVersion: "1",
1262 Recursive: true,
1263 Predicate: storage.SelectionPredicate{
1264 Limit: 500,
1265 },
1266 }, listResult)
1267 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1268 if err != nil {
1269 t.Fatalf("Failed to create GetList: %v", err)
1270 }
1271 } else {
1272 if err != nil {
1273 t.Fatalf("Failed to list objects: %v", err)
1274 }
1275 }
1276
1277 select {
1278 case <-watchClosed:
1279 case <-time.After(wait.ForeverTestTimeout):
1280 t.Errorf("timed out waiting for watch to close")
1281 }
1282 }
1283
1284 func TestCacherDontMissEventsOnReinitialization(t *testing.T) {
1285 makePod := func(i int) *example.Pod {
1286 return &example.Pod{
1287 ObjectMeta: metav1.ObjectMeta{
1288 Name: fmt.Sprintf("pod-%d", i),
1289 Namespace: "ns",
1290 ResourceVersion: fmt.Sprintf("%d", i),
1291 },
1292 }
1293 }
1294
1295 listCalls, watchCalls := 0, 0
1296 backingStorage := &dummyStorage{
1297 getListFn: func(_ context.Context, _ string, _ storage.ListOptions, listObj runtime.Object) error {
1298 podList := listObj.(*example.PodList)
1299 var err error
1300 switch listCalls {
1301 case 0:
1302 podList.ListMeta = metav1.ListMeta{ResourceVersion: "1"}
1303 case 1:
1304 podList.ListMeta = metav1.ListMeta{ResourceVersion: "10"}
1305 default:
1306 t.Errorf("unexpected list call: %d", listCalls)
1307 err = fmt.Errorf("unexpected list call")
1308 }
1309 listCalls++
1310 return err
1311 },
1312 watchFn: func(_ context.Context, _ string, _ storage.ListOptions) (watch.Interface, error) {
1313 var w *watch.FakeWatcher
1314 var err error
1315 switch watchCalls {
1316 case 0:
1317 w = watch.NewFakeWithChanSize(10, false)
1318 for i := 2; i < 8; i++ {
1319 w.Add(makePod(i))
1320 }
1321 // Emit an error to force relisting.
1322 w.Error(nil)
1323 w.Stop()
1324 case 1:
1325 w = watch.NewFakeWithChanSize(10, false)
1326 for i := 12; i < 18; i++ {
1327 w.Add(makePod(i))
1328 }
1329 // Keep the watch open to avoid another reinitialization,
1330 // but register it for cleanup.
1331 t.Cleanup(func() { w.Stop() })
1332 default:
1333 t.Errorf("unexpected watch call: %d", watchCalls)
1334 err = fmt.Errorf("unexpected watch call")
1335 }
1336 watchCalls++
1337 return w, err
1338 },
1339 }
1340 cacher, _, err := newTestCacher(backingStorage)
1341 if err != nil {
1342 t.Fatalf("Couldn't create cacher: %v", err)
1343 }
1344 defer cacher.Stop()
1345
1346 concurrency := 1000
1347 wg := sync.WaitGroup{}
1348 wg.Add(concurrency)
1349
1350 // Ensure that test doesn't deadlock if cacher already processed everything
1351 // and get back into Pending state before some watches get called.
1352 ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
1353 defer cancel()
1354
1355 for i := 0; i < concurrency; i++ {
1356 go func() {
1357 defer wg.Done()
1358 w, err := cacher.Watch(ctx, "pods", storage.ListOptions{ResourceVersion: "1", Predicate: storage.Everything})
1359 if err != nil {
1360 // Watch failed to initialize (this most probably means that cacher
1361 // already moved back to Pending state before watch initialized.
1362 // Ignore this case.
1363 return
1364 }
1365 defer w.Stop()
1366
1367 prevRV := -1
1368 for event := range w.ResultChan() {
1369 if event.Type == watch.Error {
1370 break
1371 }
1372 object := event.Object
1373 if co, ok := object.(runtime.CacheableObject); ok {
1374 object = co.GetObject()
1375 }
1376 rv, err := strconv.Atoi(object.(*example.Pod).ResourceVersion)
1377 if err != nil {
1378 t.Errorf("incorrect resource version: %v", err)
1379 return
1380 }
1381 if prevRV != -1 && prevRV+1 != rv {
1382 t.Errorf("unexpected event received, prevRV=%d, rv=%d", prevRV, rv)
1383 return
1384 }
1385 prevRV = rv
1386 }
1387
1388 }()
1389 }
1390 wg.Wait()
1391 }
1392
1393 func TestCacherNoLeakWithMultipleWatchers(t *testing.T) {
1394 backingStorage := &dummyStorage{}
1395 cacher, _, err := newTestCacher(backingStorage)
1396 if err != nil {
1397 t.Fatalf("Couldn't create cacher: %v", err)
1398 }
1399 defer cacher.Stop()
1400
1401 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1402 if err := cacher.ready.wait(context.Background()); err != nil {
1403 t.Fatalf("unexpected error waiting for the cache to be ready")
1404 }
1405 }
1406
1407 pred := storage.Everything
1408 pred.AllowWatchBookmarks = true
1409
1410 // run the collision test for 3 seconds to let ~2 buckets expire
1411 stopCh := make(chan struct{})
1412 var watchErr error
1413 time.AfterFunc(3*time.Second, func() { close(stopCh) })
1414
1415 wg := &sync.WaitGroup{}
1416
1417 wg.Add(1)
1418 go func() {
1419 defer wg.Done()
1420 for {
1421 select {
1422 case <-stopCh:
1423 return
1424 default:
1425 ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
1426 defer cancel()
1427 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: pred})
1428 if err != nil {
1429 watchErr = fmt.Errorf("Failed to create watch: %v", err)
1430 return
1431 }
1432 w.Stop()
1433 }
1434 }
1435 }()
1436
1437 wg.Add(1)
1438 go func() {
1439 defer wg.Done()
1440 for {
1441 select {
1442 case <-stopCh:
1443 return
1444 default:
1445 cacher.Lock()
1446 cacher.bookmarkWatchers.popExpiredWatchersThreadUnsafe()
1447 cacher.Unlock()
1448 }
1449 }
1450 }()
1451
1452 // wait for adding/removing watchers to end
1453 wg.Wait()
1454
1455 if watchErr != nil {
1456 t.Fatal(watchErr)
1457 }
1458
1459 // wait out the expiration period and pop expired watchers
1460 time.Sleep(2 * time.Second)
1461 cacher.Lock()
1462 defer cacher.Unlock()
1463 cacher.bookmarkWatchers.popExpiredWatchersThreadUnsafe()
1464 if len(cacher.bookmarkWatchers.watchersBuckets) != 0 {
1465 numWatchers := 0
1466 for bucketID, v := range cacher.bookmarkWatchers.watchersBuckets {
1467 numWatchers += len(v)
1468 t.Errorf("there are %v watchers at bucket Id %v with start Id %v", len(v), bucketID, cacher.bookmarkWatchers.startBucketID)
1469 }
1470 t.Errorf("unexpected bookmark watchers %v", numWatchers)
1471 }
1472 }
1473
1474 func testCacherSendBookmarkEvents(t *testing.T, allowWatchBookmarks, expectedBookmarks bool) {
1475 backingStorage := &dummyStorage{}
1476 cacher, _, err := newTestCacher(backingStorage)
1477 if err != nil {
1478 t.Fatalf("Couldn't create cacher: %v", err)
1479 }
1480 defer cacher.Stop()
1481
1482 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1483 if err := cacher.ready.wait(context.Background()); err != nil {
1484 t.Fatalf("unexpected error waiting for the cache to be ready")
1485 }
1486 }
1487 pred := storage.Everything
1488 pred.AllowWatchBookmarks = allowWatchBookmarks
1489
1490 ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
1491 defer cancel()
1492 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "0", Predicate: pred})
1493 if err != nil {
1494 t.Fatalf("Failed to create watch: %v", err)
1495 }
1496
1497 resourceVersion := uint64(1000)
1498 errc := make(chan error, 1)
1499 go func() {
1500 deadline := time.Now().Add(time.Second)
1501 for i := 0; time.Now().Before(deadline); i++ {
1502 err := cacher.watchCache.Add(&examplev1.Pod{
1503 ObjectMeta: metav1.ObjectMeta{
1504 Name: fmt.Sprintf("pod-%d", i),
1505 Namespace: "ns",
1506 ResourceVersion: fmt.Sprintf("%v", resourceVersion+uint64(i)),
1507 }})
1508 if err != nil {
1509 errc <- fmt.Errorf("failed to add a pod: %v", err)
1510 return
1511 }
1512 time.Sleep(100 * time.Millisecond)
1513 }
1514 }()
1515
1516 timeoutCh := time.After(2 * time.Second)
1517 lastObservedRV := uint64(0)
1518 for {
1519 select {
1520 case err := <-errc:
1521 t.Fatal(err)
1522 return
1523 case event, ok := <-w.ResultChan():
1524 if !ok {
1525 t.Fatal("Unexpected closed")
1526 }
1527 rv, err := cacher.versioner.ObjectResourceVersion(event.Object)
1528 if err != nil {
1529 t.Errorf("failed to parse resource version from %#v: %v", event.Object, err)
1530 }
1531 if event.Type == watch.Bookmark {
1532 if !expectedBookmarks {
1533 t.Fatalf("Unexpected bookmark events received")
1534 }
1535
1536 if rv < lastObservedRV {
1537 t.Errorf("Unexpected bookmark event resource version %v (last %v)", rv, lastObservedRV)
1538 }
1539 return
1540 }
1541 lastObservedRV = rv
1542 case <-timeoutCh:
1543 if expectedBookmarks {
1544 t.Fatal("Unexpected timeout to receive a bookmark event")
1545 }
1546 return
1547 }
1548 }
1549 }
1550
1551 func TestCacherSendBookmarkEvents(t *testing.T) {
1552 testCases := []struct {
1553 allowWatchBookmarks bool
1554 expectedBookmarks bool
1555 }{
1556 {
1557 allowWatchBookmarks: true,
1558 expectedBookmarks: true,
1559 },
1560 {
1561 allowWatchBookmarks: false,
1562 expectedBookmarks: false,
1563 },
1564 }
1565
1566 for _, tc := range testCases {
1567 testCacherSendBookmarkEvents(t, tc.allowWatchBookmarks, tc.expectedBookmarks)
1568 }
1569 }
1570
1571 func TestInitialEventsEndBookmark(t *testing.T) {
1572 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
1573 forceRequestWatchProgressSupport(t)
1574
1575 backingStorage := &dummyStorage{}
1576 cacher, _, err := newTestCacher(backingStorage)
1577 if err != nil {
1578 t.Fatalf("Couldn't create cacher: %v", err)
1579 }
1580 defer cacher.Stop()
1581
1582 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1583 if err := cacher.ready.wait(context.Background()); err != nil {
1584 t.Fatalf("unexpected error waiting for the cache to be ready")
1585 }
1586 }
1587
1588 makePod := func(index uint64) *example.Pod {
1589 return &example.Pod{
1590 ObjectMeta: metav1.ObjectMeta{
1591 Name: fmt.Sprintf("pod-%d", index),
1592 Namespace: "ns",
1593 ResourceVersion: fmt.Sprintf("%v", 100+index),
1594 },
1595 }
1596 }
1597
1598 numberOfPods := 3
1599 var expectedPodEvents []watch.Event
1600 for i := 1; i <= numberOfPods; i++ {
1601 pod := makePod(uint64(i))
1602 if err := cacher.watchCache.Add(pod); err != nil {
1603 t.Fatalf("failed to add a pod: %v", err)
1604 }
1605 expectedPodEvents = append(expectedPodEvents, watch.Event{Type: watch.Added, Object: pod})
1606 }
1607 var currentResourceVersion uint64 = 100 + 3
1608
1609 trueVal, falseVal := true, false
1610
1611 scenarios := []struct {
1612 name string
1613 allowWatchBookmarks bool
1614 sendInitialEvents *bool
1615 }{
1616 {
1617 name: "allowWatchBookmarks=false, sendInitialEvents=false",
1618 allowWatchBookmarks: false,
1619 sendInitialEvents: &falseVal,
1620 },
1621 {
1622 name: "allowWatchBookmarks=false, sendInitialEvents=true",
1623 allowWatchBookmarks: false,
1624 sendInitialEvents: &trueVal,
1625 },
1626 {
1627 name: "allowWatchBookmarks=true, sendInitialEvents=true",
1628 allowWatchBookmarks: true,
1629 sendInitialEvents: &trueVal,
1630 },
1631 {
1632 name: "allowWatchBookmarks=true, sendInitialEvents=false",
1633 allowWatchBookmarks: true,
1634 sendInitialEvents: &falseVal,
1635 },
1636 {
1637 name: "allowWatchBookmarks=false, sendInitialEvents=nil",
1638 allowWatchBookmarks: true,
1639 },
1640 }
1641
1642 for _, scenario := range scenarios {
1643 t.Run(scenario.name, func(t *testing.T) {
1644 expectedWatchEvents := expectedPodEvents
1645 if scenario.allowWatchBookmarks && scenario.sendInitialEvents != nil && *scenario.sendInitialEvents {
1646 expectedWatchEvents = append(expectedWatchEvents, watch.Event{
1647 Type: watch.Bookmark,
1648 Object: &example.Pod{
1649 ObjectMeta: metav1.ObjectMeta{
1650 ResourceVersion: strconv.FormatUint(currentResourceVersion, 10),
1651 Annotations: map[string]string{metav1.InitialEventsAnnotationKey: "true"},
1652 },
1653 },
1654 })
1655 }
1656
1657 pred := storage.Everything
1658 pred.AllowWatchBookmarks = scenario.allowWatchBookmarks
1659 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
1660 defer cancel()
1661 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "100", SendInitialEvents: scenario.sendInitialEvents, Predicate: pred})
1662 if err != nil {
1663 t.Fatalf("Failed to create watch: %v", err)
1664 }
1665 storagetesting.TestCheckResultsInStrictOrder(t, w, expectedWatchEvents)
1666 storagetesting.TestCheckNoMoreResultsWithIgnoreFunc(t, w, nil)
1667 })
1668 }
1669 }
1670
1671 func TestCacherSendsMultipleWatchBookmarks(t *testing.T) {
1672 backingStorage := &dummyStorage{}
1673 cacher, _, err := newTestCacher(backingStorage)
1674 if err != nil {
1675 t.Fatalf("Couldn't create cacher: %v", err)
1676 }
1677 defer cacher.Stop()
1678 // Update bookmarkFrequency to speed up test.
1679 // Note that the frequency lower than 1s doesn't change much due to
1680 // resolution how frequency we recompute.
1681 cacher.bookmarkWatchers.bookmarkFrequency = time.Second
1682
1683 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1684 if err := cacher.ready.wait(context.Background()); err != nil {
1685 t.Fatalf("unexpected error waiting for the cache to be ready")
1686 }
1687 }
1688 pred := storage.Everything
1689 pred.AllowWatchBookmarks = true
1690
1691 makePod := func(index int) *examplev1.Pod {
1692 return &examplev1.Pod{
1693 ObjectMeta: metav1.ObjectMeta{
1694 Name: fmt.Sprintf("pod-%d", index),
1695 Namespace: "ns",
1696 ResourceVersion: fmt.Sprintf("%v", 100+index),
1697 },
1698 }
1699 }
1700
1701 // Create pod to initialize watch cache.
1702 if err := cacher.watchCache.Add(makePod(0)); err != nil {
1703 t.Fatalf("failed to add a pod: %v", err)
1704 }
1705
1706 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
1707 defer cancel()
1708 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "100", Predicate: pred})
1709 if err != nil {
1710 t.Fatalf("Failed to create watch: %v", err)
1711 }
1712
1713 // Create one more pod, to ensure that current RV is higher and thus
1714 // bookmarks will be delievere (events are delivered for RV higher
1715 // than the max from init events).
1716 if err := cacher.watchCache.Add(makePod(1)); err != nil {
1717 t.Fatalf("failed to add a pod: %v", err)
1718 }
1719
1720 timeoutCh := time.After(5 * time.Second)
1721 lastObservedRV := uint64(0)
1722 // Ensure that a watcher gets two bookmarks.
1723 for observedBookmarks := 0; observedBookmarks < 2; {
1724 select {
1725 case event, ok := <-w.ResultChan():
1726 if !ok {
1727 t.Fatal("Unexpected closed")
1728 }
1729 rv, err := cacher.versioner.ObjectResourceVersion(event.Object)
1730 if err != nil {
1731 t.Errorf("failed to parse resource version from %#v: %v", event.Object, err)
1732 }
1733 if event.Type == watch.Bookmark {
1734 observedBookmarks++
1735 if rv < lastObservedRV {
1736 t.Errorf("Unexpected bookmark event resource version %v (last %v)", rv, lastObservedRV)
1737 }
1738 }
1739 lastObservedRV = rv
1740 case <-timeoutCh:
1741 t.Fatal("Unexpected timeout to receive bookmark events")
1742 }
1743 }
1744 }
1745
1746 func TestDispatchingBookmarkEventsWithConcurrentStop(t *testing.T) {
1747 backingStorage := &dummyStorage{}
1748 cacher, _, err := newTestCacher(backingStorage)
1749 if err != nil {
1750 t.Fatalf("Couldn't create cacher: %v", err)
1751 }
1752 defer cacher.Stop()
1753
1754 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1755 if err := cacher.ready.wait(context.Background()); err != nil {
1756 t.Fatalf("unexpected error waiting for the cache to be ready")
1757 }
1758 }
1759
1760 // Ensure there is some budget for slowing down processing.
1761 cacher.dispatchTimeoutBudget.returnUnused(100 * time.Millisecond)
1762
1763 resourceVersion := uint64(1000)
1764 err = cacher.watchCache.Add(&examplev1.Pod{
1765 ObjectMeta: metav1.ObjectMeta{
1766 Name: "pod-0",
1767 Namespace: "ns",
1768 ResourceVersion: fmt.Sprintf("%v", resourceVersion),
1769 }})
1770 if err != nil {
1771 t.Fatalf("failed to add a pod: %v", err)
1772 }
1773
1774 for i := 0; i < 1000; i++ {
1775 pred := storage.Everything
1776 pred.AllowWatchBookmarks = true
1777 ctx, cancel := context.WithTimeout(context.Background(), time.Second)
1778 defer cancel()
1779 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: pred})
1780 if err != nil {
1781 t.Fatalf("Failed to create watch: %v", err)
1782 }
1783 bookmark := &watchCacheEvent{
1784 Type: watch.Bookmark,
1785 ResourceVersion: uint64(i),
1786 Object: cacher.newFunc(),
1787 }
1788 err = cacher.versioner.UpdateObject(bookmark.Object, bookmark.ResourceVersion)
1789 if err != nil {
1790 t.Fatalf("failure to update version of object (%d) %#v", bookmark.ResourceVersion, bookmark.Object)
1791 }
1792
1793 wg := sync.WaitGroup{}
1794 wg.Add(2)
1795 go func() {
1796 cacher.processEvent(bookmark)
1797 wg.Done()
1798 }()
1799
1800 go func() {
1801 w.Stop()
1802 wg.Done()
1803 }()
1804
1805 done := make(chan struct{})
1806 go func() {
1807 for range w.ResultChan() {
1808 }
1809 close(done)
1810 }()
1811
1812 select {
1813 case <-done:
1814 case <-time.After(time.Second):
1815 t.Fatal("receive result timeout")
1816 }
1817 w.Stop()
1818 wg.Wait()
1819 }
1820 }
1821
1822 func TestBookmarksOnResourceVersionUpdates(t *testing.T) {
1823 backingStorage := &dummyStorage{}
1824 cacher, _, err := newTestCacher(backingStorage)
1825 if err != nil {
1826 t.Fatalf("Couldn't create cacher: %v", err)
1827 }
1828 defer cacher.Stop()
1829
1830 // Ensure that bookmarks are sent more frequently than every 1m.
1831 cacher.bookmarkWatchers = newTimeBucketWatchers(clock.RealClock{}, 2*time.Second)
1832
1833 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1834 if err := cacher.ready.wait(context.Background()); err != nil {
1835 t.Fatalf("unexpected error waiting for the cache to be ready")
1836 }
1837 }
1838
1839 makePod := func(i int) *examplev1.Pod {
1840 return &examplev1.Pod{
1841 ObjectMeta: metav1.ObjectMeta{
1842 Name: fmt.Sprintf("pod-%d", i),
1843 Namespace: "ns",
1844 ResourceVersion: fmt.Sprintf("%d", i),
1845 },
1846 }
1847 }
1848 if err := cacher.watchCache.Add(makePod(1000)); err != nil {
1849 t.Errorf("error: %v", err)
1850 }
1851
1852 pred := storage.Everything
1853 pred.AllowWatchBookmarks = true
1854
1855 w, err := cacher.Watch(context.TODO(), "/pods/ns", storage.ListOptions{
1856 ResourceVersion: "1000",
1857 Predicate: pred,
1858 })
1859 if err != nil {
1860 t.Fatalf("Failed to create watch: %v", err)
1861 }
1862
1863 expectedRV := 2000
1864
1865 var rcErr error
1866
1867 wg := sync.WaitGroup{}
1868 wg.Add(1)
1869 go func() {
1870 defer wg.Done()
1871 for {
1872 event, ok := <-w.ResultChan()
1873 if !ok {
1874 rcErr = errors.New("Unexpected closed channel")
1875 return
1876 }
1877 rv, err := cacher.versioner.ObjectResourceVersion(event.Object)
1878 if err != nil {
1879 t.Errorf("failed to parse resource version from %#v: %v", event.Object, err)
1880 }
1881 if event.Type == watch.Bookmark && rv == uint64(expectedRV) {
1882 return
1883 }
1884 }
1885 }()
1886
1887 // Simulate progress notify event.
1888 cacher.watchCache.UpdateResourceVersion(strconv.Itoa(expectedRV))
1889
1890 wg.Wait()
1891 if rcErr != nil {
1892 t.Fatal(rcErr)
1893 }
1894 }
1895
1896 type fakeTimeBudget struct{}
1897
1898 func (f *fakeTimeBudget) takeAvailable() time.Duration {
1899 return 2 * time.Second
1900 }
1901
1902 func (f *fakeTimeBudget) returnUnused(_ time.Duration) {}
1903
1904 func TestStartingResourceVersion(t *testing.T) {
1905 backingStorage := &dummyStorage{}
1906 cacher, _, err := newTestCacher(backingStorage)
1907 if err != nil {
1908 t.Fatalf("Couldn't create cacher: %v", err)
1909 }
1910 defer cacher.Stop()
1911
1912 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1913 if err := cacher.ready.wait(context.Background()); err != nil {
1914 t.Fatalf("unexpected error waiting for the cache to be ready")
1915 }
1916 }
1917
1918 // Ensure there is some budget for slowing down processing.
1919 // We use the fakeTimeBudget to prevent this test from flaking under
1920 // the following conditions:
1921 // 1) in total we create 11 events that has to be processed by the watcher
1922 // 2) the size of the channels are set to 10 for the watcher
1923 // 3) if the test is cpu-starved and the internal goroutine is not picking
1924 // up these events from the channel, after consuming the whole time
1925 // budget (defaulted to 100ms) on waiting, we will simply close the watch,
1926 // which will cause the test failure
1927 // Using fakeTimeBudget gives us always a budget to wait and have a test
1928 // pick up something from ResultCh in the meantime.
1929 //
1930 // The same can potentially happen in production, but in that case a watch
1931 // can be resumed by the client. This doesn't work in the case of this test,
1932 // because we explicitly want to test the behavior that object changes are
1933 // happening after the watch was initiated.
1934 cacher.dispatchTimeoutBudget = &fakeTimeBudget{}
1935
1936 makePod := func(i int) *examplev1.Pod {
1937 return &examplev1.Pod{
1938 ObjectMeta: metav1.ObjectMeta{
1939 Name: "foo",
1940 Namespace: "ns",
1941 Labels: map[string]string{"foo": strconv.Itoa(i)},
1942 ResourceVersion: fmt.Sprintf("%d", i),
1943 },
1944 }
1945 }
1946
1947 if err := cacher.watchCache.Add(makePod(1000)); err != nil {
1948 t.Errorf("error: %v", err)
1949 }
1950 // Advance RV by 10.
1951 startVersion := uint64(1010)
1952
1953 watcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", storage.ListOptions{ResourceVersion: strconv.FormatUint(startVersion, 10), Predicate: storage.Everything})
1954 if err != nil {
1955 t.Fatalf("Unexpected error: %v", err)
1956 }
1957 defer watcher.Stop()
1958
1959 for i := 1; i <= 11; i++ {
1960 if err := cacher.watchCache.Update(makePod(1000 + i)); err != nil {
1961 t.Errorf("error: %v", err)
1962 }
1963 }
1964
1965 e, ok := <-watcher.ResultChan()
1966 if !ok {
1967 t.Errorf("unexpectedly closed watch")
1968 }
1969 object := e.Object
1970 if co, ok := object.(runtime.CacheableObject); ok {
1971 object = co.GetObject()
1972 }
1973 pod := object.(*examplev1.Pod)
1974 podRV, err := cacher.versioner.ParseResourceVersion(pod.ResourceVersion)
1975 if err != nil {
1976 t.Fatalf("unexpected error: %v", err)
1977 }
1978
1979 // event should have at least rv + 1, since we're starting the watch at rv
1980 if podRV <= startVersion {
1981 t.Errorf("expected event with resourceVersion of at least %d, got %d", startVersion+1, podRV)
1982 }
1983 }
1984
1985 func TestDispatchEventWillNotBeBlockedByTimedOutWatcher(t *testing.T) {
1986 backingStorage := &dummyStorage{}
1987 cacher, _, err := newTestCacher(backingStorage)
1988 if err != nil {
1989 t.Fatalf("Couldn't create cacher: %v", err)
1990 }
1991 defer cacher.Stop()
1992
1993 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
1994 if err := cacher.ready.wait(context.Background()); err != nil {
1995 t.Fatalf("unexpected error waiting for the cache to be ready")
1996 }
1997 }
1998
1999 // Ensure there is some budget for slowing down processing.
2000 // We use the fakeTimeBudget to prevent this test from flaking under
2001 // the following conditions:
2002 // 1) the watch w1 is blocked, so we were consuming the whole budget once
2003 // its buffer was filled in (10 items)
2004 // 2) the budget is refreshed once per second, so it basically wasn't
2005 // happening in the test at all
2006 // 3) if the test was cpu-starved and we weren't able to consume events
2007 // from w2 ResultCh it could have happened that its buffer was also
2008 // filling in and given we no longer had timeBudget (consumed in (1))
2009 // trying to put next item was simply breaking the watch
2010 // Using fakeTimeBudget gives us always a budget to wait and have a test
2011 // pick up something from ResultCh in the meantime.
2012 cacher.dispatchTimeoutBudget = &fakeTimeBudget{}
2013
2014 makePod := func(i int) *examplev1.Pod {
2015 return &examplev1.Pod{
2016 ObjectMeta: metav1.ObjectMeta{
2017 Name: fmt.Sprintf("pod-%d", 1000+i),
2018 Namespace: "ns",
2019 ResourceVersion: fmt.Sprintf("%d", 1000+i),
2020 },
2021 }
2022 }
2023 if err := cacher.watchCache.Add(makePod(0)); err != nil {
2024 t.Errorf("error: %v", err)
2025 }
2026
2027 totalPods := 50
2028
2029 // Create watcher that will be blocked.
2030 w1, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: storage.Everything})
2031 if err != nil {
2032 t.Fatalf("Failed to create watch: %v", err)
2033 }
2034 defer w1.Stop()
2035
2036 // Create fast watcher and ensure it will get all objects.
2037 w2, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: storage.Everything})
2038 if err != nil {
2039 t.Fatalf("Failed to create watch: %v", err)
2040 }
2041 defer w2.Stop()
2042
2043 // Now push a ton of object to cache.
2044 for i := 1; i < totalPods; i++ {
2045 cacher.watchCache.Add(makePod(i))
2046 }
2047
2048 shouldContinue := true
2049 eventsCount := 0
2050 for shouldContinue {
2051 select {
2052 case event, ok := <-w2.ResultChan():
2053 if !ok {
2054 shouldContinue = false
2055 break
2056 }
2057 if event.Type == watch.Added {
2058 eventsCount++
2059 if eventsCount == totalPods {
2060 shouldContinue = false
2061 }
2062 }
2063 case <-time.After(wait.ForeverTestTimeout):
2064 shouldContinue = false
2065 w2.Stop()
2066 }
2067 }
2068 if eventsCount != totalPods {
2069 t.Errorf("watcher is blocked by slower one (count: %d)", eventsCount)
2070 }
2071 }
2072
2073 func verifyEvents(t *testing.T, w watch.Interface, events []watch.Event, strictOrder bool) {
2074 _, _, line, _ := goruntime.Caller(1)
2075 actualEvents := make([]watch.Event, len(events))
2076 for idx := range events {
2077 select {
2078 case event := <-w.ResultChan():
2079 actualEvents[idx] = event
2080 case <-time.After(wait.ForeverTestTimeout):
2081 t.Logf("(called from line %d)", line)
2082 t.Errorf("Timed out waiting for an event")
2083 }
2084 }
2085 validateEvents := func(expected, actual watch.Event) (bool, []string) {
2086 errors := []string{}
2087 if e, a := expected.Type, actual.Type; e != a {
2088 errors = append(errors, fmt.Sprintf("Expected: %s, got: %s", e, a))
2089 }
2090 actualObject := actual.Object
2091 if co, ok := actualObject.(runtime.CacheableObject); ok {
2092 actualObject = co.GetObject()
2093 }
2094 if e, a := expected.Object, actualObject; !apiequality.Semantic.DeepEqual(e, a) {
2095 errors = append(errors, fmt.Sprintf("Expected: %#v, got: %#v", e, a))
2096 }
2097 return len(errors) == 0, errors
2098 }
2099
2100 if len(events) != len(actualEvents) {
2101 t.Fatalf("unexpected number of events: %d, expected: %d, acutalEvents: %#v, expectedEvents:%#v", len(actualEvents), len(events), actualEvents, events)
2102 }
2103
2104 if strictOrder {
2105 for idx, expectedEvent := range events {
2106 valid, errors := validateEvents(expectedEvent, actualEvents[idx])
2107 if !valid {
2108 t.Logf("(called from line %d)", line)
2109 for _, err := range errors {
2110 t.Error(err)
2111 }
2112 }
2113 }
2114 }
2115 for _, expectedEvent := range events {
2116 validated := false
2117 for _, actualEvent := range actualEvents {
2118 if validated, _ = validateEvents(expectedEvent, actualEvent); validated {
2119 break
2120 }
2121 }
2122 if !validated {
2123 t.Fatalf("Expected: %#v but didn't find", expectedEvent)
2124 }
2125 }
2126 }
2127
2128 func TestCachingDeleteEvents(t *testing.T) {
2129 backingStorage := &dummyStorage{}
2130 cacher, _, err := newTestCacher(backingStorage)
2131 if err != nil {
2132 t.Fatalf("Couldn't create cacher: %v", err)
2133 }
2134 defer cacher.Stop()
2135
2136 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
2137 if err := cacher.ready.wait(context.Background()); err != nil {
2138 t.Fatalf("unexpected error waiting for the cache to be ready")
2139 }
2140 }
2141
2142 fooPredicate := storage.SelectionPredicate{
2143 Label: labels.SelectorFromSet(map[string]string{"foo": "true"}),
2144 Field: fields.Everything(),
2145 }
2146 barPredicate := storage.SelectionPredicate{
2147 Label: labels.SelectorFromSet(map[string]string{"bar": "true"}),
2148 Field: fields.Everything(),
2149 }
2150
2151 createWatch := func(pred storage.SelectionPredicate) watch.Interface {
2152 w, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "999", Predicate: pred})
2153 if err != nil {
2154 t.Fatalf("Failed to create watch: %v", err)
2155 }
2156 return w
2157 }
2158
2159 allEventsWatcher := createWatch(storage.Everything)
2160 defer allEventsWatcher.Stop()
2161 fooEventsWatcher := createWatch(fooPredicate)
2162 defer fooEventsWatcher.Stop()
2163 barEventsWatcher := createWatch(barPredicate)
2164 defer barEventsWatcher.Stop()
2165
2166 makePod := func(labels map[string]string, rv string) *examplev1.Pod {
2167 return &examplev1.Pod{
2168 ObjectMeta: metav1.ObjectMeta{
2169 Name: "pod",
2170 Namespace: "ns",
2171 Labels: labels,
2172 ResourceVersion: rv,
2173 },
2174 }
2175 }
2176 pod1 := makePod(map[string]string{"foo": "true", "bar": "true"}, "1001")
2177 pod2 := makePod(map[string]string{"foo": "true"}, "1002")
2178 pod3 := makePod(map[string]string{}, "1003")
2179 pod4 := makePod(map[string]string{}, "1004")
2180 pod1DeletedAt2 := pod1.DeepCopyObject().(*examplev1.Pod)
2181 pod1DeletedAt2.ResourceVersion = "1002"
2182 pod2DeletedAt3 := pod2.DeepCopyObject().(*examplev1.Pod)
2183 pod2DeletedAt3.ResourceVersion = "1003"
2184
2185 allEvents := []watch.Event{
2186 {Type: watch.Added, Object: pod1.DeepCopy()},
2187 {Type: watch.Modified, Object: pod2.DeepCopy()},
2188 {Type: watch.Modified, Object: pod3.DeepCopy()},
2189 {Type: watch.Deleted, Object: pod4.DeepCopy()},
2190 }
2191 fooEvents := []watch.Event{
2192 {Type: watch.Added, Object: pod1.DeepCopy()},
2193 {Type: watch.Modified, Object: pod2.DeepCopy()},
2194 {Type: watch.Deleted, Object: pod2DeletedAt3.DeepCopy()},
2195 }
2196 barEvents := []watch.Event{
2197 {Type: watch.Added, Object: pod1.DeepCopy()},
2198 {Type: watch.Deleted, Object: pod1DeletedAt2.DeepCopy()},
2199 }
2200
2201 cacher.watchCache.Add(pod1)
2202 cacher.watchCache.Update(pod2)
2203 cacher.watchCache.Update(pod3)
2204 cacher.watchCache.Delete(pod4)
2205
2206 verifyEvents(t, allEventsWatcher, allEvents, true)
2207 verifyEvents(t, fooEventsWatcher, fooEvents, true)
2208 verifyEvents(t, barEventsWatcher, barEvents, true)
2209 }
2210
2211 func testCachingObjects(t *testing.T, watchersCount int) {
2212 backingStorage := &dummyStorage{}
2213 cacher, _, err := newTestCacher(backingStorage)
2214 if err != nil {
2215 t.Fatalf("Couldn't create cacher: %v", err)
2216 }
2217 defer cacher.Stop()
2218
2219 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
2220 if err := cacher.ready.wait(context.Background()); err != nil {
2221 t.Fatalf("unexpected error waiting for the cache to be ready")
2222 }
2223 }
2224
2225 dispatchedEvents := []*watchCacheEvent{}
2226 cacher.watchCache.eventHandler = func(event *watchCacheEvent) {
2227 dispatchedEvents = append(dispatchedEvents, event)
2228 cacher.processEvent(event)
2229 }
2230
2231 watchers := make([]watch.Interface, 0, watchersCount)
2232 for i := 0; i < watchersCount; i++ {
2233 w, err := cacher.Watch(context.TODO(), "pods/ns", storage.ListOptions{ResourceVersion: "1000", Predicate: storage.Everything})
2234 if err != nil {
2235 t.Fatalf("Failed to create watch: %v", err)
2236 }
2237 defer w.Stop()
2238 watchers = append(watchers, w)
2239 }
2240
2241 makePod := func(name, rv string) *examplev1.Pod {
2242 return &examplev1.Pod{
2243 ObjectMeta: metav1.ObjectMeta{
2244 Name: name,
2245 Namespace: "ns",
2246 ResourceVersion: rv,
2247 },
2248 }
2249 }
2250 pod1 := makePod("pod", "1001")
2251 pod2 := makePod("pod", "1002")
2252 pod3 := makePod("pod", "1003")
2253
2254 cacher.watchCache.Add(pod1)
2255 cacher.watchCache.Update(pod2)
2256 cacher.watchCache.Delete(pod3)
2257
2258 // At this point, we already have dispatchedEvents fully propagated.
2259
2260 verifyEvents := func(w watch.Interface) {
2261 var event watch.Event
2262 for index := range dispatchedEvents {
2263 select {
2264 case event = <-w.ResultChan():
2265 case <-time.After(wait.ForeverTestTimeout):
2266 t.Fatalf("timeout watiching for the event")
2267 }
2268
2269 var object runtime.Object
2270 if _, ok := event.Object.(runtime.CacheableObject); !ok {
2271 t.Fatalf("Object in %s event should support caching: %#v", event.Type, event.Object)
2272 }
2273 object = event.Object.(runtime.CacheableObject).GetObject()
2274
2275 if event.Type == watch.Deleted {
2276 resourceVersion, err := cacher.versioner.ObjectResourceVersion(cacher.watchCache.cache[index].PrevObject)
2277 if err != nil {
2278 t.Fatalf("Failed to parse resource version: %v", err)
2279 }
2280 updateResourceVersion(object, cacher.versioner, resourceVersion)
2281 }
2282
2283 var e runtime.Object
2284 switch event.Type {
2285 case watch.Added, watch.Modified:
2286 e = cacher.watchCache.cache[index].Object
2287 case watch.Deleted:
2288 e = cacher.watchCache.cache[index].PrevObject
2289 default:
2290 t.Errorf("unexpected watch event: %#v", event)
2291 }
2292 if a := object; !reflect.DeepEqual(a, e) {
2293 t.Errorf("event object messed up for %s: %#v, expected: %#v", event.Type, a, e)
2294 }
2295 }
2296 }
2297
2298 for i := range watchers {
2299 verifyEvents(watchers[i])
2300 }
2301 }
2302
2303 func TestCachingObjects(t *testing.T) {
2304 t.Run("single watcher", func(t *testing.T) { testCachingObjects(t, 1) })
2305 t.Run("many watcher", func(t *testing.T) { testCachingObjects(t, 3) })
2306 }
2307
2308 func TestCacheIntervalInvalidationStopsWatch(t *testing.T) {
2309 backingStorage := &dummyStorage{}
2310 cacher, _, err := newTestCacher(backingStorage)
2311 if err != nil {
2312 t.Fatalf("Couldn't create cacher: %v", err)
2313 }
2314 defer cacher.Stop()
2315
2316 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
2317 if err := cacher.ready.wait(context.Background()); err != nil {
2318 t.Fatalf("unexpected error waiting for the cache to be ready")
2319 }
2320 }
2321
2322 // Ensure there is enough budget for slow processing since
2323 // the entire watch cache is going to be served through the
2324 // interval and events won't be popped from the cacheWatcher's
2325 // input channel until much later.
2326 cacher.dispatchTimeoutBudget.returnUnused(100 * time.Millisecond)
2327
2328 // We define a custom index validator such that the interval is
2329 // able to serve the first bufferSize elements successfully, but
2330 // on trying to fill it's buffer again, the indexValidator simulates
2331 // an invalidation leading to the watch being closed and the number
2332 // of events we actually process to be bufferSize, each event of
2333 // type watch.Added.
2334 valid := true
2335 invalidateCacheInterval := func() {
2336 valid = false
2337 }
2338 once := sync.Once{}
2339 indexValidator := func(index int) bool {
2340 isValid := valid && (index >= cacher.watchCache.startIndex)
2341 once.Do(invalidateCacheInterval)
2342 return isValid
2343 }
2344 cacher.watchCache.indexValidator = indexValidator
2345
2346 makePod := func(i int) *examplev1.Pod {
2347 return &examplev1.Pod{
2348 ObjectMeta: metav1.ObjectMeta{
2349 Name: fmt.Sprintf("pod-%d", 1000+i),
2350 Namespace: "ns",
2351 ResourceVersion: fmt.Sprintf("%d", 1000+i),
2352 },
2353 }
2354 }
2355
2356 // 250 is arbitrary, point is to have enough elements such that
2357 // it generates more than bufferSize number of events allowing
2358 // us to simulate the invalidation of the cache interval.
2359 totalPods := 250
2360 for i := 0; i < totalPods; i++ {
2361 err := cacher.watchCache.Add(makePod(i))
2362 if err != nil {
2363 t.Errorf("error: %v", err)
2364 }
2365 }
2366 ctx, cancel := context.WithCancel(context.Background())
2367 defer cancel()
2368
2369 w, err := cacher.Watch(ctx, "pods/ns", storage.ListOptions{
2370 ResourceVersion: "999",
2371 Predicate: storage.Everything,
2372 })
2373 if err != nil {
2374 t.Fatalf("Failed to create watch: %v", err)
2375 }
2376 defer w.Stop()
2377
2378 received := 0
2379 resChan := w.ResultChan()
2380 for event := range resChan {
2381 received++
2382 t.Logf("event type: %v, events received so far: %d", event.Type, received)
2383 if event.Type != watch.Added {
2384 t.Errorf("unexpected event type, expected: %s, got: %s, event: %v", watch.Added, event.Type, event)
2385 }
2386 }
2387 // Since the watch is stopped after the interval is invalidated,
2388 // we should have processed exactly bufferSize number of elements.
2389 if received != bufferSize {
2390 t.Errorf("unexpected number of events received, expected: %d, got: %d", bufferSize+1, received)
2391 }
2392 }
2393
2394 func TestWaitUntilWatchCacheFreshAndForceAllEvents(t *testing.T) {
2395 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
2396 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true)
2397 forceRequestWatchProgressSupport(t)
2398
2399 scenarios := []struct {
2400 name string
2401 opts storage.ListOptions
2402 backingStorage *dummyStorage
2403 verifyBackingStore func(t *testing.T, s *dummyStorage)
2404 }{
2405 {
2406 name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=105",
2407 opts: storage.ListOptions{
2408 Predicate: func() storage.SelectionPredicate {
2409 p := storage.Everything
2410 p.AllowWatchBookmarks = true
2411 return p
2412 }(),
2413 SendInitialEvents: pointer.Bool(true),
2414 ResourceVersion: "105",
2415 },
2416 verifyBackingStore: func(t *testing.T, s *dummyStorage) {
2417 require.NotEqual(t, 0, s.getRequestWatchProgressCounter(), "expected store.RequestWatchProgressCounter to be > 0. It looks like watch progress wasn't requested!")
2418 },
2419 },
2420
2421 {
2422 name: "legacy: allowWatchBookmarks=false, sendInitialEvents=true, RV=unset",
2423 opts: storage.ListOptions{
2424 Predicate: func() storage.SelectionPredicate {
2425 p := storage.Everything
2426 p.AllowWatchBookmarks = false
2427 return p
2428 }(),
2429 SendInitialEvents: pointer.Bool(true),
2430 },
2431 backingStorage: func() *dummyStorage {
2432 hasBeenPrimed := false
2433 s := &dummyStorage{}
2434 s.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
2435 listAccessor, err := meta.ListAccessor(listObj)
2436 if err != nil {
2437 return err
2438 }
2439 // the first call to this function
2440 // primes the cacher
2441 if !hasBeenPrimed {
2442 listAccessor.SetResourceVersion("100")
2443 hasBeenPrimed = true
2444 return nil
2445 }
2446 listAccessor.SetResourceVersion("105")
2447 return nil
2448 }
2449 s.getRVFn = func(_ context.Context) (uint64, error) {
2450 // the first call to this function
2451 // primes the cacher
2452 if !hasBeenPrimed {
2453 hasBeenPrimed = true
2454 return 100, nil
2455 }
2456 return 105, nil
2457 }
2458 return s
2459 }(),
2460 verifyBackingStore: func(t *testing.T, s *dummyStorage) {
2461 require.NotEqual(t, 0, s.getRequestWatchProgressCounter(), "expected store.RequestWatchProgressCounter to be > 0. It looks like watch progress wasn't requested!")
2462 },
2463 },
2464
2465 {
2466 name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=unset",
2467 opts: storage.ListOptions{
2468 Predicate: func() storage.SelectionPredicate {
2469 p := storage.Everything
2470 p.AllowWatchBookmarks = true
2471 return p
2472 }(),
2473 SendInitialEvents: pointer.Bool(true),
2474 },
2475 backingStorage: func() *dummyStorage {
2476 hasBeenPrimed := false
2477 s := &dummyStorage{}
2478 s.getListFn = func(_ context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
2479 listAccessor, err := meta.ListAccessor(listObj)
2480 if err != nil {
2481 return err
2482 }
2483 // the first call to this function
2484 // primes the cacher
2485 if !hasBeenPrimed {
2486 listAccessor.SetResourceVersion("100")
2487 hasBeenPrimed = true
2488 return nil
2489 }
2490 listAccessor.SetResourceVersion("105")
2491 return nil
2492 }
2493 s.getRVFn = func(_ context.Context) (uint64, error) {
2494 // the first call to this function
2495 // primes the cacher
2496 if !hasBeenPrimed {
2497 hasBeenPrimed = true
2498 return 100, nil
2499 }
2500 return 105, nil
2501 }
2502 return s
2503 }(),
2504 verifyBackingStore: func(t *testing.T, s *dummyStorage) {
2505 require.NotEqual(t, 0, s.getRequestWatchProgressCounter(), "expected store.RequestWatchProgressCounter to be > 0. It looks like watch progress wasn't requested!")
2506 },
2507 },
2508 }
2509
2510 for _, scenario := range scenarios {
2511 t.Run(scenario.name, func(t *testing.T) {
2512 t.Parallel()
2513 var backingStorage *dummyStorage
2514 if scenario.backingStorage != nil {
2515 backingStorage = scenario.backingStorage
2516 } else {
2517 backingStorage = &dummyStorage{}
2518 }
2519 cacher, _, err := newTestCacher(backingStorage)
2520 if err != nil {
2521 t.Fatalf("Couldn't create cacher: %v", err)
2522 }
2523 defer cacher.Stop()
2524
2525 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
2526 if err := cacher.ready.wait(context.Background()); err != nil {
2527 t.Fatalf("unexpected error waiting for the cache to be ready")
2528 }
2529 }
2530
2531 w, err := cacher.Watch(context.Background(), "pods/ns", scenario.opts)
2532 require.NoError(t, err, "failed to create watch: %v")
2533 defer w.Stop()
2534 var expectedErr *apierrors.StatusError
2535 if !errors.As(storage.NewTooLargeResourceVersionError(105, 100, resourceVersionTooHighRetrySeconds), &expectedErr) {
2536 t.Fatalf("Unable to convert NewTooLargeResourceVersionError to apierrors.StatusError")
2537 }
2538 verifyEvents(t, w, []watch.Event{
2539 {
2540 Type: watch.Error,
2541 Object: &metav1.Status{
2542 Status: metav1.StatusFailure,
2543 Message: expectedErr.Error(),
2544 Details: expectedErr.ErrStatus.Details,
2545 Reason: metav1.StatusReasonTimeout,
2546 Code: 504,
2547 },
2548 },
2549 }, true)
2550
2551 go func(t *testing.T) {
2552 err := cacher.watchCache.Add(makeTestPodDetails("pod1", 105, "node1", map[string]string{"label": "value1"}))
2553 if err != nil {
2554 t.Errorf("failed adding a pod to the watchCache %v", err)
2555 }
2556 }(t)
2557 w, err = cacher.Watch(context.Background(), "pods/ns", scenario.opts)
2558 require.NoError(t, err, "failed to create watch: %v")
2559 defer w.Stop()
2560 verifyEvents(t, w, []watch.Event{
2561 {
2562 Type: watch.Added,
2563 Object: makeTestPodDetails("pod1", 105, "node1", map[string]string{"label": "value1"}),
2564 },
2565 }, true)
2566 if scenario.verifyBackingStore != nil {
2567 scenario.verifyBackingStore(t, backingStorage)
2568 }
2569 })
2570 }
2571 }
2572
2573 type fakeStorage struct {
2574 pods []example.Pod
2575 storage.Interface
2576 }
2577
2578 func newObjectStorage(fakePods []example.Pod) *fakeStorage {
2579 return &fakeStorage{
2580 pods: fakePods,
2581 }
2582 }
2583
2584 func (m fakeStorage) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
2585 podList := listObj.(*example.PodList)
2586 podList.ListMeta = metav1.ListMeta{ResourceVersion: "12345"}
2587 podList.Items = m.pods
2588 return nil
2589 }
2590 func (m fakeStorage) Watch(_ context.Context, _ string, _ storage.ListOptions) (watch.Interface, error) {
2591 return newDummyWatch(), nil
2592 }
2593
2594 func BenchmarkCacher_GetList(b *testing.B) {
2595 testCases := []struct {
2596 totalObjectNum int
2597 expectObjectNum int
2598 }{
2599 {
2600 totalObjectNum: 5000,
2601 expectObjectNum: 50,
2602 },
2603 {
2604 totalObjectNum: 5000,
2605 expectObjectNum: 500,
2606 },
2607 {
2608 totalObjectNum: 5000,
2609 expectObjectNum: 1000,
2610 },
2611 {
2612 totalObjectNum: 5000,
2613 expectObjectNum: 2500,
2614 },
2615 {
2616 totalObjectNum: 5000,
2617 expectObjectNum: 5000,
2618 },
2619 }
2620 for _, tc := range testCases {
2621 b.Run(
2622 fmt.Sprintf("totalObjectNum=%d, expectObjectNum=%d", tc.totalObjectNum, tc.expectObjectNum),
2623 func(b *testing.B) {
2624 // create sample pods
2625 fakePods := make([]example.Pod, tc.totalObjectNum, tc.totalObjectNum)
2626 for i := range fakePods {
2627 fakePods[i].Namespace = "default"
2628 fakePods[i].Name = fmt.Sprintf("pod-%d", i)
2629 fakePods[i].ResourceVersion = strconv.Itoa(i)
2630 if i%(tc.totalObjectNum/tc.expectObjectNum) == 0 {
2631 fakePods[i].Spec.NodeName = "node-0"
2632 }
2633 data := make([]byte, 1024*2, 1024*2) // 2k labels
2634 rand.Read(data)
2635 fakePods[i].Spec.NodeSelector = map[string]string{
2636 "key": string(data),
2637 }
2638 }
2639
2640 // build test cacher
2641 store := newObjectStorage(fakePods)
2642 cacher, _, err := newTestCacher(store)
2643 if err != nil {
2644 b.Fatalf("new cacher: %v", err)
2645 }
2646 defer cacher.Stop()
2647 delegator := NewCacheDelegator(cacher, store)
2648 defer delegator.Stop()
2649
2650 // prepare result and pred
2651 parsedField, err := fields.ParseSelector("spec.nodeName=node-0")
2652 if err != nil {
2653 b.Fatalf("parse selector: %v", err)
2654 }
2655 pred := storage.SelectionPredicate{
2656 Label: labels.Everything(),
2657 Field: parsedField,
2658 }
2659
2660 // now we start benchmarking
2661 b.ResetTimer()
2662 for i := 0; i < b.N; i++ {
2663 result := &example.PodList{}
2664 err = delegator.GetList(context.TODO(), "pods", storage.ListOptions{
2665 Predicate: pred,
2666 Recursive: true,
2667 ResourceVersion: "12345",
2668 }, result)
2669 if err != nil {
2670 b.Fatalf("GetList cache: %v", err)
2671 }
2672 if len(result.Items) != tc.expectObjectNum {
2673 b.Fatalf("expect %d but got %d", tc.expectObjectNum, len(result.Items))
2674 }
2675 }
2676 })
2677 }
2678 }
2679
2680 // TestWatchListIsSynchronisedWhenNoEventsFromStoreReceived makes sure that
2681 // a bookmark event will be delivered even if the cacher has not received an event.
2682 func TestWatchListIsSynchronisedWhenNoEventsFromStoreReceived(t *testing.T) {
2683 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
2684 backingStorage := &dummyStorage{}
2685 cacher, _, err := newTestCacher(backingStorage)
2686 require.NoError(t, err, "failed to create cacher")
2687 defer cacher.Stop()
2688
2689 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
2690 if err := cacher.ready.wait(context.Background()); err != nil {
2691 t.Fatalf("unexpected error waiting for the cache to be ready")
2692 }
2693 }
2694
2695 pred := storage.Everything
2696 pred.AllowWatchBookmarks = true
2697 opts := storage.ListOptions{
2698 Predicate: pred,
2699 SendInitialEvents: pointer.Bool(true),
2700 }
2701 w, err := cacher.Watch(context.Background(), "pods/ns", opts)
2702 require.NoError(t, err, "failed to create watch: %v")
2703 defer w.Stop()
2704
2705 verifyEvents(t, w, []watch.Event{
2706 {Type: watch.Bookmark, Object: &example.Pod{
2707 ObjectMeta: metav1.ObjectMeta{
2708 ResourceVersion: "100",
2709 Annotations: map[string]string{metav1.InitialEventsAnnotationKey: "true"},
2710 },
2711 }},
2712 }, true)
2713 }
2714
2715 func TestForgetWatcher(t *testing.T) {
2716 backingStorage := &dummyStorage{}
2717 cacher, _, err := newTestCacher(backingStorage)
2718 require.NoError(t, err)
2719 defer cacher.Stop()
2720
2721 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
2722 if err := cacher.ready.wait(context.Background()); err != nil {
2723 t.Fatalf("unexpected error waiting for the cache to be ready")
2724 }
2725 }
2726
2727 assertCacherInternalState := func(expectedWatchersCounter, expectedValueWatchersCounter int) {
2728 cacher.Lock()
2729 defer cacher.Unlock()
2730
2731 require.Len(t, cacher.watchers.allWatchers, expectedWatchersCounter)
2732 require.Len(t, cacher.watchers.valueWatchers, expectedValueWatchersCounter)
2733 }
2734 assertCacherInternalState(0, 0)
2735
2736 var forgetWatcherFn func(bool)
2737 var forgetCounter int
2738 forgetWatcherWrapped := func(drainWatcher bool) {
2739 forgetCounter++
2740 forgetWatcherFn(drainWatcher)
2741 }
2742 w := newCacheWatcher(
2743 0,
2744 func(_ string, _ labels.Set, _ fields.Set) bool { return true },
2745 nil,
2746 storage.APIObjectVersioner{},
2747 testingclock.NewFakeClock(time.Now()).Now().Add(2*time.Minute),
2748 true,
2749 schema.GroupResource{Resource: "pods"},
2750 "1",
2751 )
2752 forgetWatcherFn = forgetWatcher(cacher, w, 0, namespacedName{}, "", false)
2753 addWatcher := func(w *cacheWatcher) {
2754 cacher.Lock()
2755 defer cacher.Unlock()
2756
2757 cacher.watchers.addWatcher(w, 0, namespacedName{}, "", false)
2758 }
2759
2760 addWatcher(w)
2761 assertCacherInternalState(1, 0)
2762
2763 forgetWatcherWrapped(false)
2764 assertCacherInternalState(0, 0)
2765 require.Equal(t, 1, forgetCounter)
2766
2767 forgetWatcherWrapped(false)
2768 assertCacherInternalState(0, 0)
2769 require.Equal(t, 2, forgetCounter)
2770 }
2771
2772 // TestGetWatchCacheResourceVersion test the following cases:
2773 //
2774 // +-----------------+---------------------+-----------------------+
2775 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents |
2776 // +=================+=====================+=======================+
2777 // | Unset | true/false | nil/true/false |
2778 // | 0 | true/false | nil/true/false |
2779 // | 95 | true/false | nil/true/false |
2780 // +-----------------+---------------------+-----------------------+
2781 // where:
2782 // - false indicates the value of the param was set to "false" by a test case
2783 // - true indicates the value of the param was set to "true" by a test case
2784 func TestGetWatchCacheResourceVersion(t *testing.T) {
2785 listOptions := func(allowBookmarks bool, sendInitialEvents *bool, rv string) storage.ListOptions {
2786 p := storage.Everything
2787 p.AllowWatchBookmarks = allowBookmarks
2788
2789 opts := storage.ListOptions{}
2790 opts.Predicate = p
2791 opts.SendInitialEvents = sendInitialEvents
2792 opts.ResourceVersion = rv
2793 return opts
2794 }
2795
2796 scenarios := []struct {
2797 name string
2798 opts storage.ListOptions
2799
2800 expectedWatchResourceVersion int
2801 }{
2802 // +-----------------+---------------------+-----------------------+
2803 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents |
2804 // +=================+=====================+=======================+
2805 // | Unset | true/false | nil/true/false |
2806 // +-----------------+---------------------+-----------------------+
2807 {
2808 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=nil",
2809 opts: listOptions(true, nil, ""),
2810 // Expecting RV 0, due to https://github.com/kubernetes/kubernetes/pull/123935 reverted to serving those requests from watch cache.
2811 // Set to 100, when WatchFromStorageWithoutResourceVersion is set to true.
2812 expectedWatchResourceVersion: 0,
2813 },
2814 {
2815 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=true",
2816 opts: listOptions(true, pointer.Bool(true), ""),
2817 expectedWatchResourceVersion: 100,
2818 },
2819 {
2820 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=false",
2821 opts: listOptions(true, pointer.Bool(false), ""),
2822 expectedWatchResourceVersion: 100,
2823 },
2824 {
2825 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=nil",
2826 opts: listOptions(false, nil, ""),
2827 // Expecting RV 0, due to https://github.com/kubernetes/kubernetes/pull/123935 reverted to serving those requests from watch cache.
2828 // Set to 100, when WatchFromStorageWithoutResourceVersion is set to true.
2829 expectedWatchResourceVersion: 0,
2830 },
2831 {
2832 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=true, legacy",
2833 opts: listOptions(false, pointer.Bool(true), ""),
2834 expectedWatchResourceVersion: 100,
2835 },
2836 {
2837 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=false",
2838 opts: listOptions(false, pointer.Bool(false), ""),
2839 expectedWatchResourceVersion: 100,
2840 },
2841 // +-----------------+---------------------+-----------------------+
2842 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents |
2843 // +=================+=====================+=======================+
2844 // | 0 | true/false | nil/true/false |
2845 // +-----------------+---------------------+-----------------------+
2846 {
2847 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=nil",
2848 opts: listOptions(true, nil, "0"),
2849 expectedWatchResourceVersion: 0,
2850 },
2851 {
2852 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=true",
2853 opts: listOptions(true, pointer.Bool(true), "0"),
2854 expectedWatchResourceVersion: 0,
2855 },
2856 {
2857 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=false",
2858 opts: listOptions(true, pointer.Bool(false), "0"),
2859 expectedWatchResourceVersion: 0,
2860 },
2861 {
2862 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=nil",
2863 opts: listOptions(false, nil, "0"),
2864 expectedWatchResourceVersion: 0,
2865 },
2866 {
2867 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=true",
2868 opts: listOptions(false, pointer.Bool(true), "0"),
2869 expectedWatchResourceVersion: 0,
2870 },
2871 {
2872 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=false",
2873 opts: listOptions(false, pointer.Bool(false), "0"),
2874 expectedWatchResourceVersion: 0,
2875 },
2876 // +-----------------+---------------------+-----------------------+
2877 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents |
2878 // +=================+=====================+=======================+
2879 // | 95 | true/false | nil/true/false |
2880 // +-----------------+---------------------+-----------------------+
2881 {
2882 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=nil",
2883 opts: listOptions(true, nil, "95"),
2884 expectedWatchResourceVersion: 95,
2885 },
2886 {
2887 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=true",
2888 opts: listOptions(true, pointer.Bool(true), "95"),
2889 expectedWatchResourceVersion: 95,
2890 },
2891 {
2892 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=false",
2893 opts: listOptions(true, pointer.Bool(false), "95"),
2894 expectedWatchResourceVersion: 95,
2895 },
2896 {
2897 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=nil",
2898 opts: listOptions(false, nil, "95"),
2899 expectedWatchResourceVersion: 95,
2900 },
2901 {
2902 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=true",
2903 opts: listOptions(false, pointer.Bool(true), "95"),
2904 expectedWatchResourceVersion: 95,
2905 },
2906 {
2907 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=false",
2908 opts: listOptions(false, pointer.Bool(false), "95"),
2909 expectedWatchResourceVersion: 95,
2910 },
2911 }
2912
2913 for _, scenario := range scenarios {
2914 t.Run(scenario.name, func(t *testing.T) {
2915 backingStorage := &dummyStorage{}
2916 cacher, _, err := newTestCacher(backingStorage)
2917 require.NoError(t, err, "couldn't create cacher")
2918 defer cacher.Stop()
2919
2920 parsedResourceVersion := 0
2921 if len(scenario.opts.ResourceVersion) > 0 {
2922 parsedResourceVersion, err = strconv.Atoi(scenario.opts.ResourceVersion)
2923 require.NoError(t, err)
2924 }
2925
2926 actualResourceVersion, err := cacher.getWatchCacheResourceVersion(context.TODO(), uint64(parsedResourceVersion), scenario.opts)
2927 require.NoError(t, err)
2928 require.Equal(t, uint64(scenario.expectedWatchResourceVersion), actualResourceVersion, "received unexpected ResourceVersion")
2929 })
2930 }
2931 }
2932
2933 // TestGetBookmarkAfterResourceVersionLockedFunc test the following cases:
2934 //
2935 // +-----------------+---------------------+-----------------------+
2936 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents |
2937 // +=================+=====================+=======================+
2938 // | Unset | true/false | nil/true/false |
2939 // | 0 | true/false | nil/true/false |
2940 // | 95 | true/false | nil/true/false |
2941 // +-----------------+---------------------+-----------------------+
2942 // where:
2943 // - false indicates the value of the param was set to "false" by a test case
2944 // - true indicates the value of the param was set to "true" by a test case
2945 func TestGetBookmarkAfterResourceVersionLockedFunc(t *testing.T) {
2946 listOptions := func(allowBookmarks bool, sendInitialEvents *bool, rv string) storage.ListOptions {
2947 p := storage.Everything
2948 p.AllowWatchBookmarks = allowBookmarks
2949
2950 opts := storage.ListOptions{}
2951 opts.Predicate = p
2952 opts.SendInitialEvents = sendInitialEvents
2953 opts.ResourceVersion = rv
2954 return opts
2955 }
2956
2957 scenarios := []struct {
2958 name string
2959 opts storage.ListOptions
2960 requiredResourceVersion int
2961 watchCacheResourceVersion int
2962
2963 expectedBookmarkResourceVersion int
2964 }{
2965 // +-----------------+---------------------+-----------------------+
2966 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents |
2967 // +=================+=====================+=======================+
2968 // | Unset | true/false | nil/true/false |
2969 // +-----------------+---------------------+-----------------------+
2970 {
2971 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=nil",
2972 opts: listOptions(true, nil, ""),
2973 requiredResourceVersion: 100,
2974 watchCacheResourceVersion: 99,
2975 expectedBookmarkResourceVersion: 0,
2976 },
2977 {
2978 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=true",
2979 opts: listOptions(true, pointer.Bool(true), ""),
2980 requiredResourceVersion: 100,
2981 watchCacheResourceVersion: 99,
2982 expectedBookmarkResourceVersion: 100,
2983 },
2984 {
2985 name: "RV=unset, allowWatchBookmarks=true, sendInitialEvents=false",
2986 opts: listOptions(true, pointer.Bool(false), ""),
2987 requiredResourceVersion: 100,
2988 watchCacheResourceVersion: 99,
2989 expectedBookmarkResourceVersion: 0,
2990 },
2991 {
2992 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=nil",
2993 opts: listOptions(false, nil, ""),
2994 requiredResourceVersion: 100,
2995 watchCacheResourceVersion: 99,
2996 expectedBookmarkResourceVersion: 0,
2997 },
2998 {
2999 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=true",
3000 opts: listOptions(false, pointer.Bool(true), ""),
3001 requiredResourceVersion: 100,
3002 watchCacheResourceVersion: 99,
3003 expectedBookmarkResourceVersion: 0,
3004 },
3005 {
3006 name: "RV=unset, allowWatchBookmarks=false, sendInitialEvents=false",
3007 opts: listOptions(false, pointer.Bool(false), ""),
3008 requiredResourceVersion: 100,
3009 watchCacheResourceVersion: 99,
3010 expectedBookmarkResourceVersion: 0,
3011 },
3012 // +-----------------+---------------------+-----------------------+
3013 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents |
3014 // +=================+=====================+=======================+
3015 // | 0 | true/false | nil/true/false |
3016 // +-----------------+---------------------+-----------------------+
3017 {
3018 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=nil",
3019 opts: listOptions(true, nil, "0"),
3020 requiredResourceVersion: 0,
3021 watchCacheResourceVersion: 99,
3022 expectedBookmarkResourceVersion: 0,
3023 },
3024 {
3025 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=true",
3026 opts: listOptions(true, pointer.Bool(true), "0"),
3027 requiredResourceVersion: 0,
3028 watchCacheResourceVersion: 99,
3029 expectedBookmarkResourceVersion: 99,
3030 },
3031 {
3032 name: "RV=0, allowWatchBookmarks=true, sendInitialEvents=false",
3033 opts: listOptions(true, pointer.Bool(false), "0"),
3034 requiredResourceVersion: 0,
3035 watchCacheResourceVersion: 99,
3036 expectedBookmarkResourceVersion: 0,
3037 },
3038 {
3039 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=nil",
3040 opts: listOptions(false, nil, "0"),
3041 requiredResourceVersion: 0,
3042 watchCacheResourceVersion: 99,
3043 expectedBookmarkResourceVersion: 0,
3044 },
3045 {
3046 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=true",
3047 opts: listOptions(false, pointer.Bool(true), "0"),
3048 requiredResourceVersion: 0,
3049 watchCacheResourceVersion: 99,
3050 expectedBookmarkResourceVersion: 0,
3051 },
3052 {
3053 name: "RV=0, allowWatchBookmarks=false, sendInitialEvents=false",
3054 opts: listOptions(false, pointer.Bool(false), "0"),
3055 requiredResourceVersion: 0,
3056 watchCacheResourceVersion: 99,
3057 expectedBookmarkResourceVersion: 0,
3058 },
3059 // +-----------------+---------------------+-----------------------+
3060 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents |
3061 // +=================+=====================+=======================+
3062 // | 95 | true/false | nil/true/false |
3063 // +-----------------+---------------------+-----------------------+
3064 {
3065 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=nil",
3066 opts: listOptions(true, nil, "95"),
3067 requiredResourceVersion: 0,
3068 watchCacheResourceVersion: 99,
3069 expectedBookmarkResourceVersion: 0,
3070 },
3071 {
3072 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=true",
3073 opts: listOptions(true, pointer.Bool(true), "95"),
3074 requiredResourceVersion: 0,
3075 watchCacheResourceVersion: 99,
3076 expectedBookmarkResourceVersion: 95,
3077 },
3078 {
3079 name: "RV=95, allowWatchBookmarks=true, sendInitialEvents=false",
3080 opts: listOptions(true, pointer.Bool(false), "95"),
3081 requiredResourceVersion: 0,
3082 watchCacheResourceVersion: 99,
3083 expectedBookmarkResourceVersion: 0,
3084 },
3085 {
3086 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=nil",
3087 opts: listOptions(false, nil, "95"),
3088 requiredResourceVersion: 100,
3089 watchCacheResourceVersion: 99,
3090 expectedBookmarkResourceVersion: 0,
3091 },
3092 {
3093 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=true",
3094 opts: listOptions(false, pointer.Bool(true), "95"),
3095 requiredResourceVersion: 0,
3096 watchCacheResourceVersion: 99,
3097 expectedBookmarkResourceVersion: 0,
3098 },
3099 {
3100 name: "RV=95, allowWatchBookmarks=false, sendInitialEvents=false",
3101 opts: listOptions(false, pointer.Bool(false), "95"),
3102 requiredResourceVersion: 0,
3103 watchCacheResourceVersion: 99,
3104 expectedBookmarkResourceVersion: 0,
3105 },
3106 }
3107 for _, scenario := range scenarios {
3108 t.Run(scenario.name, func(t *testing.T) {
3109 backingStorage := &dummyStorage{}
3110 cacher, _, err := newTestCacher(backingStorage)
3111 require.NoError(t, err, "couldn't create cacher")
3112
3113 defer cacher.Stop()
3114
3115 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
3116 if err := cacher.ready.wait(context.Background()); err != nil {
3117 t.Fatalf("unexpected error waiting for the cache to be ready")
3118 }
3119 }
3120
3121 cacher.watchCache.UpdateResourceVersion(fmt.Sprintf("%d", scenario.watchCacheResourceVersion))
3122 parsedResourceVersion := 0
3123 if len(scenario.opts.ResourceVersion) > 0 {
3124 parsedResourceVersion, err = strconv.Atoi(scenario.opts.ResourceVersion)
3125 require.NoError(t, err)
3126 }
3127
3128 getBookMarkFn, err := cacher.getBookmarkAfterResourceVersionLockedFunc(uint64(parsedResourceVersion), uint64(scenario.requiredResourceVersion), scenario.opts)
3129 require.NoError(t, err)
3130 cacher.watchCache.RLock()
3131 defer cacher.watchCache.RUnlock()
3132 getBookMarkResourceVersion := getBookMarkFn()
3133 require.Equal(t, uint64(scenario.expectedBookmarkResourceVersion), getBookMarkResourceVersion, "received unexpected ResourceVersion")
3134 })
3135 }
3136 }
3137
3138 func TestWatchStreamSeparation(t *testing.T) {
3139 server, etcdStorage := newEtcdTestStorage(t, etcd3testing.PathPrefix())
3140 t.Cleanup(func() {
3141 server.Terminate(t)
3142 })
3143 setupOpts := &setupOptions{}
3144 withDefaults(setupOpts)
3145 config := Config{
3146 Storage: etcdStorage,
3147 Versioner: storage.APIObjectVersioner{},
3148 GroupResource: schema.GroupResource{Resource: "pods"},
3149 EventsHistoryWindow: DefaultEventFreshDuration,
3150 ResourcePrefix: setupOpts.resourcePrefix,
3151 KeyFunc: setupOpts.keyFunc,
3152 GetAttrsFunc: GetPodAttrs,
3153 NewFunc: newPod,
3154 NewListFunc: newPodList,
3155 IndexerFuncs: setupOpts.indexerFuncs,
3156 Codec: codecs.LegacyCodec(examplev1.SchemeGroupVersion),
3157 Clock: setupOpts.clock,
3158 }
3159 tcs := []struct {
3160 name string
3161 separateCacheWatchRPC bool
3162 useWatchCacheContextMetadata bool
3163 expectBookmarkOnWatchCache bool
3164 expectBookmarkOnEtcd bool
3165 }{
3166 {
3167 name: "common RPC > both get bookmarks",
3168 separateCacheWatchRPC: false,
3169 expectBookmarkOnEtcd: true,
3170 expectBookmarkOnWatchCache: true,
3171 },
3172 {
3173 name: "separate RPC > only etcd gets bookmarks",
3174 separateCacheWatchRPC: true,
3175 expectBookmarkOnEtcd: true,
3176 expectBookmarkOnWatchCache: false,
3177 },
3178 {
3179 name: "separate RPC & watch cache context > only watch cache gets bookmarks",
3180 separateCacheWatchRPC: true,
3181 useWatchCacheContextMetadata: true,
3182 expectBookmarkOnEtcd: false,
3183 expectBookmarkOnWatchCache: true,
3184 },
3185 }
3186 for _, tc := range tcs {
3187 t.Run(tc.name, func(t *testing.T) {
3188 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SeparateCacheWatchRPC, tc.separateCacheWatchRPC)
3189 cacher, err := NewCacherFromConfig(config)
3190 if err != nil {
3191 t.Fatalf("Failed to initialize cacher: %v", err)
3192 }
3193 defer cacher.Stop()
3194
3195 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
3196 if err := cacher.ready.wait(context.TODO()); err != nil {
3197 t.Fatalf("unexpected error waiting for the cache to be ready")
3198 }
3199 }
3200
3201 getCacherRV := func() uint64 {
3202 cacher.watchCache.RLock()
3203 defer cacher.watchCache.RUnlock()
3204 return cacher.watchCache.resourceVersion
3205 }
3206 waitContext, cancel := context.WithTimeout(context.Background(), 2*time.Second)
3207 defer cancel()
3208 waitForEtcdBookmark := watchAndWaitForBookmark(t, waitContext, cacher.storage)
3209
3210 var out example.Pod
3211 err = cacher.storage.Create(context.Background(), "foo", &example.Pod{}, &out, 0)
3212 if err != nil {
3213 t.Fatal(err)
3214 }
3215 err = cacher.storage.Delete(context.Background(), "foo", &out, nil, storage.ValidateAllObjectFunc, &example.Pod{}, storage.DeleteOptions{})
3216 if err != nil {
3217 t.Fatal(err)
3218 }
3219 versioner := storage.APIObjectVersioner{}
3220 var lastResourceVersion uint64
3221 lastResourceVersion, err = versioner.ObjectResourceVersion(&out)
3222 if err != nil {
3223 t.Fatal(err)
3224 }
3225
3226 var contextMetadata metadata.MD
3227 if tc.useWatchCacheContextMetadata {
3228 contextMetadata = metadata.New(map[string]string{"source": "cache"})
3229 }
3230 // For the first 100ms from watch creation, watch progress requests are ignored.
3231 time.Sleep(200 * time.Millisecond)
3232 err = cacher.storage.RequestWatchProgress(metadata.NewOutgoingContext(context.Background(), contextMetadata))
3233 if err != nil {
3234 t.Fatal(err)
3235 }
3236 // Give time for bookmark to arrive
3237 time.Sleep(time.Second)
3238
3239 etcdWatchResourceVersion := waitForEtcdBookmark()
3240 gotEtcdWatchBookmark := etcdWatchResourceVersion == lastResourceVersion
3241 if gotEtcdWatchBookmark != tc.expectBookmarkOnEtcd {
3242 t.Errorf("Unexpected etcd bookmark check result, rv: %d, lastRV: %d, wantMatching: %v", etcdWatchResourceVersion, lastResourceVersion, tc.expectBookmarkOnEtcd)
3243 }
3244
3245 watchCacheResourceVersion := getCacherRV()
3246 cacherGotBookmark := watchCacheResourceVersion == lastResourceVersion
3247 if cacherGotBookmark != tc.expectBookmarkOnWatchCache {
3248 t.Errorf("Unexpected watch cache bookmark check result, rv: %d, lastRV: %d, wantMatching: %v", watchCacheResourceVersion, lastResourceVersion, tc.expectBookmarkOnWatchCache)
3249 }
3250 })
3251 }
3252 }
3253
3254 func TestComputeListLimit(t *testing.T) {
3255 scenarios := []struct {
3256 name string
3257 opts storage.ListOptions
3258 expectedLimit int64
3259 }{
3260 {
3261 name: "limit is zero",
3262 opts: storage.ListOptions{
3263 Predicate: storage.SelectionPredicate{
3264 Limit: 0,
3265 },
3266 },
3267 expectedLimit: 0,
3268 },
3269 {
3270 name: "limit is positive, RV is unset",
3271 opts: storage.ListOptions{
3272 Predicate: storage.SelectionPredicate{
3273 Limit: 1,
3274 },
3275 ResourceVersion: "",
3276 },
3277 expectedLimit: 1,
3278 },
3279 {
3280 name: "limit is positive, RV = 100",
3281 opts: storage.ListOptions{
3282 Predicate: storage.SelectionPredicate{
3283 Limit: 1,
3284 },
3285 ResourceVersion: "100",
3286 },
3287 expectedLimit: 1,
3288 },
3289 {
3290 name: "legacy case: limit is positive, RV = 0",
3291 opts: storage.ListOptions{
3292 Predicate: storage.SelectionPredicate{
3293 Limit: 1,
3294 },
3295 ResourceVersion: "0",
3296 },
3297 expectedLimit: 0,
3298 },
3299 }
3300
3301 for _, scenario := range scenarios {
3302 t.Run(scenario.name, func(t *testing.T) {
3303 actualLimit := computeListLimit(scenario.opts)
3304 if actualLimit != scenario.expectedLimit {
3305 t.Errorf("computeListLimit returned = %v, expected %v", actualLimit, scenario.expectedLimit)
3306 }
3307 })
3308 }
3309 }
3310
3311 func watchAndWaitForBookmark(t *testing.T, ctx context.Context, etcdStorage storage.Interface) func() (resourceVersion uint64) {
3312 opts := storage.ListOptions{ResourceVersion: "", Predicate: storage.Everything, Recursive: true}
3313 opts.Predicate.AllowWatchBookmarks = true
3314 w, err := etcdStorage.Watch(ctx, "/pods/", opts)
3315 if err != nil {
3316 t.Fatal(err)
3317 }
3318
3319 versioner := storage.APIObjectVersioner{}
3320 var rv uint64
3321 var wg sync.WaitGroup
3322 wg.Add(1)
3323 go func() {
3324 defer wg.Done()
3325 for event := range w.ResultChan() {
3326 if event.Type == watch.Bookmark {
3327 rv, err = versioner.ObjectResourceVersion(event.Object)
3328 break
3329 }
3330 }
3331 }()
3332 return func() (resourceVersion uint64) {
3333 defer w.Stop()
3334 wg.Wait()
3335 if err != nil {
3336 t.Fatal(err)
3337 }
3338 return rv
3339 }
3340 }
3341
3342 // TODO(p0lyn0mial): forceRequestWatchProgressSupport inits the storage layer
3343 // so that tests that require storage.RequestWatchProgress pass
3344 //
3345 // In the future we could have a function that would allow for setting the feature
3346 // only for duration of a test.
3347 func forceRequestWatchProgressSupport(t *testing.T) {
3348 if etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress) {
3349 return
3350 }
3351
3352 server, _ := newEtcdTestStorage(t, etcd3testing.PathPrefix())
3353 defer server.Terminate(t)
3354 if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, wait.ForeverTestTimeout, true, func(_ context.Context) (bool, error) {
3355 return etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress), nil
3356 }); err != nil {
3357 t.Fatalf("failed to wait for required %v storage feature to initialize", storage.RequestWatchProgress)
3358 }
3359 }
3360
3361 func TestListIndexer(t *testing.T) {
3362 ctx, cacher, terminate := testSetup(t, withNodeNameAndNamespaceIndex)
3363 t.Cleanup(terminate)
3364 tests := []struct {
3365 name string
3366 requestedNamespace string
3367 recursive bool
3368 fieldSelector fields.Selector
3369 indexFields []string
3370 expectIndex string
3371 }{
3372 {
3373 name: "request without namespace, without field selector",
3374 recursive: true,
3375 fieldSelector: fields.Everything(),
3376 },
3377 {
3378 name: "request without namespace, field selector with metadata.namespace",
3379 recursive: true,
3380 fieldSelector: fields.ParseSelectorOrDie("metadata.namespace=namespace"),
3381 },
3382 {
3383 name: "request without namespace, field selector with spec.nodename",
3384 recursive: true,
3385 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName=node"),
3386 indexFields: []string{"spec.nodeName"},
3387 expectIndex: "f:spec.nodeName",
3388 },
3389 {
3390 name: "request without namespace, field selector with spec.nodename to filter out",
3391 recursive: true,
3392 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName!=node"),
3393 indexFields: []string{"spec.nodeName"},
3394 },
3395 {
3396 name: "request with namespace, without field selector",
3397 requestedNamespace: "namespace",
3398 recursive: true,
3399 fieldSelector: fields.Everything(),
3400 },
3401 {
3402 name: "request with namespace, field selector with matched metadata.namespace",
3403 requestedNamespace: "namespace",
3404 recursive: true,
3405 fieldSelector: fields.ParseSelectorOrDie("metadata.namespace=namespace"),
3406 },
3407 {
3408 name: "request with namespace, field selector with non-matched metadata.namespace",
3409 requestedNamespace: "namespace",
3410 recursive: true,
3411 fieldSelector: fields.ParseSelectorOrDie("metadata.namespace=namespace"),
3412 },
3413 {
3414 name: "request with namespace, field selector with spec.nodename",
3415 requestedNamespace: "namespace",
3416 recursive: true,
3417 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName=node"),
3418 indexFields: []string{"spec.nodeName"},
3419 expectIndex: "f:spec.nodeName",
3420 },
3421 {
3422 name: "request with namespace, field selector with spec.nodename to filter out",
3423 requestedNamespace: "namespace",
3424 recursive: true,
3425 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName!=node"),
3426 indexFields: []string{"spec.nodeName"},
3427 },
3428 {
3429 name: "request without namespace, field selector with metadata.name",
3430 recursive: true,
3431 fieldSelector: fields.ParseSelectorOrDie("metadata.name=name"),
3432 },
3433 {
3434 name: "request without namespace, field selector with metadata.name and metadata.namespace",
3435 recursive: true,
3436 fieldSelector: fields.SelectorFromSet(fields.Set{
3437 "metadata.name": "name",
3438 "metadata.namespace": "namespace",
3439 }),
3440 },
3441 {
3442 name: "request without namespace, field selector with metadata.name and spec.nodeName",
3443 recursive: true,
3444 fieldSelector: fields.SelectorFromSet(fields.Set{
3445 "metadata.name": "name",
3446 "spec.nodeName": "node",
3447 }),
3448 indexFields: []string{"spec.nodeName"},
3449 expectIndex: "f:spec.nodeName",
3450 },
3451 {
3452 name: "request without namespace, field selector with metadata.name, and with spec.nodeName to filter out watch",
3453 recursive: true,
3454 fieldSelector: fields.AndSelectors(
3455 fields.ParseSelectorOrDie("spec.nodeName!=node"),
3456 fields.SelectorFromSet(fields.Set{"metadata.name": "name"}),
3457 ),
3458 indexFields: []string{"spec.nodeName"},
3459 },
3460 {
3461 name: "request with namespace, with field selector metadata.name",
3462 requestedNamespace: "namespace",
3463 fieldSelector: fields.ParseSelectorOrDie("metadata.name=name"),
3464 },
3465 {
3466 name: "request with namespace, with field selector metadata.name and metadata.namespace",
3467 requestedNamespace: "namespace",
3468 fieldSelector: fields.SelectorFromSet(fields.Set{
3469 "metadata.name": "name",
3470 "metadata.namespace": "namespace",
3471 }),
3472 },
3473 {
3474 name: "request with namespace, with field selector metadata.name, metadata.namespace and spec.nodename",
3475 requestedNamespace: "namespace",
3476 fieldSelector: fields.SelectorFromSet(fields.Set{
3477 "metadata.name": "name",
3478 "metadata.namespace": "namespace",
3479 "spec.nodeName": "node",
3480 }),
3481 indexFields: []string{"spec.nodeName"},
3482 },
3483 {
3484 name: "request with namespace, with field selector metadata.name, metadata.namespace, and with spec.nodename to filter out",
3485 requestedNamespace: "namespace",
3486 fieldSelector: fields.AndSelectors(
3487 fields.ParseSelectorOrDie("spec.nodeName!=node"),
3488 fields.SelectorFromSet(fields.Set{"metadata.name": "name", "metadata.namespace": "namespace"}),
3489 ),
3490 indexFields: []string{"spec.nodeName"},
3491 },
3492 }
3493 for _, tt := range tests {
3494 t.Run(tt.name, func(t *testing.T) {
3495 pred := storagetesting.CreatePodPredicate(tt.fieldSelector, true, tt.indexFields)
3496 _, usedIndex, err := cacher.cacher.listItems(ctx, 0, "/pods/"+tt.requestedNamespace, storage.ListOptions{Predicate: pred, Recursive: tt.recursive})
3497 if err != nil {
3498 t.Errorf("Unexpected error: %v", err)
3499 }
3500 if usedIndex != tt.expectIndex {
3501 t.Errorf("Index doesn't match, expected: %q, got: %q", tt.expectIndex, usedIndex)
3502 }
3503 })
3504 }
3505 }
3506
3507 func TestRetryAfterForUnreadyCache(t *testing.T) {
3508 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
3509 t.Skipf("the test requires %v to be enabled", features.ResilientWatchCacheInitialization)
3510 }
3511 backingStorage := &dummyStorage{}
3512 clock := testingclock.NewFakeClock(time.Now())
3513 cacher, _, err := newTestCacherWithoutSyncing(backingStorage, clock)
3514 if err != nil {
3515 t.Fatalf("Couldn't create cacher: %v", err)
3516 }
3517 defer cacher.Stop()
3518 if err = cacher.ready.wait(context.Background()); err != nil {
3519 t.Fatalf("Unexpected error waiting for the cache to be ready")
3520 }
3521
3522 cacher.ready.setError(nil)
3523 clock.Step(14 * time.Second)
3524
3525 opts := storage.ListOptions{
3526 ResourceVersion: "0",
3527 Predicate: storage.Everything,
3528 }
3529 result := &example.PodList{}
3530 delegator := NewCacheDelegator(cacher, backingStorage)
3531 defer delegator.Stop()
3532 err = delegator.GetList(context.TODO(), "/pods/ns", opts, result)
3533
3534 if !apierrors.IsTooManyRequests(err) {
3535 t.Fatalf("Unexpected GetList error: %v", err)
3536 }
3537 var statusError apierrors.APIStatus
3538 if !errors.As(err, &statusError) {
3539 t.Fatalf("Unexpected error: %v, expected apierrors.APIStatus", err)
3540 }
3541 if statusError.Status().Details == nil {
3542 t.Fatalf("Expected to get status details, got none")
3543 }
3544 if statusError.Status().Details.RetryAfterSeconds != 2 {
3545 t.Fatalf("Unexpected retry after: %v", statusError.Status().Details.RetryAfterSeconds)
3546 }
3547 }