Debian Codegraph @

pkg/storage/testing/watcher_tests.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package testing
18
19 import (
20 "context"
21 "fmt"
22 "net/http"
23 "strings"
24 "sync"
25 "testing"
26 "time"
27
28 "github.com/stretchr/testify/require"
29
30 apiequality "k8s.io/apimachinery/pkg/api/equality"
31 "k8s.io/apimachinery/pkg/api/errors"
32 "k8s.io/apimachinery/pkg/api/meta"
33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34 "k8s.io/apimachinery/pkg/fields"
35 "k8s.io/apimachinery/pkg/labels"
36 "k8s.io/apimachinery/pkg/runtime"
37 "k8s.io/apimachinery/pkg/util/wait"
38 "k8s.io/apimachinery/pkg/watch"
39 "k8s.io/apiserver/pkg/apis/example"
40 genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
41 "k8s.io/apiserver/pkg/features"
42 "k8s.io/apiserver/pkg/storage"
43 "k8s.io/apiserver/pkg/storage/value"
44 utilfeature "k8s.io/apiserver/pkg/util/feature"
45 utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
46 featuregatetesting "k8s.io/component-base/featuregate/testing"
47 "k8s.io/utils/pointer"
48 )
49
50 func RunTestWatch(ctx context.Context, t *testing.T, store storage.Interface) {
51 testWatch(ctx, t, store, false)
52 testWatch(ctx, t, store, true)
53 }
54
55 // It tests that
56 // - first occurrence of objects should notify Add event
57 // - update should trigger Modified event
58 // - update that gets filtered should trigger Deleted event
59 func testWatch(ctx context.Context, t *testing.T, store storage.Interface, recursive bool) {
60 basePod := &example.Pod{
61 ObjectMeta: metav1.ObjectMeta{Name: "foo"},
62 Spec: example.PodSpec{NodeName: ""},
63 }
64 basePodAssigned := &example.Pod{
65 ObjectMeta: metav1.ObjectMeta{Name: "foo"},
66 Spec: example.PodSpec{NodeName: "bar"},
67 }
68
69 selectedPod := func(pod *example.Pod) *example.Pod {
70 result := pod.DeepCopy()
71 result.Labels = map[string]string{"select": "true"}
72 return result
73 }
74
75 tests := []struct {
76 name string
77 namespace string
78 key string
79 pred storage.SelectionPredicate
80 watchTests []*testWatchStruct
81 }{{
82 name: "create a key",
83 namespace: fmt.Sprintf("test-ns-1-%t", recursive),
84 watchTests: []*testWatchStruct{{basePod, true, watch.Added}},
85 pred: storage.Everything,
86 }, {
87 name: "key updated to match predicate",
88 namespace: fmt.Sprintf("test-ns-2-%t", recursive),
89 watchTests: []*testWatchStruct{{basePod, false, ""}, {basePodAssigned, true, watch.Added}},
90 pred: storage.SelectionPredicate{
91 Label: labels.Everything(),
92 Field: fields.ParseSelectorOrDie("spec.nodeName=bar"),
93 GetAttrs: func(obj runtime.Object) (labels.Set, fields.Set, error) {
94 pod := obj.(*example.Pod)
95 return nil, fields.Set{"spec.nodeName": pod.Spec.NodeName}, nil
96 },
97 },
98 }, {
99 name: "update",
100 namespace: fmt.Sprintf("test-ns-3-%t", recursive),
101 watchTests: []*testWatchStruct{{basePod, true, watch.Added}, {basePodAssigned, true, watch.Modified}},
102 pred: storage.Everything,
103 }, {
104 name: "delete because of being filtered",
105 namespace: fmt.Sprintf("test-ns-4-%t", recursive),
106 watchTests: []*testWatchStruct{{basePod, true, watch.Added}, {basePodAssigned, true, watch.Deleted}},
107 pred: storage.SelectionPredicate{
108 Label: labels.Everything(),
109 Field: fields.ParseSelectorOrDie("spec.nodeName!=bar"),
110 GetAttrs: func(obj runtime.Object) (labels.Set, fields.Set, error) {
111 pod := obj.(*example.Pod)
112 return nil, fields.Set{"spec.nodeName": pod.Spec.NodeName}, nil
113 },
114 },
115 }, {
116 name: "filtering",
117 namespace: fmt.Sprintf("test-ns-5-%t", recursive),
118 watchTests: []*testWatchStruct{
119 {selectedPod(basePod), true, watch.Added},
120 {basePod, true, watch.Deleted},
121 {selectedPod(basePod), true, watch.Added},
122 {selectedPod(basePodAssigned), true, watch.Modified},
123 {nil, true, watch.Deleted},
124 },
125 pred: storage.SelectionPredicate{
126 Label: labels.SelectorFromSet(labels.Set{"select": "true"}),
127 Field: fields.Everything(),
128 GetAttrs: func(obj runtime.Object) (labels.Set, fields.Set, error) {
129 pod := obj.(*example.Pod)
130 return labels.Set(pod.Labels), nil, nil
131 },
132 },
133 }}
134 for _, tt := range tests {
135 t.Run(tt.name, func(t *testing.T) {
136 watchKey := fmt.Sprintf("/pods/%s", tt.namespace)
137 key := watchKey + "/foo"
138 if !recursive {
139 watchKey = key
140 }
141
142 // Get the current RV from which we can start watching.
143 out := &example.PodList{}
144 if err := store.GetList(ctx, watchKey, storage.ListOptions{ResourceVersion: "", Predicate: tt.pred, Recursive: recursive}, out); err != nil {
145 t.Fatalf("List failed: %v", err)
146 }
147
148 w, err := store.Watch(ctx, watchKey, storage.ListOptions{ResourceVersion: out.ResourceVersion, Predicate: tt.pred, Recursive: recursive})
149 if err != nil {
150 t.Fatalf("Watch failed: %v", err)
151 }
152
153 // Create a pod in a different namespace first to ensure
154 // that its corresponding event will not be propagated.
155 badKey := fmt.Sprintf("/pods/%s-bad/foo", tt.namespace)
156 badOut := &example.Pod{}
157 err = store.GuaranteedUpdate(ctx, badKey, badOut, true, nil, storage.SimpleUpdate(
158 func(runtime.Object) (runtime.Object, error) {
159 obj := basePod.DeepCopy()
160 obj.Namespace = fmt.Sprintf("%s-bad", tt.namespace)
161 return obj, nil
162 }), nil)
163 if err != nil {
164 t.Fatalf("GuaranteedUpdate of bad pod failed: %v", err)
165 }
166
167 var prevObj *example.Pod
168 for _, watchTest := range tt.watchTests {
169 out := &example.Pod{}
170 if watchTest.obj != nil {
171 err := store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate(
172 func(runtime.Object) (runtime.Object, error) {
173 obj := watchTest.obj.DeepCopy()
174 obj.Namespace = tt.namespace
175 return obj, nil
176 }), nil)
177 if err != nil {
178 t.Fatalf("GuaranteedUpdate failed: %v", err)
179 }
180 } else {
181 err := store.Delete(ctx, key, out, nil, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{})
182 if err != nil {
183 t.Fatalf("Delete failed: %v", err)
184 }
185 }
186 if watchTest.expectEvent {
187 expectObj := out
188 if watchTest.watchType == watch.Deleted {
189 expectObj = prevObj
190 expectObj.ResourceVersion = out.ResourceVersion
191 }
192 testCheckResult(t, w, watch.Event{Type: watchTest.watchType, Object: expectObj})
193 }
194 prevObj = out
195 }
196 w.Stop()
197 testCheckStop(t, w)
198 })
199 }
200 }
201
202 // RunTestWatchFromZero tests that
203 // - watch from 0 should sync up and grab the object added before
204 // - For testing with etcd, watch from 0 is able to return events for objects
205 // whose previous version has been compacted. If testing with cacher, we
206 // expect compaction to be nil.
207 func RunTestWatchFromZero(ctx context.Context, t *testing.T, store storage.Interface, compaction Compaction) {
208 key, storedObj := testPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}})
209
210 w, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything})
211 if err != nil {
212 t.Fatalf("Watch failed: %v", err)
213 }
214 testCheckResult(t, w, watch.Event{Type: watch.Added, Object: storedObj})
215
216 // Update
217 out := &example.Pod{}
218 err = store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate(
219 func(runtime.Object) (runtime.Object, error) {
220 return &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns", Annotations: map[string]string{"a": "1"}}}, nil
221 }), nil)
222 if err != nil {
223 t.Fatalf("GuaranteedUpdate failed: %v", err)
224 }
225
226 // Check that we receive a modified watch event. This check also
227 // indirectly ensures that the cache is synced. This is important
228 // when testing with the Cacher since we may have to allow for slow
229 // processing by allowing updates to propagate to the watch cache.
230 // This allows for that.
231 testCheckResult(t, w, watch.Event{Type: watch.Modified, Object: out})
232 w.Stop()
233
234 // Make sure when we watch from 0 we receive an ADDED event
235 w, err = store.Watch(ctx, key, storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything})
236 if err != nil {
237 t.Fatalf("Watch failed: %v", err)
238 }
239
240 testCheckResult(t, w, watch.Event{Type: watch.Added, Object: out})
241 w.Stop()
242
243 // Compact previous versions
244 if compaction == nil {
245 t.Skip("compaction callback not provided")
246 }
247
248 // Update again
249 newOut := &example.Pod{}
250 err = store.GuaranteedUpdate(ctx, key, newOut, true, nil, storage.SimpleUpdate(
251 func(runtime.Object) (runtime.Object, error) {
252 return &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}}, nil
253 }), nil)
254 if err != nil {
255 t.Fatalf("GuaranteedUpdate failed: %v", err)
256 }
257
258 // Compact previous versions
259 compaction(ctx, t, newOut.ResourceVersion)
260
261 // Make sure we can still watch from 0 and receive an ADDED event
262 w, err = store.Watch(ctx, key, storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything})
263 defer w.Stop()
264 if err != nil {
265 t.Fatalf("Watch failed: %v", err)
266 }
267 testCheckResult(t, w, watch.Event{Type: watch.Added, Object: newOut})
268
269 // Make sure we can't watch from older resource versions anymoer and get a "Gone" error.
270 tooOldWatcher, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: out.ResourceVersion, Predicate: storage.Everything})
271 if err != nil {
272 t.Fatalf("Watch failed: %v", err)
273 }
274 defer tooOldWatcher.Stop()
275 expiredError := errors.NewResourceExpired("").ErrStatus
276 // TODO(wojtek-t): It seems that etcd is currently returning a different error,
277 // being an Internal error of "etcd event received with PrevKv=nil".
278 // We temporary allow both but we should unify here.
279 internalError := metav1.Status{
280 Status: metav1.StatusFailure,
281 Code: http.StatusInternalServerError,
282 Reason: metav1.StatusReasonInternalError,
283 }
284 testCheckResultFunc(t, tooOldWatcher, func(actualEvent watch.Event) {
285 expectNoDiff(t, "incorrect event type", watch.Error, actualEvent.Type)
286 if !apiequality.Semantic.DeepDerivative(&expiredError, actualEvent.Object) && !apiequality.Semantic.DeepDerivative(&internalError, actualEvent.Object) {
287 t.Errorf("expected: %#v; got %#v", &expiredError, actualEvent.Object)
288 }
289 })
290 }
291
292 func RunTestDeleteTriggerWatch(ctx context.Context, t *testing.T, store storage.Interface) {
293 key, storedObj := testPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}})
294 w, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: storedObj.ResourceVersion, Predicate: storage.Everything})
295 if err != nil {
296 t.Fatalf("Watch failed: %v", err)
297 }
298 if err := store.Delete(ctx, key, &example.Pod{}, nil, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}); err != nil {
299 t.Fatalf("Delete failed: %v", err)
300 }
301 testCheckEventType(t, w, watch.Deleted)
302 }
303
304 func RunTestWatchFromNonZero(ctx context.Context, t *testing.T, store storage.Interface) {
305 key, storedObj := testPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}})
306
307 w, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: storedObj.ResourceVersion, Predicate: storage.Everything})
308 if err != nil {
309 t.Fatalf("Watch failed: %v", err)
310 }
311 out := &example.Pod{}
312 store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate(
313 func(runtime.Object) (runtime.Object, error) {
314 newObj := storedObj.DeepCopy()
315 newObj.Annotations = map[string]string{"version": "2"}
316 return newObj, nil
317 }), nil)
318 testCheckResult(t, w, watch.Event{Type: watch.Modified, Object: out})
319 }
320
321 func RunTestDelayedWatchDelivery(ctx context.Context, t *testing.T, store storage.Interface) {
322 _, storedObj := testPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}})
323 startRV := storedObj.ResourceVersion
324
325 watcher, err := store.Watch(ctx, "/pods/test-ns", storage.ListOptions{ResourceVersion: startRV, Predicate: storage.Everything, Recursive: true})
326 if err != nil {
327 t.Fatalf("Unexpected error: %v", err)
328 }
329
330 // Depending on the implementation, different number of events that
331 // should be delivered to the watcher can be created before it will
332 // block the implementation and as a result force the watcher to be
333 // closed (as otherwise events would have to be dropped).
334 // For now, this number is smallest for Cacher and it equals 21 for it.
335 totalPods := 21
336 for i := 0; i < totalPods; i++ {
337 out := &example.Pod{}
338 pod := &example.Pod{
339 ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("foo-%d", i), Namespace: "test-ns"},
340 }
341 err := store.GuaranteedUpdate(ctx, computePodKey(pod), out, true, nil, storage.SimpleUpdate(
342 func(runtime.Object) (runtime.Object, error) {
343 return pod, nil
344 }), nil)
345 if err != nil {
346 t.Errorf("GuaranteedUpdate failed: %v", err)
347 }
348 }
349
350 // Now stop the watcher and check if the consecutive events are being delivered.
351 watcher.Stop()
352
353 watched := 0
354 for {
355 event, ok := <-watcher.ResultChan()
356 if !ok {
357 break
358 }
359 object := event.Object
360 if co, ok := object.(runtime.CacheableObject); ok {
361 object = co.GetObject()
362 }
363 if a, e := object.(*example.Pod).Name, fmt.Sprintf("foo-%d", watched); e != a {
364 t.Errorf("Unexpected object watched: %s, expected %s", a, e)
365 }
366 watched++
367 }
368 // We expect at least N events to be delivered, depending on the implementation.
369 // For now, this number is smallest for Cacher and it equals 10 (size of the out buffer).
370 if watched < 10 {
371 t.Errorf("Unexpected number of events: %v, expected: %v", watched, totalPods)
372 }
373 }
374
375 func RunTestWatchError(ctx context.Context, t *testing.T, store InterfaceWithPrefixTransformer) {
376 obj := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}}
377 key := computePodKey(obj)
378
379 // Compute the initial resource version from which we can start watching later.
380 list := &example.PodList{}
381 storageOpts := storage.ListOptions{
382 ResourceVersion: "0",
383 Predicate: storage.Everything,
384 Recursive: true,
385 }
386 if err := store.GetList(ctx, "/pods", storageOpts, list); err != nil {
387 t.Errorf("Unexpected error: %v", err)
388 }
389
390 if err := store.GuaranteedUpdate(ctx, key, &example.Pod{}, true, nil, storage.SimpleUpdate(
391 func(runtime.Object) (runtime.Object, error) {
392 return obj, nil
393 }), nil); err != nil {
394 t.Fatalf("GuaranteedUpdate failed: %v", err)
395 }
396
397 // Now trigger watch error by injecting failing transformer.
398 revertTransformer := store.UpdatePrefixTransformer(
399 func(previousTransformer *PrefixTransformer) value.Transformer {
400 return &failingTransformer{}
401 })
402 defer revertTransformer()
403
404 w, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: list.ResourceVersion, Predicate: storage.Everything})
405 if err != nil {
406 t.Fatalf("Watch failed: %v", err)
407 }
408 testCheckEventType(t, w, watch.Error)
409 }
410
411 func RunTestWatchWithUnsafeDelete(ctx context.Context, t *testing.T, store InterfaceWithCorruptTransformer) {
412 obj := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}}
413 key := computePodKey(obj)
414
415 out := &example.Pod{}
416 if err := store.Create(ctx, key, obj, out, 0); err != nil {
417 t.Fatalf("failed to create object in the store: %v", err)
418 }
419
420 // Compute the initial resource version from which we can start watching later.
421 list := &example.PodList{}
422 storageOpts := storage.ListOptions{
423 ResourceVersion: "0",
424 Predicate: storage.Everything,
425 Recursive: true,
426 }
427 if err := store.GetList(ctx, "/pods", storageOpts, list); err != nil {
428 t.Errorf("Unexpected error: %v", err)
429 }
430
431 // Now trigger watch error by injecting failing transformer.
432 revertTransformer := store.CorruptTransformer()
433 defer revertTransformer()
434
435 w, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: list.ResourceVersion, Predicate: storage.Everything})
436 if err != nil {
437 t.Fatalf("Watch failed: %v", err)
438 }
439
440 // normal deletetion should fail
441 if err := store.Delete(ctx, key, &example.Pod{}, nil, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}); err == nil {
442 t.Fatalf("Expected normal Delete to fail")
443 }
444 if err := store.Delete(ctx, key, &example.Pod{}, nil, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{IgnoreStoreReadError: true}); err != nil {
445 t.Fatalf("Expected unsafe Delete to succeed, but got: %v", err)
446 }
447
448 testCheckResultFunc(t, w, func(got watch.Event) {
449 if want, got := watch.Error, got.Type; want != got {
450 t.Errorf("Expected event type: %q, but got: %q", want, got)
451 }
452 switch v := got.Object.(type) {
453 case *metav1.Status:
454 if want, got := metav1.StatusReasonStoreReadError, v.Reason; want != got {
455 t.Errorf("Expected reason: %q, but got: %q", want, got)
456 }
457 if want := "saw a DELETED event, but object data is corrupt"; !strings.Contains(v.Message, want) {
458 t.Errorf("Expected Message to contain: %q, but got: %q", want, v.Message)
459 }
460 default:
461 t.Errorf("expected an metav1 Status object, but got: %v", got.Object)
462 }
463 })
464 }
465
466 func RunTestWatchContextCancel(ctx context.Context, t *testing.T, store storage.Interface) {
467 canceledCtx, cancel := context.WithCancel(ctx)
468 cancel()
469 // When we watch with a canceled context, we should detect that it's context canceled.
470 // We won't take it as error and also close the watcher.
471 w, err := store.Watch(canceledCtx, "/pods/not-existing", storage.ListOptions{
472 ResourceVersion: "0",
473 Predicate: storage.Everything,
474 })
475 if err != nil {
476 t.Fatal(err)
477 }
478
479 select {
480 case _, ok := <-w.ResultChan():
481 if ok {
482 t.Error("ResultChan() should be closed")
483 }
484 case <-time.After(wait.ForeverTestTimeout):
485 t.Errorf("timeout after %v", wait.ForeverTestTimeout)
486 }
487 }
488
489 func RunTestWatcherTimeout(ctx context.Context, t *testing.T, store storage.Interface) {
490 // initialRV is used to initate the watcher at the beginning of the world.
491 podList := example.PodList{}
492 options := storage.ListOptions{
493 Predicate: storage.Everything,
494 Recursive: true,
495 }
496 if err := store.GetList(ctx, "/pods", options, &podList); err != nil {
497 t.Fatalf("Failed to list pods: %v", err)
498 }
499 initialRV := podList.ResourceVersion
500
501 options = storage.ListOptions{
502 ResourceVersion: initialRV,
503 Predicate: storage.Everything,
504 Recursive: true,
505 }
506
507 // Create a number of watchers that will not be reading any result.
508 nonReadingWatchers := 50
509 for i := 0; i < nonReadingWatchers; i++ {
510 watcher, err := store.Watch(ctx, "/pods/test-ns", options)
511 if err != nil {
512 t.Fatalf("Unexpected error: %v", err)
513 }
514 defer watcher.Stop()
515 }
516
517 // Create a second watcher that will be reading result.
518 readingWatcher, err := store.Watch(ctx, "/pods/test-ns", options)
519 if err != nil {
520 t.Fatalf("Unexpected error: %v", err)
521 }
522 defer readingWatcher.Stop()
523
524 // Depending on the implementation, different number of events that
525 // should be delivered to the watcher can be created before it will
526 // block the implementation and as a result force the watcher to be
527 // closed (as otherwise events would have to be dropped).
528 // For now, this number is smallest for Cacher and it equals 21 for it.
529 //
530 // Create more events to ensure that we're not blocking other watchers
531 // forever.
532 startTime := time.Now()
533 for i := 0; i < 22; i++ {
534 out := &example.Pod{}
535 pod := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("foo-%d", i), Namespace: "test-ns"}}
536 if err := store.Create(ctx, computePodKey(pod), pod, out, 0); err != nil {
537 t.Fatalf("Create failed: %v", err)
538 }
539 testCheckResult(t, readingWatcher, watch.Event{Type: watch.Added, Object: out})
540 }
541 if time.Since(startTime) > time.Duration(250*nonReadingWatchers)*time.Millisecond {
542 t.Errorf("waiting for events took too long: %v", time.Since(startTime))
543 }
544 }
545
546 func RunTestWatchDeleteEventObjectHaveLatestRV(ctx context.Context, t *testing.T, store storage.Interface) {
547 key, storedObj := testPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}})
548
549 watchCtx, cancel := context.WithTimeout(ctx, wait.ForeverTestTimeout)
550 t.Cleanup(cancel)
551 w, err := store.Watch(watchCtx, key, storage.ListOptions{ResourceVersion: storedObj.ResourceVersion, Predicate: storage.Everything})
552 if err != nil {
553 t.Fatalf("Watch failed: %v", err)
554 }
555
556 deletedObj := &example.Pod{}
557 if err := store.Delete(ctx, key, deletedObj, &storage.Preconditions{}, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}); err != nil {
558 t.Fatalf("Delete failed: %v", err)
559 }
560
561 // Verify that ResourceVersion has changed on deletion.
562 if storedObj.ResourceVersion == deletedObj.ResourceVersion {
563 t.Fatalf("ResourceVersion didn't changed on deletion: %s", deletedObj.ResourceVersion)
564 }
565
566 testCheckResult(t, w, watch.Event{Type: watch.Deleted, Object: deletedObj})
567 }
568
569 func RunTestWatchInitializationSignal(ctx context.Context, t *testing.T, store storage.Interface) {
570 ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
571 t.Cleanup(cancel)
572 initSignal := utilflowcontrol.NewInitializationSignal()
573 ctx = utilflowcontrol.WithInitializationSignal(ctx, initSignal)
574
575 key, storedObj := testPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}})
576 _, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: storedObj.ResourceVersion, Predicate: storage.Everything})
577 if err != nil {
578 t.Fatalf("Watch failed: %v", err)
579 }
580
581 initSignal.Wait()
582 }
583
584 // RunOptionalTestProgressNotify tests ProgressNotify feature of ListOptions.
585 // Given this feature is currently not explicitly used by higher layers of Kubernetes
586 // (it rather is used by wrappers of storage.Interface to implement its functionalities)
587 // this test is currently considered optional.
588 func RunOptionalTestProgressNotify(ctx context.Context, t *testing.T, store storage.Interface) {
589 input := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "name", Namespace: "test-ns"}}
590 key := computePodKey(input)
591 out := &example.Pod{}
592 if err := store.Create(ctx, key, input, out, 0); err != nil {
593 t.Fatalf("Create failed: %v", err)
594 }
595 validateResourceVersion := resourceVersionNotOlderThan(out.ResourceVersion)
596
597 opts := storage.ListOptions{
598 ResourceVersion: out.ResourceVersion,
599 Predicate: storage.Everything,
600 ProgressNotify: true,
601 }
602 w, err := store.Watch(ctx, key, opts)
603 if err != nil {
604 t.Fatalf("Watch failed: %v", err)
605 }
606
607 // when we send a bookmark event, the client expects the event to contain an
608 // object of the correct type, but with no fields set other than the resourceVersion
609 testCheckResultFunc(t, w, func(actualEvent watch.Event) {
610 expectNoDiff(t, "incorrect event type", watch.Bookmark, actualEvent.Type)
611 // first, check that we have the correct resource version
612 obj, ok := actualEvent.Object.(metav1.Object)
613 if !ok {
614 t.Fatalf("got %T, not metav1.Object", actualEvent.Object)
615 }
616 if err := validateResourceVersion(obj.GetResourceVersion()); err != nil {
617 t.Fatal(err)
618 }
619
620 // then, check that we have the right type and content
621 pod, ok := actualEvent.Object.(*example.Pod)
622 if !ok {
623 t.Fatalf("got %T, not *example.Pod", actualEvent.Object)
624 }
625 pod.ResourceVersion = ""
626 expectNoDiff(t, "bookmark event should contain an object with no fields set other than resourceVersion", &example.Pod{}, pod)
627 })
628 }
629
630 // It tests watches of cluster-scoped resources.
631 func RunTestClusterScopedWatch(ctx context.Context, t *testing.T, store storage.Interface) {
632 tests := []struct {
633 name string
634 // For watch request, the name of object is specified with field selector
635 // "metadata.name=objectName". So in this watch tests, we should set the
636 // requestedName and field selector "metadata.name=requestedName" at the
637 // same time or set neighter of them.
638 requestedName string
639 recursive bool
640 fieldSelector fields.Selector
641 indexFields []string
642 watchTests []*testWatchStruct
643 }{
644 {
645 name: "cluster-wide watch, request without name, without field selector",
646 recursive: true,
647 fieldSelector: fields.Everything(),
648 watchTests: []*testWatchStruct{
649 {basePod("t1-foo1"), true, watch.Added},
650 {basePodUpdated("t1-foo1"), true, watch.Modified},
651 {basePodAssigned("t1-foo2", "t1-bar1"), true, watch.Added},
652 },
653 },
654 {
655 name: "cluster-wide watch, request without name, field selector with spec.nodeName",
656 recursive: true,
657 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName=t2-bar1"),
658 indexFields: []string{"spec.nodeName"},
659 watchTests: []*testWatchStruct{
660 {basePod("t2-foo1"), false, ""},
661 {basePodAssigned("t2-foo1", "t2-bar1"), true, watch.Added},
662 },
663 },
664 {
665 name: "cluster-wide watch, request without name, field selector with spec.nodeName to filter out watch",
666 recursive: true,
667 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName!=t3-bar1"),
668 indexFields: []string{"spec.nodeName"},
669 watchTests: []*testWatchStruct{
670 {basePod("t3-foo1"), true, watch.Added},
671 {basePod("t3-foo2"), true, watch.Added},
672 {basePodUpdated("t3-foo1"), true, watch.Modified},
673 {basePodAssigned("t3-foo1", "t3-bar1"), true, watch.Deleted},
674 },
675 },
676 {
677 name: "cluster-wide watch, request with name, field selector with metadata.name",
678 requestedName: "t4-foo1",
679 fieldSelector: fields.ParseSelectorOrDie("metadata.name=t4-foo1"),
680 watchTests: []*testWatchStruct{
681 {basePod("t4-foo1"), true, watch.Added},
682 {basePod("t4-foo2"), false, ""},
683 {basePodUpdated("t4-foo1"), true, watch.Modified},
684 {basePodUpdated("t4-foo2"), false, ""},
685 },
686 },
687 {
688 name: "cluster-wide watch, request with name, field selector with metadata.name and spec.nodeName",
689 requestedName: "t5-foo1",
690 fieldSelector: fields.SelectorFromSet(fields.Set{
691 "metadata.name": "t5-foo1",
692 "spec.nodeName": "t5-bar1",
693 }),
694 indexFields: []string{"spec.nodeName"},
695 watchTests: []*testWatchStruct{
696 {basePod("t5-foo1"), false, ""},
697 {basePod("t5-foo2"), false, ""},
698 {basePodUpdated("t5-foo1"), false, ""},
699 {basePodUpdated("t5-foo2"), false, ""},
700 {basePodAssigned("t5-foo1", "t5-bar1"), true, watch.Added},
701 },
702 },
703 {
704 name: "cluster-wide watch, request with name, field selector with metadata.name, and with spec.nodeName to filter out watch",
705 requestedName: "t6-foo1",
706 fieldSelector: fields.AndSelectors(
707 fields.ParseSelectorOrDie("spec.nodeName!=t6-bar1"),
708 fields.SelectorFromSet(fields.Set{"metadata.name": "t6-foo1"}),
709 ),
710 indexFields: []string{"spec.nodeName"},
711 watchTests: []*testWatchStruct{
712 {basePod("t6-foo1"), true, watch.Added},
713 {basePod("t6-foo2"), false, ""},
714 {basePodUpdated("t6-foo1"), true, watch.Modified},
715 {basePodAssigned("t6-foo1", "t6-bar1"), true, watch.Deleted},
716 {basePodAssigned("t6-foo2", "t6-bar1"), false, ""},
717 },
718 },
719 }
720 for _, tt := range tests {
721 t.Run(tt.name, func(t *testing.T) {
722 requestInfo := &genericapirequest.RequestInfo{}
723 requestInfo.Name = tt.requestedName
724 requestInfo.Namespace = ""
725 ctx = genericapirequest.WithRequestInfo(ctx, requestInfo)
726 ctx = genericapirequest.WithNamespace(ctx, "")
727
728 watchKey := "/pods"
729 if tt.requestedName != "" {
730 watchKey += "/" + tt.requestedName
731 }
732
733 predicate := CreatePodPredicate(tt.fieldSelector, false, tt.indexFields)
734
735 list := &example.PodList{}
736 opts := storage.ListOptions{
737 ResourceVersion: "",
738 Predicate: predicate,
739 Recursive: true,
740 }
741 if err := store.GetList(ctx, "/pods", opts, list); err != nil {
742 t.Errorf("Unexpected error: %v", err)
743 }
744
745 opts.ResourceVersion = list.ResourceVersion
746 opts.Recursive = tt.recursive
747
748 w, err := store.Watch(ctx, watchKey, opts)
749 if err != nil {
750 t.Fatalf("Watch failed: %v", err)
751 }
752
753 currentObjs := map[string]*example.Pod{}
754 for _, watchTest := range tt.watchTests {
755 out := &example.Pod{}
756 key := "pods/" + watchTest.obj.Name
757 err := store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate(
758 func(runtime.Object) (runtime.Object, error) {
759 obj := watchTest.obj.DeepCopy()
760 return obj, nil
761 }), nil)
762 if err != nil {
763 t.Fatalf("GuaranteedUpdate failed: %v", err)
764 }
765
766 expectObj := out
767 if watchTest.watchType == watch.Deleted {
768 expectObj = currentObjs[watchTest.obj.Name]
769 expectObj.ResourceVersion = out.ResourceVersion
770 delete(currentObjs, watchTest.obj.Name)
771 } else {
772 currentObjs[watchTest.obj.Name] = out
773 }
774 if watchTest.expectEvent {
775 testCheckResult(t, w, watch.Event{Type: watchTest.watchType, Object: expectObj})
776 }
777 }
778 w.Stop()
779 testCheckStop(t, w)
780 })
781 }
782 }
783
784 // It tests watch of namespace-scoped resources.
785 func RunTestNamespaceScopedWatch(ctx context.Context, t *testing.T, store storage.Interface) {
786 tests := []struct {
787 name string
788 // For watch request, the name of object is specified with field selector
789 // "metadata.name=objectName". So in this watch tests, we should set the
790 // requestedName and field selector "metadata.name=requestedName" at the
791 // same time or set neighter of them.
792 requestedName string
793 requestedNamespace string
794 recursive bool
795 fieldSelector fields.Selector
796 indexFields []string
797 watchTests []*testWatchStruct
798 }{
799 {
800 name: "namespaced watch, request without name, request without namespace, without field selector",
801 recursive: true,
802 fieldSelector: fields.Everything(),
803 watchTests: []*testWatchStruct{
804 {baseNamespacedPod("t1-foo1", "t1-ns1"), true, watch.Added},
805 {baseNamespacedPod("t1-foo2", "t1-ns2"), true, watch.Added},
806 {baseNamespacedPodUpdated("t1-foo1", "t1-ns1"), true, watch.Modified},
807 {baseNamespacedPodUpdated("t1-foo2", "t1-ns2"), true, watch.Modified},
808 },
809 },
810 {
811 name: "namespaced watch, request without name, request without namespace, field selector with metadata.namespace",
812 recursive: true,
813 fieldSelector: fields.ParseSelectorOrDie("metadata.namespace=t2-ns1"),
814 watchTests: []*testWatchStruct{
815 {baseNamespacedPod("t2-foo1", "t2-ns1"), true, watch.Added},
816 {baseNamespacedPod("t2-foo1", "t2-ns2"), false, ""},
817 {baseNamespacedPodUpdated("t2-foo1", "t2-ns1"), true, watch.Modified},
818 {baseNamespacedPodUpdated("t2-foo1", "t2-ns2"), false, ""},
819 },
820 },
821 {
822 name: "namespaced watch, request without name, request without namespace, field selector with spec.nodename",
823 recursive: true,
824 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName=t3-bar1"),
825 indexFields: []string{"spec.nodeName"},
826 watchTests: []*testWatchStruct{
827 {baseNamespacedPod("t3-foo1", "t3-ns1"), false, ""},
828 {baseNamespacedPod("t3-foo2", "t3-ns2"), false, ""},
829 {baseNamespacedPodAssigned("t3-foo1", "t3-ns1", "t3-bar1"), true, watch.Added},
830 {baseNamespacedPodAssigned("t3-foo2", "t3-ns2", "t3-bar1"), true, watch.Added},
831 },
832 },
833 {
834 name: "namespaced watch, request without name, request without namespace, field selector with spec.nodename to filter out watch",
835 recursive: true,
836 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName!=t4-bar1"),
837 indexFields: []string{"spec.nodeName"},
838 watchTests: []*testWatchStruct{
839 {baseNamespacedPod("t4-foo1", "t4-ns1"), true, watch.Added},
840 {baseNamespacedPod("t4-foo2", "t4-ns1"), true, watch.Added},
841 {baseNamespacedPodUpdated("t4-foo1", "t4-ns1"), true, watch.Modified},
842 {baseNamespacedPodAssigned("t4-foo1", "t4-ns1", "t4-bar1"), true, watch.Deleted},
843 },
844 },
845 {
846 name: "namespaced watch, request without name, request with namespace, without field selector",
847 requestedNamespace: "t5-ns1",
848 recursive: true,
849 fieldSelector: fields.Everything(),
850 watchTests: []*testWatchStruct{
851 {baseNamespacedPod("t5-foo1", "t5-ns1"), true, watch.Added},
852 {baseNamespacedPod("t5-foo1", "t5-ns2"), false, ""},
853 {baseNamespacedPod("t5-foo2", "t5-ns1"), true, watch.Added},
854 {baseNamespacedPodUpdated("t5-foo1", "t5-ns1"), true, watch.Modified},
855 {baseNamespacedPodUpdated("t5-foo1", "t5-ns2"), false, ""},
856 },
857 },
858 {
859 name: "namespaced watch, request without name, request with namespace, field selector with matched metadata.namespace",
860 requestedNamespace: "t6-ns1",
861 recursive: true,
862 fieldSelector: fields.ParseSelectorOrDie("metadata.namespace=t6-ns1"),
863 watchTests: []*testWatchStruct{
864 {baseNamespacedPod("t6-foo1", "t6-ns1"), true, watch.Added},
865 {baseNamespacedPod("t6-foo1", "t6-ns2"), false, ""},
866 {baseNamespacedPodUpdated("t6-foo1", "t6-ns1"), true, watch.Modified},
867 },
868 },
869 {
870 name: "namespaced watch, request without name, request with namespace, field selector with non-matched metadata.namespace",
871 requestedNamespace: "t7-ns1",
872 recursive: true,
873 fieldSelector: fields.ParseSelectorOrDie("metadata.namespace=t7-ns2"),
874 watchTests: []*testWatchStruct{
875 {baseNamespacedPod("t7-foo1", "t7-ns1"), false, ""},
876 {baseNamespacedPod("t7-foo1", "t7-ns2"), false, ""},
877 {baseNamespacedPodUpdated("t7-foo1", "t7-ns1"), false, ""},
878 {baseNamespacedPodUpdated("t7-foo1", "t7-ns2"), false, ""},
879 },
880 },
881 {
882 name: "namespaced watch, request without name, request with namespace, field selector with spec.nodename",
883 requestedNamespace: "t8-ns1",
884 recursive: true,
885 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName=t8-bar2"),
886 indexFields: []string{"spec.nodeName"},
887 watchTests: []*testWatchStruct{
888 {baseNamespacedPod("t8-foo1", "t8-ns1"), false, ""},
889 {baseNamespacedPodAssigned("t8-foo1", "t8-ns1", "t8-bar1"), false, ""},
890 {baseNamespacedPodAssigned("t8-foo1", "t8-ns2", "t8-bar2"), false, ""},
891 {baseNamespacedPodAssigned("t8-foo1", "t8-ns1", "t8-bar2"), true, watch.Added},
892 },
893 },
894 {
895 name: "namespaced watch, request without name, request with namespace, field selector with spec.nodename to filter out watch",
896 requestedNamespace: "t9-ns2",
897 recursive: true,
898 fieldSelector: fields.ParseSelectorOrDie("spec.nodeName!=t9-bar1"),
899 indexFields: []string{"spec.nodeName"},
900 watchTests: []*testWatchStruct{
901 {baseNamespacedPod("t9-foo1", "t9-ns1"), false, ""},
902 {baseNamespacedPod("t9-foo1", "t9-ns2"), true, watch.Added},
903 {baseNamespacedPodAssigned("t9-foo1", "t9-ns2", "t9-bar1"), true, watch.Deleted},
904 {baseNamespacedPodAssigned("t9-foo1", "t9-ns2", "t9-bar2"), true, watch.Added},
905 },
906 },
907 {
908 name: "namespaced watch, request with name, request without namespace, field selector with metadata.name",
909 requestedName: "t10-foo1",
910 recursive: true,
911 fieldSelector: fields.ParseSelectorOrDie("metadata.name=t10-foo1"),
912 watchTests: []*testWatchStruct{
913 {baseNamespacedPod("t10-foo1", "t10-ns1"), true, watch.Added},
914 {baseNamespacedPod("t10-foo1", "t10-ns2"), true, watch.Added},
915 {baseNamespacedPod("t10-foo2", "t10-ns1"), false, ""},
916 {baseNamespacedPodUpdated("t10-foo1", "t10-ns1"), true, watch.Modified},
917 {baseNamespacedPodAssigned("t10-foo1", "t10-ns1", "t10-bar1"), true, watch.Modified},
918 },
919 },
920 {
921 name: "namespaced watch, request with name, request without namespace, field selector with metadata.name and metadata.namespace",
922 requestedName: "t11-foo1",
923 recursive: true,
924 fieldSelector: fields.SelectorFromSet(fields.Set{
925 "metadata.name": "t11-foo1",
926 "metadata.namespace": "t11-ns1",
927 }),
928 watchTests: []*testWatchStruct{
929 {baseNamespacedPod("t11-foo1", "t11-ns1"), true, watch.Added},
930 {baseNamespacedPod("t11-foo2", "t11-ns1"), false, ""},
931 {baseNamespacedPod("t11-foo1", "t11-ns2"), false, ""},
932 {baseNamespacedPodUpdated("t11-foo1", "t11-ns1"), true, watch.Modified},
933 {baseNamespacedPodAssigned("t11-foo1", "t11-ns1", "t11-bar1"), true, watch.Modified},
934 },
935 },
936 {
937 name: "namespaced watch, request with name, request without namespace, field selector with metadata.name and spec.nodeName",
938 requestedName: "t12-foo1",
939 recursive: true,
940 fieldSelector: fields.SelectorFromSet(fields.Set{
941 "metadata.name": "t12-foo1",
942 "spec.nodeName": "t12-bar1",
943 }),
944 indexFields: []string{"spec.nodeName"},
945 watchTests: []*testWatchStruct{
946 {baseNamespacedPod("t12-foo1", "t12-ns1"), false, ""},
947 {baseNamespacedPodUpdated("t12-foo1", "t12-ns1"), false, ""},
948 {baseNamespacedPodAssigned("t12-foo1", "t12-ns1", "t12-bar1"), true, watch.Added},
949 },
950 },
951 {
952 name: "namespaced watch, request with name, request without namespace, field selector with metadata.name, and with spec.nodeName to filter out watch",
953 requestedName: "t15-foo1",
954 recursive: true,
955 fieldSelector: fields.AndSelectors(
956 fields.ParseSelectorOrDie("spec.nodeName!=t15-bar1"),
957 fields.SelectorFromSet(fields.Set{"metadata.name": "t15-foo1"}),
958 ),
959 indexFields: []string{"spec.nodeName"},
960 watchTests: []*testWatchStruct{
961 {baseNamespacedPod("t15-foo1", "t15-ns1"), true, watch.Added},
962 {baseNamespacedPod("t15-foo2", "t15-ns1"), false, ""},
963 {baseNamespacedPodUpdated("t15-foo1", "t15-ns1"), true, watch.Modified},
964 {baseNamespacedPodAssigned("t15-foo1", "t15-ns1", "t15-bar1"), true, watch.Deleted},
965 {baseNamespacedPodAssigned("t15-foo1", "t15-ns1", "t15-bar2"), true, watch.Added},
966 },
967 },
968 {
969 name: "namespaced watch, request with name, request with namespace, with field selector metadata.name",
970 requestedName: "t16-foo1",
971 requestedNamespace: "t16-ns1",
972 fieldSelector: fields.ParseSelectorOrDie("metadata.name=t16-foo1"),
973 watchTests: []*testWatchStruct{
974 {baseNamespacedPod("t16-foo1", "t16-ns1"), true, watch.Added},
975 {baseNamespacedPod("t16-foo2", "t16-ns1"), false, ""},
976 {baseNamespacedPodUpdated("t16-foo1", "t16-ns1"), true, watch.Modified},
977 {baseNamespacedPodAssigned("t16-foo1", "t16-ns1", "t16-bar1"), true, watch.Modified},
978 },
979 },
980 {
981 name: "namespaced watch, request with name, request with namespace, with field selector metadata.name and metadata.namespace",
982 requestedName: "t17-foo2",
983 requestedNamespace: "t17-ns1",
984 fieldSelector: fields.SelectorFromSet(fields.Set{
985 "metadata.name": "t17-foo2",
986 "metadata.namespace": "t17-ns1",
987 }),
988 watchTests: []*testWatchStruct{
989 {baseNamespacedPod("t17-foo1", "t17-ns1"), false, ""},
990 {baseNamespacedPod("t17-foo2", "t17-ns1"), true, watch.Added},
991 {baseNamespacedPodUpdated("t17-foo1", "t17-ns1"), false, ""},
992 {baseNamespacedPodAssigned("t17-foo2", "t17-ns1", "t17-bar1"), true, watch.Modified},
993 },
994 },
995 {
996 name: "namespaced watch, request with name, request with namespace, with field selector metadata.name, metadata.namespace and spec.nodename",
997 requestedName: "t18-foo1",
998 requestedNamespace: "t18-ns1",
999 fieldSelector: fields.SelectorFromSet(fields.Set{
1000 "metadata.name": "t18-foo1",
1001 "metadata.namespace": "t18-ns1",
1002 "spec.nodeName": "t18-bar1",
1003 }),
1004 indexFields: []string{"spec.nodeName"},
1005 watchTests: []*testWatchStruct{
1006 {baseNamespacedPod("t18-foo1", "t18-ns1"), false, ""},
1007 {baseNamespacedPod("t18-foo2", "t18-ns1"), false, ""},
1008 {baseNamespacedPod("t18-foo1", "t18-ns2"), false, ""},
1009 {baseNamespacedPodUpdated("t18-foo1", "t18-ns1"), false, ""},
1010 {baseNamespacedPodAssigned("t18-foo1", "t18-ns1", "t18-bar1"), true, watch.Added},
1011 },
1012 },
1013 {
1014 name: "namespaced watch, request with name, request with namespace, with field selector metadata.name, metadata.namespace, and with spec.nodename to filter out watch",
1015 requestedName: "t19-foo2",
1016 requestedNamespace: "t19-ns1",
1017 fieldSelector: fields.AndSelectors(
1018 fields.ParseSelectorOrDie("spec.nodeName!=t19-bar1"),
1019 fields.SelectorFromSet(fields.Set{"metadata.name": "t19-foo2", "metadata.namespace": "t19-ns1"}),
1020 ),
1021 indexFields: []string{"spec.nodeName"},
1022 watchTests: []*testWatchStruct{
1023 {baseNamespacedPod("t19-foo1", "t19-ns1"), false, ""},
1024 {baseNamespacedPod("t19-foo2", "t19-ns2"), false, ""},
1025 {baseNamespacedPod("t19-foo2", "t19-ns1"), true, watch.Added},
1026 {baseNamespacedPodUpdated("t19-foo2", "t19-ns1"), true, watch.Modified},
1027 {baseNamespacedPodAssigned("t19-foo2", "t19-ns1", "t19-bar1"), true, watch.Deleted},
1028 },
1029 },
1030 }
1031 for _, tt := range tests {
1032 t.Run(tt.name, func(t *testing.T) {
1033 requestInfo := &genericapirequest.RequestInfo{}
1034 requestInfo.Name = tt.requestedName
1035 requestInfo.Namespace = tt.requestedNamespace
1036 ctx = genericapirequest.WithRequestInfo(ctx, requestInfo)
1037 ctx = genericapirequest.WithNamespace(ctx, tt.requestedNamespace)
1038
1039 watchKey := "/pods"
1040 if tt.requestedNamespace != "" {
1041 watchKey += "/" + tt.requestedNamespace
1042 if tt.requestedName != "" {
1043 watchKey += "/" + tt.requestedName
1044 }
1045 }
1046
1047 predicate := CreatePodPredicate(tt.fieldSelector, true, tt.indexFields)
1048
1049 list := &example.PodList{}
1050 opts := storage.ListOptions{
1051 ResourceVersion: "",
1052 Predicate: predicate,
1053 Recursive: true,
1054 }
1055 if err := store.GetList(ctx, "/pods", opts, list); err != nil {
1056 t.Errorf("Unexpected error: %v", err)
1057 }
1058
1059 opts.ResourceVersion = list.ResourceVersion
1060 opts.Recursive = tt.recursive
1061
1062 w, err := store.Watch(ctx, watchKey, opts)
1063 if err != nil {
1064 t.Fatalf("Watch failed: %v", err)
1065 }
1066
1067 currentObjs := map[string]*example.Pod{}
1068 for _, watchTest := range tt.watchTests {
1069 out := &example.Pod{}
1070 key := "pods/" + watchTest.obj.Namespace + "/" + watchTest.obj.Name
1071 err := store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate(
1072 func(runtime.Object) (runtime.Object, error) {
1073 obj := watchTest.obj.DeepCopy()
1074 return obj, nil
1075 }), nil)
1076 if err != nil {
1077 t.Fatalf("GuaranteedUpdate failed: %v", err)
1078 }
1079
1080 expectObj := out
1081 podIdentifier := watchTest.obj.Namespace + "/" + watchTest.obj.Name
1082 if watchTest.watchType == watch.Deleted {
1083 expectObj = currentObjs[podIdentifier]
1084 expectObj.ResourceVersion = out.ResourceVersion
1085 delete(currentObjs, podIdentifier)
1086 } else {
1087 currentObjs[podIdentifier] = out
1088 }
1089 if watchTest.expectEvent {
1090 testCheckResult(t, w, watch.Event{Type: watchTest.watchType, Object: expectObj})
1091 }
1092 }
1093 w.Stop()
1094 testCheckStop(t, w)
1095 })
1096 }
1097 }
1098
1099 // RunOptionalTestWatchDispatchBookmarkEvents tests whether bookmark events are sent.
1100 // This feature is currently implemented in watch cache layer, so this is optional.
1101 //
1102 // TODO(#109831): ProgressNotify feature is effectively implementing the same
1103 //
1104 // functionality, so we should refactor this functionality to share the same input.
1105 func RunTestWatchDispatchBookmarkEvents(ctx context.Context, t *testing.T, store storage.Interface, expectedWatchBookmarks bool) {
1106 key, storedObj := testPropagateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test-ns"}})
1107 startRV := storedObj.ResourceVersion
1108
1109 tests := []struct {
1110 name string
1111 timeout time.Duration
1112 expected bool
1113 allowWatchBookmarks bool
1114 }{
1115 { // test old client won't get Bookmark event
1116 name: "allowWatchBookmarks=false",
1117 timeout: 3 * time.Second,
1118 expected: false,
1119 allowWatchBookmarks: false,
1120 },
1121 {
1122 name: "allowWatchBookmarks=true",
1123 timeout: 3 * time.Second,
1124 expected: expectedWatchBookmarks,
1125 allowWatchBookmarks: true,
1126 },
1127 }
1128
1129 for i, tt := range tests {
1130 t.Run(tt.name, func(t *testing.T) {
1131 pred := storage.Everything
1132 pred.AllowWatchBookmarks = tt.allowWatchBookmarks
1133 ctx, cancel := context.WithTimeout(context.Background(), tt.timeout)
1134 defer cancel()
1135
1136 watcher, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: startRV, Predicate: pred})
1137 if err != nil {
1138 t.Fatalf("Unexpected error: %v", err)
1139 }
1140 defer watcher.Stop()
1141
1142 // Create events of pods in a different namespace
1143 out := &example.Pod{}
1144 obj := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: fmt.Sprintf("other-ns-%d", i)}}
1145 objKey := computePodKey(obj)
1146
1147 if err := store.Create(ctx, objKey, obj, out, 0); err != nil {
1148 t.Fatalf("Create failed: %v", err)
1149 }
1150
1151 // Now wait for Bookmark event
1152 select {
1153 case event, ok := <-watcher.ResultChan():
1154 if !ok && tt.expected {
1155 t.Errorf("Unexpected object watched (no objects)")
1156 }
1157 if tt.expected && event.Type != watch.Bookmark {
1158 t.Errorf("Unexpected object watched %#v", event)
1159 }
1160 case <-time.After(time.Second * 3):
1161 if tt.expected {
1162 t.Errorf("Unexpected object watched (timeout)")
1163 }
1164 }
1165 })
1166 }
1167 }
1168
1169 // RunOptionalTestWatchBookmarksWithCorrectResourceVersion tests whether bookmark events are
1170 // sent with correct resource versions.
1171 // This feature is currently implemented in watch cache layer, so this is optional.
1172 //
1173 // TODO(#109831): ProgressNotify feature is effectively implementing the same
1174 //
1175 // functionality, so we should refactor this functionality to share the same input.
1176 func RunTestOptionalWatchBookmarksWithCorrectResourceVersion(ctx context.Context, t *testing.T, store storage.Interface) {
1177 // Compute the initial resource version.
1178 list := &example.PodList{}
1179 storageOpts := storage.ListOptions{
1180 Predicate: storage.Everything,
1181 Recursive: true,
1182 }
1183 if err := store.GetList(ctx, "/pods", storageOpts, list); err != nil {
1184 t.Errorf("Unexpected error: %v", err)
1185 }
1186 startRV := list.ResourceVersion
1187
1188 key := "/pods/test-ns"
1189 pred := storage.Everything
1190 pred.AllowWatchBookmarks = true
1191
1192 ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
1193 defer cancel()
1194
1195 watcher, err := store.Watch(ctx, key, storage.ListOptions{ResourceVersion: startRV, Predicate: pred, Recursive: true})
1196 if err != nil {
1197 t.Fatalf("Unexpected error: %v", err)
1198 }
1199 defer watcher.Stop()
1200
1201 done := make(chan struct{})
1202 errc := make(chan error, 1)
1203 var wg sync.WaitGroup
1204 wg.Add(1)
1205 // We must wait for the waitgroup to exit before we terminate the cache or the server in prior defers.
1206 defer wg.Wait()
1207 // Call close first, so the goroutine knows to exit.
1208 defer close(done)
1209
1210 go func() {
1211 defer wg.Done()
1212 for i := 0; i < 100; i++ {
1213 select {
1214 case <-done:
1215 return
1216 default:
1217 out := &example.Pod{}
1218 pod := &example.Pod{
1219 ObjectMeta: metav1.ObjectMeta{
1220 Name: fmt.Sprintf("foo-%d", i),
1221 Namespace: "test-ns",
1222 },
1223 }
1224 podKey := computePodKey(pod)
1225 if err := store.Create(ctx, podKey, pod, out, 0); err != nil {
1226 errc <- fmt.Errorf("failed to create pod %v: %v", pod, err)
1227 return
1228 }
1229 time.Sleep(10 * time.Millisecond)
1230 }
1231 }
1232 }()
1233
1234 bookmarkReceived := false
1235 lastObservedResourceVersion := uint64(0)
1236
1237 for {
1238 select {
1239 case err := <-errc:
1240 t.Fatal(err)
1241 case event, ok := <-watcher.ResultChan():
1242 if !ok {
1243 // Make sure we have received a bookmark event
1244 if !bookmarkReceived {
1245 t.Fatalf("Unpexected error, we did not received a bookmark event")
1246 }
1247 return
1248 }
1249 rv, err := storage.APIObjectVersioner{}.ObjectResourceVersion(event.Object)
1250 if err != nil {
1251 t.Fatalf("failed to parse resourceVersion from %#v", event)
1252 }
1253 if event.Type == watch.Bookmark {
1254 bookmarkReceived = true
1255 // bookmark event has a RV greater than or equal to the before one
1256 if rv < lastObservedResourceVersion {
1257 t.Fatalf("Unexpected bookmark resourceVersion %v less than observed %v)", rv, lastObservedResourceVersion)
1258 }
1259 } else {
1260 // non-bookmark event has a RV greater than anything before
1261 if rv <= lastObservedResourceVersion {
1262 t.Fatalf("Unexpected event resourceVersion %v less than or equal to bookmark %v)", rv, lastObservedResourceVersion)
1263 }
1264 }
1265 lastObservedResourceVersion = rv
1266 }
1267 }
1268 }
1269
1270 // RunSendInitialEventsBackwardCompatibility test backward compatibility
1271 // when SendInitialEvents option is set against various implementations.
1272 // Backward compatibility is defined as RV = "" || RV = "O" and AllowWatchBookmark is set to false.
1273 // In that case we expect a watch request to be established.
1274 func RunSendInitialEventsBackwardCompatibility(ctx context.Context, t *testing.T, store storage.Interface) {
1275 opts := storage.ListOptions{Predicate: storage.Everything}
1276 opts.SendInitialEvents = pointer.Bool(true)
1277 w, err := store.Watch(ctx, "/pods", opts)
1278 require.NoError(t, err)
1279 w.Stop()
1280 }
1281
1282 // RunWatchSemantics test the following cases:
1283 //
1284 // +-----------------+---------------------+-------------------+
1285 // | ResourceVersion | AllowWatchBookmarks | SendInitialEvents |
1286 // +=================+=====================+===================+
1287 // | Unset | true/false | true/false |
1288 // | 0 | true/false | true/false |
1289 // | 1 | true/false | true/false |
1290 // | Current | true/false | true/false |
1291 // +-----------------+---------------------+-------------------+
1292 // where:
1293 // - false indicates the value of the param was set to "false" by a test case
1294 // - true indicates the value of the param was set to "true" by a test case
1295 func RunWatchSemantics(ctx context.Context, t *testing.T, store storage.Interface) {
1296 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
1297 trueVal, falseVal := true, false
1298 addEventsFromCreatedPods := func(createdInitialPods []*example.Pod) []watch.Event {
1299 var ret []watch.Event
1300 for _, createdPod := range createdInitialPods {
1301 ret = append(ret, watch.Event{Type: watch.Added, Object: createdPod})
1302 }
1303 return ret
1304 }
1305 initialEventsEndFromLastCreatedPod := func(createdInitialPods []*example.Pod) watch.Event {
1306 return watch.Event{
1307 Type: watch.Bookmark,
1308 Object: &example.Pod{
1309 ObjectMeta: metav1.ObjectMeta{
1310 ResourceVersion: createdInitialPods[len(createdInitialPods)-1].ResourceVersion,
1311 Annotations: map[string]string{metav1.InitialEventsAnnotationKey: "true"},
1312 },
1313 },
1314 }
1315 }
1316 scenarios := []struct {
1317 name string
1318 allowWatchBookmarks bool
1319 sendInitialEvents *bool
1320 resourceVersion string
1321 // useCurrentRV if set gets the current RV from the storage
1322 // after adding the initial pods which is then used to establish a new watch request
1323 useCurrentRV bool
1324
1325 initialPods []*example.Pod
1326 podsAfterEstablishingWatch []*example.Pod
1327
1328 expectedInitialEvents func(createdInitialPods []*example.Pod) []watch.Event
1329 expectedInitialEventsBookmarkWithMinimalRV func(createdInitialPods []*example.Pod) watch.Event
1330 expectedEventsAfterEstablishingWatch func(createdPodsAfterWatch []*example.Pod) []watch.Event
1331 }{
1332 {
1333 name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=unset",
1334 allowWatchBookmarks: true,
1335 sendInitialEvents: &trueVal,
1336 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1337 expectedInitialEvents: addEventsFromCreatedPods,
1338 expectedInitialEventsBookmarkWithMinimalRV: initialEventsEndFromLastCreatedPod,
1339 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1340 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1341 },
1342 {
1343 name: "allowWatchBookmarks=true, sendInitialEvents=false, RV=unset",
1344 allowWatchBookmarks: true,
1345 sendInitialEvents: &falseVal,
1346 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1347 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1348 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1349 },
1350 {
1351 name: "allowWatchBookmarks=false, sendInitialEvents=false, RV=unset",
1352 sendInitialEvents: &falseVal,
1353 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1354 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1355 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1356 },
1357 {
1358 name: "allowWatchBookmarks=false, sendInitialEvents=true, RV=unset",
1359 sendInitialEvents: &trueVal,
1360 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1361 expectedInitialEvents: addEventsFromCreatedPods,
1362 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1363 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1364 },
1365
1366 {
1367 name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=0",
1368 allowWatchBookmarks: true,
1369 sendInitialEvents: &trueVal,
1370 resourceVersion: "0",
1371 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1372 expectedInitialEvents: addEventsFromCreatedPods,
1373 expectedInitialEventsBookmarkWithMinimalRV: initialEventsEndFromLastCreatedPod,
1374 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1375 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1376 },
1377 {
1378 name: "allowWatchBookmarks=true, sendInitialEvents=false, RV=0",
1379 allowWatchBookmarks: true,
1380 sendInitialEvents: &falseVal,
1381 resourceVersion: "0",
1382 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1383 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1384 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1385 },
1386 {
1387 name: "allowWatchBookmarks=false, sendInitialEvents=false, RV=0",
1388 sendInitialEvents: &falseVal,
1389 resourceVersion: "0",
1390 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1391 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1392 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1393 },
1394 {
1395 name: "allowWatchBookmarks=false, sendInitialEvents=true, RV=0",
1396 sendInitialEvents: &trueVal,
1397 resourceVersion: "0",
1398 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1399 expectedInitialEvents: addEventsFromCreatedPods,
1400 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1401 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1402 },
1403
1404 {
1405 name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=1",
1406 allowWatchBookmarks: true,
1407 sendInitialEvents: &trueVal,
1408 resourceVersion: "1",
1409 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1410 expectedInitialEvents: addEventsFromCreatedPods,
1411 expectedInitialEventsBookmarkWithMinimalRV: initialEventsEndFromLastCreatedPod,
1412 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1413 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1414 },
1415 {
1416 name: "allowWatchBookmarks=true, sendInitialEvents=false, RV=1",
1417 allowWatchBookmarks: true,
1418 sendInitialEvents: &falseVal,
1419 resourceVersion: "1",
1420 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1421 expectedInitialEvents: addEventsFromCreatedPods,
1422 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1423 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1424 },
1425 {
1426 name: "allowWatchBookmarks=false, sendInitialEvents=false, RV=1",
1427 sendInitialEvents: &falseVal,
1428 resourceVersion: "1",
1429 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1430 expectedInitialEvents: addEventsFromCreatedPods,
1431 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1432 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1433 },
1434 {
1435 name: "allowWatchBookmarks=false, sendInitialEvents=true, RV=1",
1436 sendInitialEvents: &trueVal,
1437 resourceVersion: "1",
1438 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1439 expectedInitialEvents: addEventsFromCreatedPods,
1440 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1441 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1442 },
1443
1444 {
1445 name: "allowWatchBookmarks=true, sendInitialEvents=true, RV=useCurrentRV",
1446 allowWatchBookmarks: true,
1447 sendInitialEvents: &trueVal,
1448 useCurrentRV: true,
1449 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1450 expectedInitialEvents: addEventsFromCreatedPods,
1451 expectedInitialEventsBookmarkWithMinimalRV: initialEventsEndFromLastCreatedPod,
1452 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1453 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1454 },
1455 {
1456 name: "allowWatchBookmarks=true, sendInitialEvents=false, RV=useCurrentRV",
1457 allowWatchBookmarks: true,
1458 sendInitialEvents: &falseVal,
1459 useCurrentRV: true,
1460 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1461 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1462 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1463 },
1464 {
1465 name: "allowWatchBookmarks=false, sendInitialEvents=false, RV=useCurrentRV",
1466 sendInitialEvents: &falseVal,
1467 useCurrentRV: true,
1468 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1469 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1470 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1471 },
1472 {
1473 name: "allowWatchBookmarks=false, sendInitialEvents=true, RV=useCurrentRV",
1474 sendInitialEvents: &trueVal,
1475 useCurrentRV: true,
1476 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1477 expectedInitialEvents: addEventsFromCreatedPods,
1478 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1479 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1480 },
1481
1482 {
1483 name: "legacy, RV=0",
1484 resourceVersion: "0",
1485 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1486 expectedInitialEvents: addEventsFromCreatedPods,
1487 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1488 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1489 },
1490 {
1491 name: "legacy, RV=unset",
1492 initialPods: []*example.Pod{makePod("1"), makePod("2"), makePod("3")},
1493 expectedInitialEvents: addEventsFromCreatedPods,
1494 podsAfterEstablishingWatch: []*example.Pod{makePod("4"), makePod("5")},
1495 expectedEventsAfterEstablishingWatch: addEventsFromCreatedPods,
1496 },
1497 }
1498 for idx, scenario := range scenarios {
1499 t.Run(scenario.name, func(t *testing.T) {
1500 t.Parallel()
1501 // set up env
1502 if scenario.expectedInitialEvents == nil {
1503 scenario.expectedInitialEvents = func(_ []*example.Pod) []watch.Event { return nil }
1504 }
1505 if scenario.expectedEventsAfterEstablishingWatch == nil {
1506 scenario.expectedEventsAfterEstablishingWatch = func(_ []*example.Pod) []watch.Event { return nil }
1507 }
1508
1509 var createdPods []*example.Pod
1510 ns := fmt.Sprintf("ns-%v", idx)
1511 for _, obj := range scenario.initialPods {
1512 obj.Namespace = ns
1513 out := &example.Pod{}
1514 err := store.Create(ctx, computePodKey(obj), obj, out, 0)
1515 require.NoError(t, err, "failed to add a pod: %v", obj)
1516 createdPods = append(createdPods, out)
1517 }
1518
1519 if len(createdPods) > 0 {
1520 // this list call ensures that the cache has seen the created pods.
1521 // this makes the watch request below deterministic.
1522 listObject := &example.PodList{}
1523 opts := storage.ListOptions{
1524 Predicate: storage.Everything,
1525 Recursive: true,
1526 ResourceVersion: createdPods[len(createdPods)-1].ResourceVersion,
1527 ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan,
1528 }
1529 err := store.GetList(ctx, fmt.Sprintf("/pods/%s", ns), opts, listObject)
1530 require.NoError(t, err)
1531 require.Len(t, listObject.Items, len(createdPods))
1532 }
1533
1534 if scenario.useCurrentRV {
1535 currentStorageRV, err := store.GetCurrentResourceVersion(ctx)
1536 require.NoError(t, err)
1537 scenario.resourceVersion = fmt.Sprintf("%d", currentStorageRV)
1538 }
1539
1540 opts := storage.ListOptions{Predicate: storage.Everything, Recursive: true}
1541 opts.SendInitialEvents = scenario.sendInitialEvents
1542 opts.Predicate.AllowWatchBookmarks = scenario.allowWatchBookmarks
1543 if len(scenario.resourceVersion) > 0 {
1544 opts.ResourceVersion = scenario.resourceVersion
1545 }
1546
1547 w, err := store.Watch(context.Background(), fmt.Sprintf("/pods/%s", ns), opts)
1548 require.NoError(t, err, "failed to create watch: %v")
1549 defer w.Stop()
1550
1551 // make sure we only get initial events
1552 TestCheckResultsInStrictOrder(t, w, scenario.expectedInitialEvents(createdPods))
1553
1554 // make sure that the actual bookmark has at least RV >= to the expected one
1555 if scenario.expectedInitialEventsBookmarkWithMinimalRV != nil {
1556 testCheckResultFunc(t, w, func(actualEvent watch.Event) {
1557 expectedBookmarkEventWithMinRV := scenario.expectedInitialEventsBookmarkWithMinimalRV(createdPods)
1558 expectedObj, err := meta.Accessor(expectedBookmarkEventWithMinRV.Object)
1559 require.NoError(t, err)
1560 expectedRV, err := storage.APIObjectVersioner{}.ObjectResourceVersion(expectedBookmarkEventWithMinRV.Object)
1561 require.NoError(t, err)
1562
1563 actualObj, err := meta.Accessor(actualEvent.Object)
1564 require.NoError(t, err)
1565 actualRV, err := storage.APIObjectVersioner{}.ObjectResourceVersion(actualEvent.Object)
1566 require.NoError(t, err)
1567
1568 require.GreaterOrEqual(t, actualRV, expectedRV)
1569
1570 // once we know that the RV is at least >= the expected one
1571 // rewrite it so that we can compare the objs
1572 expectedObj.SetResourceVersion(actualObj.GetResourceVersion())
1573 expectNoDiff(t, "incorrect event", expectedBookmarkEventWithMinRV, actualEvent)
1574 })
1575 }
1576
1577 createdPods = []*example.Pod{}
1578 // add a pod that is greater than the storage's RV when the watch was started
1579 for _, obj := range scenario.podsAfterEstablishingWatch {
1580 obj.Namespace = ns
1581 out := &example.Pod{}
1582 err = store.Create(ctx, computePodKey(obj), obj, out, 0)
1583 require.NoError(t, err, "failed to add a pod: %v")
1584 createdPods = append(createdPods, out)
1585 }
1586 ignoreEventsFn := func(event watch.Event) bool { return event.Type == watch.Bookmark }
1587 testCheckResultWithIgnoreFunc(t, w, scenario.expectedEventsAfterEstablishingWatch(createdPods), ignoreEventsFn)
1588 TestCheckNoMoreResultsWithIgnoreFunc(t, w, ignoreEventsFn)
1589 })
1590 }
1591 }
1592
1593 // RunWatchSemanticInitialEventsExtended checks if the bookmark event
1594 // marking the end of the list stream contains the global RV.
1595 //
1596 // note that this scenario differs from the one in RunWatchSemantics
1597 // by adding the pod to a different ns to advance the global RV
1598 func RunWatchSemanticInitialEventsExtended(ctx context.Context, t *testing.T, store storage.Interface) {
1599 trueVal := true
1600 expectedInitialEventsInStrictOrder := func(initialPods []*example.Pod, globalResourceVersion string) []watch.Event {
1601 watchEvents := []watch.Event{}
1602 for _, initialPod := range initialPods {
1603 watchEvents = append(watchEvents, watch.Event{Type: watch.Added, Object: initialPod})
1604 }
1605 watchEvents = append(watchEvents, watch.Event{Type: watch.Bookmark, Object: &example.Pod{
1606 ObjectMeta: metav1.ObjectMeta{
1607 ResourceVersion: globalResourceVersion,
1608 Annotations: map[string]string{metav1.InitialEventsAnnotationKey: "true"},
1609 },
1610 }})
1611 return watchEvents
1612 }
1613 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
1614
1615 initialPods := []*example.Pod{}
1616 ns := "ns-foo"
1617 for _, initialPod := range []*example.Pod{makePod("1"), makePod("2"), makePod("3"), makePod("4"), makePod("5")} {
1618 initialPod.Namespace = ns
1619 out := &example.Pod{}
1620 err := store.Create(ctx, computePodKey(initialPod), initialPod, out, 0)
1621 require.NoError(t, err, "failed to add a pod: %v")
1622 initialPods = append(initialPods, out)
1623 }
1624
1625 // add the pod to a different ns to advance the global RV
1626 pod := makePod("1")
1627 pod.Namespace = "other-ns-foo"
1628 otherNsPod := &example.Pod{}
1629 err := store.Create(ctx, computePodKey(pod), pod, otherNsPod, 0)
1630 require.NoError(t, err, "failed to add a pod: %v")
1631
1632 opts := storage.ListOptions{Predicate: storage.Everything, Recursive: true}
1633 opts.SendInitialEvents = &trueVal
1634 opts.Predicate.AllowWatchBookmarks = true
1635
1636 w, err := store.Watch(context.Background(), fmt.Sprintf("/pods/%s", ns), opts)
1637 require.NoError(t, err, "failed to create watch: %v")
1638 defer w.Stop()
1639
1640 // make sure we only get initial events from the first ns
1641 // followed by the bookmark with the global RV
1642 TestCheckResultsInStrictOrder(t, w, expectedInitialEventsInStrictOrder(initialPods, otherNsPod.ResourceVersion))
1643 TestCheckNoMoreResultsWithIgnoreFunc(t, w, nil)
1644 }
1645
1646 func RunWatchListMatchSingle(ctx context.Context, t *testing.T, store storage.Interface) {
1647 trueVal := true
1648 expectedInitialEventsInStrictOrder := func(initialPod *example.Pod, globalResourceVersion string) []watch.Event {
1649 watchEvents := []watch.Event{}
1650 watchEvents = append(watchEvents, watch.Event{Type: watch.Added, Object: initialPod})
1651 watchEvents = append(watchEvents, watch.Event{Type: watch.Bookmark, Object: &example.Pod{
1652 ObjectMeta: metav1.ObjectMeta{
1653 ResourceVersion: globalResourceVersion,
1654 Annotations: map[string]string{metav1.InitialEventsAnnotationKey: "true"},
1655 },
1656 }})
1657 return watchEvents
1658 }
1659 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
1660
1661 // add the pod for which the field selector will be constructed
1662 ns := "ns-foo"
1663 expectedPod := &example.Pod{}
1664 initialPod := makePod("1")
1665 initialPod.Namespace = ns
1666 err := store.Create(ctx, computePodKey(initialPod), initialPod, expectedPod, 0)
1667 require.NoError(t, err, "failed to add a pod: %v")
1668
1669 // add more pods that won't match the field selector
1670 lastAddedPod := &example.Pod{}
1671 for _, otherPod := range []*example.Pod{makePod("2"), makePod("3"), makePod("4"), makePod("5")} {
1672 otherPod.Namespace = ns
1673 err = store.Create(ctx, computePodKey(otherPod), otherPod, lastAddedPod, 0)
1674 require.NoError(t, err, "failed to add a pod: %v")
1675 }
1676
1677 opts := storage.ListOptions{
1678 Predicate: storage.SelectionPredicate{
1679 Label: labels.Everything(),
1680 Field: fields.ParseSelectorOrDie("metadata.name=pod-1"),
1681 GetAttrs: func(obj runtime.Object) (labels.Set, fields.Set, error) {
1682 pod := obj.(*example.Pod)
1683 return nil, fields.Set{"metadata.name": pod.Name}, nil
1684 },
1685 },
1686 Recursive: true,
1687 }
1688 opts.SendInitialEvents = &trueVal
1689 opts.Predicate.AllowWatchBookmarks = true
1690
1691 w, err := store.Watch(context.Background(), "/pods", opts)
1692 require.NoError(t, err, "failed to create watch: %v")
1693 defer w.Stop()
1694
1695 // make sure we only get a single pod matching the field selector
1696 // followed by the bookmark with the global RV
1697 TestCheckResultsInStrictOrder(t, w, expectedInitialEventsInStrictOrder(expectedPod, lastAddedPod.ResourceVersion))
1698 TestCheckNoMoreResultsWithIgnoreFunc(t, w, nil)
1699 }
1700
1701 func RunWatchErrorIsBlockingFurtherEvents(ctx context.Context, t *testing.T, store InterfaceWithPrefixTransformer) {
1702 foo := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "foo"}}
1703 fooKey := fmt.Sprintf("/pods/%s/%s", foo.Namespace, foo.Name)
1704 fooCreated := &example.Pod{}
1705 if err := store.Create(context.Background(), fooKey, foo, fooCreated, 0); err != nil {
1706 t.Errorf("failed to create object: %v", err)
1707 }
1708 bar := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "bar"}}
1709 barKey := fmt.Sprintf("/pods/%s/%s", bar.Namespace, bar.Name)
1710 barCreated := &example.Pod{}
1711 if err := store.Create(context.Background(), barKey, bar, barCreated, 0); err != nil {
1712 t.Errorf("failed to create object: %v", err)
1713 }
1714
1715 // Update transformer to ensure that foo will become effectively corrupted.
1716 revertTransformer := store.UpdatePrefixTransformer(
1717 func(transformer *PrefixTransformer) value.Transformer {
1718 transformer.prefix = []byte("other-prefix")
1719 return transformer
1720 })
1721 defer revertTransformer()
1722
1723 baz := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "baz"}}
1724 bazKey := fmt.Sprintf("/pods/%s/%s", baz.Namespace, baz.Name)
1725 bazCreated := &example.Pod{}
1726 if err := store.Create(context.Background(), bazKey, baz, bazCreated, 0); err != nil {
1727 t.Errorf("failed to create object: %v", err)
1728 }
1729
1730 opts := storage.ListOptions{
1731 ResourceVersion: fooCreated.ResourceVersion,
1732 Predicate: storage.Everything,
1733 Recursive: true,
1734 }
1735
1736 // Run N concurrent watches. Given the asynchronous nature, we increase the
1737 // probability of hitting the race in at least one of those watches.
1738 concurrentWatches := 10
1739 wg := sync.WaitGroup{}
1740 for i := 0; i < concurrentWatches; i++ {
1741 wg.Add(1)
1742 go func() {
1743 defer wg.Done()
1744 w, err := store.Watch(ctx, "/pods", opts)
1745 if err != nil {
1746 t.Errorf("failed to create watch: %v", err)
1747 return
1748 }
1749
1750 // We issue the watch starting from object bar.
1751 // The object fails TransformFromStorage and generates ERROR watch event.
1752 // The further events (i.e. ADDED event for baz object) should not be
1753 // emitted, so we verify no events other than ERROR type are emitted.
1754 for {
1755 event, ok := <-w.ResultChan()
1756 if !ok {
1757 break
1758 }
1759 if event.Type != watch.Error {
1760 t.Errorf("unexpected event: %#v", event)
1761 }
1762 }
1763 }()
1764 }
1765 wg.Wait()
1766 }
1767
1768 func makePod(namePrefix string) *example.Pod {
1769 return &example.Pod{
1770 ObjectMeta: metav1.ObjectMeta{
1771 Name: fmt.Sprintf("pod-%s", namePrefix),
1772 },
1773 }
1774 }
1775
1776 type testWatchStruct struct {
1777 obj *example.Pod
1778 expectEvent bool
1779 watchType watch.EventType
1780 }
1781
1782 func basePod(podName string) *example.Pod {
1783 return baseNamespacedPod(podName, "")
1784 }
1785
1786 func basePodUpdated(podName string) *example.Pod {
1787 return baseNamespacedPodUpdated(podName, "")
1788 }
1789
1790 func basePodAssigned(podName, nodeName string) *example.Pod {
1791 return baseNamespacedPodAssigned(podName, "", nodeName)
1792 }
1793
1794 func baseNamespacedPod(podName, namespace string) *example.Pod {
1795 return &example.Pod{
1796 ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: namespace},
1797 }
1798 }
1799
1800 func baseNamespacedPodUpdated(podName, namespace string) *example.Pod {
1801 return &example.Pod{
1802 ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: namespace},
1803 Status: example.PodStatus{Phase: "Running"},
1804 }
1805 }
1806
1807 func baseNamespacedPodAssigned(podName, namespace, nodeName string) *example.Pod {
1808 return &example.Pod{
1809 ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: namespace},
1810 Spec: example.PodSpec{NodeName: nodeName},
1811 }
1812 }