Debian Codegraph @

pkg/storage/etcd3/watcher_test.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package etcd3
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "sync"
24 "testing"
25 "time"
26
27 "github.com/google/go-cmp/cmp"
28 clientv3 "go.etcd.io/etcd/client/v3"
29
30 apierrors "k8s.io/apimachinery/pkg/api/errors"
31 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
32 "k8s.io/apimachinery/pkg/fields"
33 "k8s.io/apimachinery/pkg/labels"
34 "k8s.io/apimachinery/pkg/watch"
35 "k8s.io/apiserver/pkg/apis/example"
36 "k8s.io/apiserver/pkg/features"
37 "k8s.io/apiserver/pkg/storage"
38 "k8s.io/apiserver/pkg/storage/etcd3/testserver"
39 storagetesting "k8s.io/apiserver/pkg/storage/testing"
40 utilfeature "k8s.io/apiserver/pkg/util/feature"
41 featuregatetesting "k8s.io/component-base/featuregate/testing"
42 "k8s.io/utils/ptr"
43 )
44
45 func TestWatch(t *testing.T) {
46 ctx, store, _ := testSetup(t)
47 storagetesting.RunTestWatch(ctx, t, store)
48 }
49
50 func TestClusterScopedWatch(t *testing.T) {
51 ctx, store, _ := testSetup(t)
52 storagetesting.RunTestClusterScopedWatch(ctx, t, store)
53 }
54
55 func TestNamespaceScopedWatch(t *testing.T) {
56 ctx, store, _ := testSetup(t)
57 storagetesting.RunTestNamespaceScopedWatch(ctx, t, store)
58 }
59
60 func TestDeleteTriggerWatch(t *testing.T) {
61 ctx, store, _ := testSetup(t)
62 storagetesting.RunTestDeleteTriggerWatch(ctx, t, store)
63 }
64
65 func TestWatchFromZero(t *testing.T) {
66 ctx, store, client := testSetup(t)
67 storagetesting.RunTestWatchFromZero(ctx, t, store, compactStorage(client.Client))
68 }
69
70 // TestWatchFromNonZero tests that
71 // - watch from non-0 should just watch changes after given version
72 func TestWatchFromNoneZero(t *testing.T) {
73 ctx, store, _ := testSetup(t)
74 storagetesting.RunTestWatchFromNonZero(ctx, t, store)
75 }
76
77 func TestDelayedWatchDelivery(t *testing.T) {
78 ctx, store, _ := testSetup(t)
79 storagetesting.RunTestDelayedWatchDelivery(ctx, t, store)
80 }
81
82 func TestWatchError(t *testing.T) {
83 ctx, store, _ := testSetup(t)
84 storagetesting.RunTestWatchError(ctx, t, &storeWithPrefixTransformer{store})
85 }
86
87 func TestWatchContextCancel(t *testing.T) {
88 ctx, store, _ := testSetup(t)
89 storagetesting.RunTestWatchContextCancel(ctx, t, store)
90 }
91
92 func TestWatcherTimeout(t *testing.T) {
93 ctx, store, _ := testSetup(t)
94 storagetesting.RunTestWatcherTimeout(ctx, t, store)
95 }
96
97 func TestWatchDeleteEventObjectHaveLatestRV(t *testing.T) {
98 ctx, store, _ := testSetup(t)
99 storagetesting.RunTestWatchDeleteEventObjectHaveLatestRV(ctx, t, store)
100 }
101
102 func TestWatchInitializationSignal(t *testing.T) {
103 ctx, store, _ := testSetup(t)
104 storagetesting.RunTestWatchInitializationSignal(ctx, t, store)
105 }
106
107 func TestProgressNotify(t *testing.T) {
108 clusterConfig := testserver.NewTestConfig(t)
109 clusterConfig.ExperimentalWatchProgressNotifyInterval = time.Second
110 ctx, store, _ := testSetup(t, withClientConfig(clusterConfig))
111
112 storagetesting.RunOptionalTestProgressNotify(ctx, t, store)
113 }
114
115 func TestWatchWithUnsafeDelete(t *testing.T) {
116 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AllowUnsafeMalformedObjectDeletion, true)
117 ctx, store, _ := testSetup(t)
118 storagetesting.RunTestWatchWithUnsafeDelete(ctx, t, &storeWithCorruptedTransformer{store})
119 }
120
121 // TestWatchDispatchBookmarkEvents makes sure that
122 // setting allowWatchBookmarks query param against
123 // etcd implementation doesn't have any effect.
124 func TestWatchDispatchBookmarkEvents(t *testing.T) {
125 clusterConfig := testserver.NewTestConfig(t)
126 clusterConfig.ExperimentalWatchProgressNotifyInterval = time.Second
127 ctx, store, _ := testSetup(t, withClientConfig(clusterConfig))
128
129 storagetesting.RunTestWatchDispatchBookmarkEvents(ctx, t, store, false)
130 }
131
132 func TestSendInitialEventsBackwardCompatibility(t *testing.T) {
133 ctx, store, _ := testSetup(t)
134 storagetesting.RunSendInitialEventsBackwardCompatibility(ctx, t, store)
135 }
136
137 func TestEtcdWatchSemantics(t *testing.T) {
138 ctx, store, _ := testSetup(t)
139 storagetesting.RunWatchSemantics(ctx, t, store)
140 }
141
142 func TestEtcdWatchSemanticsWithConcurrentDecode(t *testing.T) {
143 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConcurrentWatchObjectDecode, true)
144 ctx, store, _ := testSetup(t)
145 storagetesting.RunWatchSemantics(ctx, t, store)
146 }
147
148 func TestEtcdWatchSemanticInitialEventsExtended(t *testing.T) {
149 ctx, store, _ := testSetup(t)
150 storagetesting.RunWatchSemanticInitialEventsExtended(ctx, t, store)
151 }
152
153 func TestWatchListMatchSingle(t *testing.T) {
154 ctx, store, _ := testSetup(t)
155 storagetesting.RunWatchListMatchSingle(ctx, t, store)
156 }
157
158 func TestWatchErrorEventIsBlockingFurtherEvent(t *testing.T) {
159 ctx, store, _ := testSetup(t)
160 storagetesting.RunWatchErrorIsBlockingFurtherEvents(ctx, t, &storeWithPrefixTransformer{store})
161 }
162
163 // =======================================================================
164 // Implementation-specific tests are following.
165 // The following tests are exercising the details of the implementation
166 // not the actual user-facing contract of storage interface.
167 // As such, they may focus e.g. on non-functional aspects like performance
168 // impact.
169 // =======================================================================
170
171 func TestWatchErrResultNotBlockAfterCancel(t *testing.T) {
172 origCtx, store, _ := testSetup(t)
173 ctx, cancel := context.WithCancel(origCtx)
174 w := store.watcher.createWatchChan(ctx, "/abc", 0, false, false, storage.Everything)
175 // make resultChan and errChan blocking to ensure ordering.
176 w.resultChan = make(chan watch.Event)
177 w.errChan = make(chan error)
178 // The event flow goes like:
179 // - first we send an error, it should block on resultChan.
180 // - Then we cancel ctx. The blocking on resultChan should be freed up
181 // and run() goroutine should return.
182 var wg sync.WaitGroup
183 wg.Add(1)
184 go func() {
185 w.run(false, true)
186 wg.Done()
187 }()
188 w.errChan <- fmt.Errorf("some error")
189 cancel()
190 wg.Wait()
191 }
192
193 // TestWatchErrorIncorrectConfiguration checks if an error
194 // will be returned when the storage hasn't been properly
195 // initialised for watch requests
196 func TestWatchErrorIncorrectConfiguration(t *testing.T) {
197 scenarios := []struct {
198 name string
199 setupFn func(opts *setupOptions)
200 requestOpts storage.ListOptions
201 enableWatchList bool
202 expectedErr error
203 }{
204 {
205 name: "no newFunc provided",
206 setupFn: func(opts *setupOptions) { opts.newFunc = nil },
207 requestOpts: storage.ListOptions{ProgressNotify: true},
208 expectedErr: apierrors.NewInternalError(errors.New("progressNotify for watch is unsupported by the etcd storage because no newFunc was provided")),
209 },
210 }
211 for _, scenario := range scenarios {
212 t.Run(scenario.name, func(t *testing.T) {
213 if scenario.enableWatchList {
214 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
215 }
216 origCtx, store, _ := testSetup(t, scenario.setupFn)
217 ctx, cancel := context.WithCancel(origCtx)
218 defer cancel()
219
220 w, err := store.watcher.Watch(ctx, "/abc", 0, scenario.requestOpts)
221 if err == nil {
222 t.Fatalf("expected an error but got none")
223 }
224 if w != nil {
225 t.Fatalf("didn't expect a watcher because the test assumes incorrect store initialisation")
226 }
227 if err.Error() != scenario.expectedErr.Error() {
228 t.Fatalf("unexpected err = %v, expected = %v", err, scenario.expectedErr)
229 }
230 })
231 }
232 }
233
234 func TestTooLargeResourceVersionErrorForWatchList(t *testing.T) {
235 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WatchList, true)
236 origCtx, store, _ := testSetup(t)
237 ctx, cancel := context.WithCancel(origCtx)
238 defer cancel()
239 requestOpts := storage.ListOptions{
240 SendInitialEvents: ptr.To(true),
241 Recursive: true,
242 Predicate: storage.SelectionPredicate{
243 Field: fields.Everything(),
244 Label: labels.Everything(),
245 AllowWatchBookmarks: true,
246 },
247 }
248 var expectedErr *apierrors.StatusError
249 if !errors.As(storage.NewTooLargeResourceVersionError(uint64(102), 1, 0), &expectedErr) {
250 t.Fatalf("Unable to convert NewTooLargeResourceVersionError to apierrors.StatusError")
251 }
252
253 w, err := store.watcher.Watch(ctx, "/abc", int64(102), requestOpts)
254 if err != nil {
255 t.Fatal(err)
256 }
257 defer w.Stop()
258
259 actualEvent := <-w.ResultChan()
260 if actualEvent.Type != watch.Error {
261 t.Fatalf("Unexpected type of the event: %v, expected: %v", actualEvent.Type, watch.Error)
262 }
263 actualErr, ok := actualEvent.Object.(*metav1.Status)
264 if !ok {
265 t.Fatalf("Expected *apierrors.StatusError, got: %#v", actualEvent.Object)
266 }
267
268 if actualErr.Details.RetryAfterSeconds <= 0 {
269 t.Fatalf("RetryAfterSeconds must be > 0, actual value: %v", actualErr.Details.RetryAfterSeconds)
270 }
271 // rewrite the Details as it contains retry seconds
272 // and validate the whole struct
273 expectedErr.ErrStatus.Details = actualErr.Details
274 if diff := cmp.Diff(*actualErr, expectedErr.ErrStatus); diff != "" {
275 t.Fatalf("Unexpected error returned, diff: %v", diff)
276 }
277 }
278
279 func TestWatchChanSync(t *testing.T) {
280 testCases := []struct {
281 name string
282 watchKey string
283 watcherMaxLimit int64
284 expectEventCount int
285 expectGetCount int
286 }{
287 {
288 name: "None of the current objects match watchKey: sync with empty page",
289 watchKey: "/pods/test/",
290 watcherMaxLimit: 1,
291 expectGetCount: 1,
292 },
293 {
294 name: "The number of current objects is less than defaultWatcherMaxLimit: sync with one page",
295 watchKey: "/pods/",
296 watcherMaxLimit: 3,
297 expectEventCount: 2,
298 expectGetCount: 1,
299 },
300 {
301 name: "a new item added to etcd before returning a second page is not returned: sync with two page",
302 watchKey: "/pods/",
303 watcherMaxLimit: 1,
304 expectEventCount: 2,
305 expectGetCount: 2,
306 },
307 }
308
309 for _, testCase := range testCases {
310 t.Run(testCase.name, func(t *testing.T) {
311 defaultWatcherMaxLimit = testCase.watcherMaxLimit
312
313 origCtx, store, _ := testSetup(t)
314 initList, err := initStoreData(origCtx, store)
315 if err != nil {
316 t.Fatal(err)
317 }
318
319 kvWrapper := newEtcdClientKVWrapper(store.client.KV)
320 kvWrapper.getReactors = append(kvWrapper.getReactors, func() {
321 barThird := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "third", Name: "bar"}}
322 podKey := fmt.Sprintf("/pods/%s/%s", barThird.Namespace, barThird.Name)
323 storedObj := &example.Pod{}
324
325 err := store.Create(context.Background(), podKey, barThird, storedObj, 0)
326 if err != nil {
327 t.Errorf("failed to create object: %v", err)
328 }
329 })
330
331 store.client.KV = kvWrapper
332
333 w := store.watcher.createWatchChan(
334 origCtx,
335 testCase.watchKey,
336 0,
337 true,
338 false,
339 storage.Everything)
340
341 err = w.sync()
342 if err != nil {
343 t.Fatal(err)
344 }
345
346 // close incomingEventChan so we can read incomingEventChan non-blocking
347 close(w.incomingEventChan)
348
349 eventsReceived := 0
350 for event := range w.incomingEventChan {
351 eventsReceived++
352 storagetesting.ExpectContains(t, "incorrect list pods", initList, event.key)
353 }
354
355 if eventsReceived != testCase.expectEventCount {
356 t.Errorf("Unexpected number of events: %v, expected: %v", eventsReceived, testCase.expectEventCount)
357 }
358
359 if kvWrapper.getCallCounter != testCase.expectGetCount {
360 t.Errorf("Unexpected called times of client.KV.Get() : %v, expected: %v", kvWrapper.getCallCounter, testCase.expectGetCount)
361 }
362 })
363 }
364 }
365
366 // NOTE: it's not thread-safe
367 type etcdClientKVWrapper struct {
368 clientv3.KV
369 // keeps track of the number of times Get method is called
370 getCallCounter int
371 // getReactors is called after the etcd KV's get function is executed.
372 getReactors []func()
373 }
374
375 func newEtcdClientKVWrapper(kv clientv3.KV) *etcdClientKVWrapper {
376 return &etcdClientKVWrapper{
377 KV: kv,
378 getCallCounter: 0,
379 }
380 }
381
382 func (ecw *etcdClientKVWrapper) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
383 resp, err := ecw.KV.Get(ctx, key, opts...)
384 ecw.getCallCounter++
385 if err != nil {
386 return nil, err
387 }
388
389 if len(ecw.getReactors) > 0 {
390 reactor := ecw.getReactors[0]
391 ecw.getReactors = ecw.getReactors[1:]
392 reactor()
393 }
394
395 return resp, nil
396 }
397
398 func initStoreData(ctx context.Context, store storage.Interface) ([]interface{}, error) {
399 barFirst := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "first", Name: "bar"}}
400 barSecond := &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "second", Name: "bar"}}
401
402 preset := []struct {
403 key string
404 obj *example.Pod
405 storedObj *example.Pod
406 }{
407 {
408 key: fmt.Sprintf("/pods/%s/%s", barFirst.Namespace, barFirst.Name),
409 obj: barFirst,
410 },
411 {
412 key: fmt.Sprintf("/pods/%s/%s", barSecond.Namespace, barSecond.Name),
413 obj: barSecond,
414 },
415 }
416
417 for i, ps := range preset {
418 preset[i].storedObj = &example.Pod{}
419 err := store.Create(ctx, ps.key, ps.obj, preset[i].storedObj, 0)
420 if err != nil {
421 return nil, fmt.Errorf("failed to create object: %w", err)
422 }
423 }
424
425 var created []interface{}
426 for _, item := range preset {
427 created = append(created, item.key)
428 }
429 return created, nil
430 }