Debian Codegraph @

pkg/admission/plugin/policy/internal/generic/controller_test.go

raw
1 /*
2 Copyright 2022 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package generic_test
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "reflect"
24 "sync"
25 "sync/atomic"
26 "testing"
27 "time"
28
29 "github.com/google/go-cmp/cmp"
30 "github.com/stretchr/testify/require"
31
32 k8serrors "k8s.io/apimachinery/pkg/api/errors"
33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
35 "k8s.io/apimachinery/pkg/labels"
36 "k8s.io/apimachinery/pkg/runtime"
37 "k8s.io/apimachinery/pkg/runtime/schema"
38 "k8s.io/apimachinery/pkg/runtime/serializer"
39 "k8s.io/apimachinery/pkg/util/wait"
40 "k8s.io/apimachinery/pkg/watch"
41
42 "k8s.io/apiserver/pkg/admission/plugin/policy/internal/generic"
43
44 clienttesting "k8s.io/client-go/testing"
45 "k8s.io/client-go/tools/cache"
46 )
47
48 type testInformer struct {
49 cache.SharedIndexInformer
50
51 lock sync.Mutex
52 registrations map[interface{}]struct{}
53 }
54
55 func (t *testInformer) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) {
56 res, err := t.SharedIndexInformer.AddEventHandler(handler)
57 if err != nil {
58 return res, err
59 }
60
61 func() {
62 t.lock.Lock()
63 defer t.lock.Unlock()
64 if t.registrations == nil {
65 t.registrations = make(map[interface{}]struct{})
66 }
67 t.registrations[res] = struct{}{}
68 }()
69
70 return res, err
71 }
72
73 func (t *testInformer) RemoveEventHandler(registration cache.ResourceEventHandlerRegistration) error {
74 func() {
75 t.lock.Lock()
76 defer t.lock.Unlock()
77
78 if _, ok := t.registrations[registration]; !ok {
79 panic("removing unknown event handler?")
80 }
81 delete(t.registrations, registration)
82 }()
83
84 return t.SharedIndexInformer.RemoveEventHandler(registration)
85 }
86
87 var (
88 scheme *runtime.Scheme = runtime.NewScheme()
89 codecs serializer.CodecFactory = serializer.NewCodecFactory(scheme)
90 fakeGVR schema.GroupVersionResource = schema.GroupVersionResource{
91 Group: "fake.example.com",
92 Version: "v1",
93 Resource: "fakes",
94 }
95 fakeGVK schema.GroupVersionKind = fakeGVR.GroupVersion().WithKind("Fake")
96 fakeGVKList schema.GroupVersionKind = fakeGVR.GroupVersion().WithKind("FakeList")
97 )
98
99 func init() {
100 scheme.AddKnownTypeWithName(fakeGVK, &unstructured.Unstructured{})
101 scheme.AddKnownTypeWithName(fakeGVKList, &unstructured.UnstructuredList{})
102 }
103
104 func setupTest(ctx context.Context, customReconciler func(string, string, runtime.Object) error) (
105 tracker clienttesting.ObjectTracker,
106 controller generic.Controller[*unstructured.Unstructured],
107 informer *testInformer,
108 waitForReconcile func(runtime.Object) error,
109 verifyNoMoreEvents func() bool,
110 ) {
111 tracker = clienttesting.NewObjectTracker(scheme, codecs.UniversalDecoder())
112 reconciledObjects := make(chan runtime.Object)
113
114 // Set up fake informers that return instances of mock Policy definitoins
115 // and mock policy bindings
116 informer = &testInformer{SharedIndexInformer: cache.NewSharedIndexInformer(&cache.ListWatch{
117 ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
118 return tracker.List(fakeGVR, fakeGVK, "")
119 },
120 WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
121 return tracker.Watch(fakeGVR, "")
122 },
123 }, &unstructured.Unstructured{}, 30*time.Second, nil)}
124
125 reconciler := func(namespace, name string, newObj *unstructured.Unstructured) error {
126 var err error
127 copied := newObj.DeepCopyObject()
128 if customReconciler != nil {
129 err = customReconciler(namespace, name, newObj)
130 }
131 select {
132 case reconciledObjects <- copied:
133 case <-ctx.Done():
134 panic("timed out attempting to deliver reconcile event")
135 }
136 return err
137 }
138
139 waitForReconcile = func(obj runtime.Object) error {
140 select {
141 case reconciledObj := <-reconciledObjects:
142 if reflect.DeepEqual(obj, reconciledObj) {
143 return nil
144 }
145 return fmt.Errorf("expected equal objects: %v", cmp.Diff(obj, reconciledObj))
146 case <-ctx.Done():
147 return fmt.Errorf("context done before reconcile: %w", ctx.Err())
148 }
149 }
150
151 myController := generic.NewController(
152 generic.NewInformer[*unstructured.Unstructured](informer),
153 reconciler,
154 generic.ControllerOptions{},
155 )
156
157 verifyNoMoreEvents = func() bool {
158 close(reconciledObjects) // closing means that a future attempt to send will crash
159 for leftover := range reconciledObjects {
160 panic(fmt.Errorf("leftover object which was not anticipated by test: %v", leftover))
161 }
162 // TODO(alexzielenski): this effectively doesn't test anything since the
163 // controller drops any pending events when it shuts down.
164 return true
165 }
166
167 return tracker, myController, informer, waitForReconcile, verifyNoMoreEvents
168 }
169
170 func TestReconcile(t *testing.T) {
171 testContext, testCancel := context.WithTimeout(context.Background(), 2*time.Second)
172 defer testCancel()
173
174 tracker, myController, informer, waitForReconcile, verifyNoMoreEvents := setupTest(testContext, nil)
175
176 // Add object to informer
177 initialObject := &unstructured.Unstructured{}
178 initialObject.SetUnstructuredContent(map[string]interface{}{
179 "metadata": map[string]interface{}{
180 "name": "object1",
181 "resourceVersion": "1",
182 },
183 })
184 initialObject.SetGroupVersionKind(fakeGVK)
185
186 require.NoError(t, tracker.Add(initialObject))
187
188 wg := sync.WaitGroup{}
189
190 // Start informer
191 wg.Add(1)
192 go func() {
193 defer wg.Done()
194 informer.Run(testContext.Done())
195 }()
196
197 // Start controller
198 wg.Add(1)
199 go func() {
200 defer wg.Done()
201 stopReason := myController.Run(testContext)
202 if !errors.Is(stopReason, context.Canceled) {
203 t.Errorf("expected error to be context.Canceled, but got: %v", stopReason)
204 }
205 }()
206
207 // The controller is blocked because the reconcile function sends on an
208 // unbuffered channel.
209 require.False(t, myController.HasSynced())
210
211 // Wait for all enqueued reconciliations
212 require.NoError(t, waitForReconcile(initialObject))
213
214 // Now it is safe to wait for it to Sync
215 require.True(t, cache.WaitForCacheSync(testContext.Done(), myController.HasSynced))
216
217 // Updated object
218 updatedObject := &unstructured.Unstructured{}
219 updatedObject.SetUnstructuredContent(map[string]interface{}{
220 "metadata": map[string]interface{}{
221 "name": "object1",
222 "resourceVersion": "2",
223 },
224 "newKey": "a key",
225 })
226 updatedObject.SetGroupVersionKind(fakeGVK)
227 require.NoError(t, tracker.Update(fakeGVR, updatedObject, ""))
228
229 // Wait for all enqueued reconciliations
230 require.NoError(t, waitForReconcile(updatedObject))
231 require.NoError(t, tracker.Delete(fakeGVR, updatedObject.GetNamespace(), updatedObject.GetName()))
232 require.NoError(t, waitForReconcile(nil))
233
234 testCancel()
235 wg.Wait()
236
237 verifyNoMoreEvents()
238 }
239
240 func TestShutdown(t *testing.T) {
241 testContext, testCancel := context.WithTimeout(context.Background(), 2*time.Second)
242 defer testCancel()
243
244 _, myController, informer, _, verifyNoMoreEvents := setupTest(testContext, nil)
245
246 wg := sync.WaitGroup{}
247
248 // Start informer
249 wg.Add(1)
250 go func() {
251 defer wg.Done()
252 informer.Run(testContext.Done())
253 }()
254
255 // Start controller
256 wg.Add(1)
257 go func() {
258 defer wg.Done()
259 stopReason := myController.Run(testContext)
260 if !errors.Is(stopReason, context.Canceled) {
261 t.Errorf("expected error to be context.Canceled, but got: %v", stopReason)
262 }
263 }()
264
265 // Wait for controller and informer to start up
266 require.True(t, cache.WaitForCacheSync(testContext.Done(), myController.HasSynced))
267
268 // Stop the controller and informer
269 testCancel()
270
271 // Wait for controller and informer to stop
272 wg.Wait()
273
274 // Ensure the event handler was cleaned up
275 require.Empty(t, informer.registrations)
276
277 verifyNoMoreEvents()
278 }
279
280 // Show an error is thrown informer isn't started when the controller runs
281 func TestInformerNeverStarts(t *testing.T) {
282 testContext, testCancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
283 defer testCancel()
284
285 _, myController, informer, _, verifyNoMoreEvents := setupTest(testContext, nil)
286
287 wg := sync.WaitGroup{}
288
289 // Start controller
290 wg.Add(1)
291 go func() {
292 defer wg.Done()
293 stopReason := myController.Run(testContext)
294 if !errors.Is(stopReason, context.DeadlineExceeded) {
295 t.Errorf("expected error to be context.Canceled, but got: %v", stopReason)
296 }
297 }()
298
299 // Wait for deadline to pass without syncing the cache
300 require.False(t, cache.WaitForCacheSync(testContext.Done(), myController.HasSynced))
301
302 // Wait for controller to stop (or context deadline will pass quickly)
303 wg.Wait()
304
305 // Ensure there are no event handlers
306 require.Empty(t, informer.registrations)
307
308 verifyNoMoreEvents()
309 }
310
311 // Shows that if RV does not change, the reconciler does not get called
312 func TestIgnoredUpdate(t *testing.T) {
313 testContext, testCancel := context.WithTimeout(context.Background(), 2*time.Second)
314 defer testCancel()
315
316 tracker, myController, informer, waitForReconcile, verifyNoMoreEvents := setupTest(testContext, nil)
317
318 // Add object to informer
319 initialObject := &unstructured.Unstructured{}
320 initialObject.SetUnstructuredContent(map[string]interface{}{
321 "metadata": map[string]interface{}{
322 "name": "object1",
323 "resourceVersion": "1",
324 },
325 })
326 initialObject.SetGroupVersionKind(fakeGVK)
327
328 require.NoError(t, tracker.Add(initialObject))
329
330 wg := sync.WaitGroup{}
331
332 // Start informer
333 wg.Add(1)
334 go func() {
335 defer wg.Done()
336 informer.Run(testContext.Done())
337 }()
338
339 // Start controller
340 wg.Add(1)
341 go func() {
342 defer wg.Done()
343 stopReason := myController.Run(testContext)
344 if !errors.Is(stopReason, context.Canceled) {
345 t.Errorf("expected error to be context.Canceled, but got: %v", stopReason)
346 }
347 }()
348
349 // The controller is blocked because the reconcile function sends on an
350 // unbuffered channel.
351 require.False(t, myController.HasSynced())
352
353 // Wait for all enqueued reconciliations
354 require.NoError(t, waitForReconcile(initialObject))
355
356 // Now it is safe to wait for it to Sync
357 require.True(t, cache.WaitForCacheSync(testContext.Done(), myController.HasSynced))
358
359 // Send update with the same object
360 require.NoError(t, tracker.Update(fakeGVR, initialObject, ""))
361
362 // Don't wait for it to be reconciled
363
364 testCancel()
365 wg.Wait()
366
367 // TODO(alexzielenski): Find a better way to test this since the
368 // controller drops any pending events when it shuts down.
369 verifyNoMoreEvents()
370 }
371
372 // Shows that an object which fails reconciliation will retry
373 func TestReconcileRetry(t *testing.T) {
374 testContext, testCancel := context.WithTimeout(context.Background(), 2*time.Second)
375 defer testCancel()
376
377 calls := atomic.Uint64{}
378 success := atomic.Bool{}
379 tracker, myController, _, waitForReconcile, verifyNoMoreEvents := setupTest(testContext, func(s1, s2 string, o runtime.Object) error {
380
381 if calls.Add(1) > 2 {
382 // Suddenly start liking the object
383 success.Store(true)
384 return nil
385 }
386 return errors.New("i dont like this object")
387 })
388
389 // Start informer
390 wg := sync.WaitGroup{}
391
392 wg.Add(1)
393 go func() {
394 defer wg.Done()
395 myController.Informer().Run(testContext.Done())
396 }()
397
398 // Start controller
399 wg.Add(1)
400 go func() {
401 defer wg.Done()
402 stopReason := myController.Run(testContext)
403 if !errors.Is(stopReason, context.Canceled) {
404 t.Errorf("expected error to be context.Canceled, but got: %v", stopReason)
405 }
406 }()
407
408 // Add object to informer
409 initialObject := &unstructured.Unstructured{}
410 initialObject.SetUnstructuredContent(map[string]interface{}{
411 "metadata": map[string]interface{}{
412 "name": "object1",
413 "resourceVersion": "1",
414 },
415 })
416 initialObject.SetGroupVersionKind(fakeGVK)
417 require.NoError(t, tracker.Add(initialObject))
418
419 require.NoError(t, waitForReconcile(initialObject), "initial reconcile")
420 require.NoError(t, waitForReconcile(initialObject), "previous reconcile failed, should retry quickly")
421 require.NoError(t, waitForReconcile(initialObject), "previous reconcile failed, should retry quickly")
422 // Will not try again since calls > 2 for last reconcile
423 require.True(t, success.Load(), "last call to reconcile should return success")
424 testCancel()
425 wg.Wait()
426
427 verifyNoMoreEvents()
428 }
429
430 func TestInformerList(t *testing.T) {
431 testContext, testCancel := context.WithTimeout(context.Background(), 2*time.Second)
432
433 tracker, myController, _, _, _ := setupTest(testContext, nil)
434
435 wg := sync.WaitGroup{}
436
437 wg.Add(1)
438 go func() {
439 defer wg.Done()
440 myController.Informer().Run(testContext.Done())
441 }()
442
443 defer func() {
444 testCancel()
445 wg.Wait()
446 }()
447
448 require.True(t, cache.WaitForCacheSync(testContext.Done(), myController.Informer().HasSynced))
449
450 object1 := &unstructured.Unstructured{}
451 object1.SetUnstructuredContent(map[string]interface{}{
452 "metadata": map[string]interface{}{
453 "name": "object1",
454 "resourceVersion": "object1",
455 },
456 })
457 object1.SetGroupVersionKind(fakeGVK)
458
459 object1v2 := &unstructured.Unstructured{}
460 object1v2.SetUnstructuredContent(map[string]interface{}{
461 "metadata": map[string]interface{}{
462 "name": "object1",
463 "resourceVersion": "object1v2",
464 },
465 })
466 object1v2.SetGroupVersionKind(fakeGVK)
467
468 object2 := &unstructured.Unstructured{}
469 object2.SetUnstructuredContent(map[string]interface{}{
470 "metadata": map[string]interface{}{
471 "name": "object2",
472 "resourceVersion": "object2",
473 },
474 })
475 object2.SetGroupVersionKind(fakeGVK)
476
477 object3 := &unstructured.Unstructured{}
478 object3.SetUnstructuredContent(map[string]interface{}{
479 "metadata": map[string]interface{}{
480 "name": "object3",
481 "resourceVersion": "object3",
482 },
483 })
484 object3.SetGroupVersionKind(fakeGVK)
485
486 namespacedObject1 := &unstructured.Unstructured{}
487 namespacedObject1.SetUnstructuredContent(map[string]interface{}{
488 "metadata": map[string]interface{}{
489 "name": "namespacedObject1",
490 "namespace": "test",
491 "resourceVersion": "namespacedObject1",
492 },
493 })
494 namespacedObject1.SetGroupVersionKind(fakeGVK)
495
496 namespacedObject2 := &unstructured.Unstructured{}
497 namespacedObject2.SetUnstructuredContent(map[string]interface{}{
498 "metadata": map[string]interface{}{
499 "name": "namespacedObject2",
500 "namespace": "test",
501 "resourceVersion": "namespacedObject2",
502 },
503 })
504 namespacedObject2.SetGroupVersionKind(fakeGVK)
505
506 require.NoError(t, tracker.Add(object1))
507 require.NoError(t, tracker.Add(object2))
508
509 require.NoError(t, wait.PollUntilContextTimeout(testContext, 100*time.Millisecond, 500*time.Millisecond, false, func(ctx context.Context) (done bool, err error) {
510 return myController.Informer().LastSyncResourceVersion() == object2.GetResourceVersion(), nil
511 }))
512
513 values, err := myController.Informer().List(labels.Everything())
514 require.NoError(t, err)
515 require.ElementsMatch(t, []*unstructured.Unstructured{object1, object2}, values)
516
517 require.NoError(t, tracker.Update(fakeGVR, object1v2, object1v2.GetNamespace()))
518 require.NoError(t, tracker.Delete(fakeGVR, object2.GetNamespace(), object2.GetName()))
519 require.NoError(t, tracker.Add(object3))
520
521 require.NoError(t, wait.PollUntilContextTimeout(testContext, 100*time.Millisecond, 500*time.Millisecond, false, func(ctx context.Context) (done bool, err error) {
522 return myController.Informer().LastSyncResourceVersion() == object3.GetResourceVersion(), nil
523 }))
524
525 values, err = myController.Informer().List(labels.Everything())
526 require.NoError(t, err)
527 require.ElementsMatch(t, []*unstructured.Unstructured{object1v2, object3}, values)
528
529 require.NoError(t, tracker.Add(namespacedObject1))
530 require.NoError(t, tracker.Add(namespacedObject2))
531
532 require.NoError(t, wait.PollUntilContextTimeout(testContext, 100*time.Millisecond, 500*time.Millisecond, false, func(ctx context.Context) (done bool, err error) {
533 return myController.Informer().LastSyncResourceVersion() == namespacedObject2.GetResourceVersion(), nil
534 }))
535 values, err = myController.Informer().Namespaced(namespacedObject1.GetNamespace()).List(labels.Everything())
536 require.NoError(t, err)
537 require.ElementsMatch(t, []*unstructured.Unstructured{namespacedObject1, namespacedObject2}, values)
538
539 value, err := myController.Informer().Get(object3.GetName())
540 require.NoError(t, err)
541 require.Equal(t, value, object3)
542
543 value, err = myController.Informer().Namespaced(namespacedObject1.GetNamespace()).Get(namespacedObject1.GetName())
544 require.NoError(t, err)
545 require.Equal(t, value, namespacedObject1)
546
547 _, err = myController.Informer().Get("fakeobjectname")
548 require.True(t, k8serrors.IsNotFound(err))
549
550 _, err = myController.Informer().Namespaced("test").Get("fakeobjectname")
551 require.True(t, k8serrors.IsNotFound(err))
552
553 _, err = myController.Informer().Namespaced("fakenamespace").Get("fakeobjectname")
554 require.True(t, k8serrors.IsNotFound(err))
555 }