Debian Codegraph @

pkg/server/options/encryptionconfig/controller/controller_test.go

raw
1 /*
2 Copyright 2022 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package controller
18
19 import (
20 "context"
21 "fmt"
22 "net/http"
23 "strings"
24 "sync"
25 "sync/atomic"
26 "testing"
27 "time"
28
29 "k8s.io/apiserver/pkg/features"
30 "k8s.io/apiserver/pkg/server/healthz"
31 "k8s.io/apiserver/pkg/server/options/encryptionconfig"
32 utilfeature "k8s.io/apiserver/pkg/util/feature"
33 "k8s.io/client-go/util/workqueue"
34 featuregatetesting "k8s.io/component-base/featuregate/testing"
35 "k8s.io/component-base/metrics/legacyregistry"
36 "k8s.io/component-base/metrics/testutil"
37 )
38
39 func TestController(t *testing.T) {
40 origMinKMSPluginCloseGracePeriod := minKMSPluginCloseGracePeriod
41 t.Cleanup(func() { minKMSPluginCloseGracePeriod = origMinKMSPluginCloseGracePeriod })
42 minKMSPluginCloseGracePeriod = 300 * time.Millisecond
43
44 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)
45
46 const expectedSuccessMetricValue = `
47 # HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration split by apiserver identity.
48 # TYPE apiserver_encryption_config_controller_automatic_reload_success_total counter
49 apiserver_encryption_config_controller_automatic_reload_success_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027"} 1
50 # HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
51 # TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
52 apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="success"} 1
53 `
54 const expectedFailureMetricValue = `
55 # HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration split by apiserver identity.
56 # TYPE apiserver_encryption_config_controller_automatic_reload_failures_total counter
57 apiserver_encryption_config_controller_automatic_reload_failures_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027"} 1
58 # HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
59 # TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
60 apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="failure"} 1
61 `
62
63 tests := []struct {
64 name string
65 wantECFileHash string
66 wantTransformerClosed bool
67 wantLoadCalls int
68 wantHashCalls int
69 wantAddRateLimitedCount uint64
70 wantMetrics string
71 mockLoadEncryptionConfig func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error)
72 mockGetEncryptionConfigHash func(ctx context.Context, filepath string) (string, error)
73 }{
74 {
75 name: "when invalid config is provided previous config shouldn't be changed",
76 wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
77 wantLoadCalls: 1,
78 wantHashCalls: 1,
79 wantTransformerClosed: true,
80 wantMetrics: expectedFailureMetricValue,
81 wantAddRateLimitedCount: 1,
82 mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
83 return "always changes and never errors", nil
84 },
85 mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
86 return nil, fmt.Errorf("empty config file")
87 },
88 },
89 {
90 name: "when new valid config is provided it should be updated",
91 wantECFileHash: "some new config hash",
92 wantLoadCalls: 1,
93 wantHashCalls: 1,
94 wantMetrics: expectedSuccessMetricValue,
95 wantAddRateLimitedCount: 0,
96 mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
97 return "always changes and never errors", nil
98 },
99 mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
100 return &encryptionconfig.EncryptionConfiguration{
101 HealthChecks: []healthz.HealthChecker{
102 &mockHealthChecker{
103 pluginName: "valid-plugin",
104 err: nil,
105 },
106 },
107 EncryptionFileContentHash: "some new config hash",
108 }, nil
109 },
110 },
111 {
112 name: "when same valid config is provided previous config shouldn't be changed",
113 wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
114 wantLoadCalls: 1,
115 wantHashCalls: 1,
116 wantTransformerClosed: true,
117 wantMetrics: "",
118 wantAddRateLimitedCount: 0,
119 mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
120 return "always changes and never errors", nil
121 },
122 mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
123 return &encryptionconfig.EncryptionConfiguration{
124 HealthChecks: []healthz.HealthChecker{
125 &mockHealthChecker{
126 pluginName: "valid-plugin",
127 err: nil,
128 },
129 },
130 // hash of initial "testdata/ec_config.yaml" config file before reloading
131 EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
132 }, nil
133 },
134 },
135 {
136 name: "when transformer's health check fails previous config shouldn't be changed",
137 wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
138 wantLoadCalls: 1,
139 wantHashCalls: 1,
140 wantTransformerClosed: true,
141 wantMetrics: expectedFailureMetricValue,
142 wantAddRateLimitedCount: 1,
143 mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
144 return "always changes and never errors", nil
145 },
146 mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
147 return &encryptionconfig.EncryptionConfiguration{
148 HealthChecks: []healthz.HealthChecker{
149 &mockHealthChecker{
150 pluginName: "invalid-plugin",
151 err: fmt.Errorf("mockingly failing"),
152 },
153 },
154 KMSCloseGracePeriod: 0, // use minKMSPluginCloseGracePeriod
155 EncryptionFileContentHash: "anything different",
156 }, nil
157 },
158 },
159 {
160 name: "when multiple health checks are present previous config shouldn't be changed",
161 wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
162 wantLoadCalls: 1,
163 wantHashCalls: 1,
164 wantTransformerClosed: true,
165 wantMetrics: expectedFailureMetricValue,
166 wantAddRateLimitedCount: 1,
167 mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
168 return "always changes and never errors", nil
169 },
170 mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
171 return &encryptionconfig.EncryptionConfiguration{
172 HealthChecks: []healthz.HealthChecker{
173 &mockHealthChecker{
174 pluginName: "valid-plugin",
175 err: nil,
176 },
177 &mockHealthChecker{
178 pluginName: "another-valid-plugin",
179 err: nil,
180 },
181 },
182 EncryptionFileContentHash: "anything different",
183 }, nil
184 },
185 },
186 {
187 name: "when invalid health check URL is provided previous config shouldn't be changed",
188 wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
189 wantLoadCalls: 1,
190 wantHashCalls: 1,
191 wantTransformerClosed: true,
192 wantMetrics: expectedFailureMetricValue,
193 wantAddRateLimitedCount: 1,
194 mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
195 return "always changes and never errors", nil
196 },
197 mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
198 return &encryptionconfig.EncryptionConfiguration{
199 HealthChecks: []healthz.HealthChecker{
200 &mockHealthChecker{
201 pluginName: "invalid\nname",
202 err: nil,
203 },
204 },
205 EncryptionFileContentHash: "anything different",
206 }, nil
207 },
208 },
209 {
210 name: "when config is not updated transformers are closed correctly",
211 wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
212 wantLoadCalls: 1,
213 wantHashCalls: 1,
214 wantTransformerClosed: true,
215 wantMetrics: "",
216 wantAddRateLimitedCount: 0,
217 mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
218 return "always changes and never errors", nil
219 },
220 mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
221 return &encryptionconfig.EncryptionConfiguration{
222 HealthChecks: []healthz.HealthChecker{
223 &mockHealthChecker{
224 pluginName: "valid-plugin",
225 err: nil,
226 },
227 },
228 // hash of initial "testdata/ec_config.yaml" config file before reloading
229 EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
230 }, nil
231 },
232 },
233 {
234 name: "when config hash is not updated transformers are closed correctly",
235 wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
236 wantLoadCalls: 0,
237 wantHashCalls: 1,
238 wantTransformerClosed: true,
239 wantMetrics: "",
240 wantAddRateLimitedCount: 0,
241 mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
242 // hash of initial "testdata/ec_config.yaml" config file before reloading
243 return "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
244 },
245 mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
246 return nil, fmt.Errorf("should not be called")
247 },
248 },
249 {
250 name: "when config hash errors transformers are closed correctly",
251 wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
252 wantLoadCalls: 0,
253 wantHashCalls: 1,
254 wantTransformerClosed: true,
255 wantMetrics: expectedFailureMetricValue,
256 wantAddRateLimitedCount: 1,
257 mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
258 return "", fmt.Errorf("some io error")
259 },
260 mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
261 return nil, fmt.Errorf("should not be called")
262 },
263 },
264 }
265
266 for _, test := range tests {
267 t.Run(test.name, func(t *testing.T) {
268 ctxServer, closeServer := context.WithCancel(context.Background())
269 ctxTransformers, closeTransformers := context.WithCancel(ctxServer)
270 t.Cleanup(closeServer)
271 t.Cleanup(closeTransformers)
272
273 legacyregistry.Reset()
274
275 // load initial encryption config
276 encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(
277 ctxTransformers,
278 "testdata/ec_config.yaml",
279 true,
280 "test-apiserver",
281 )
282 if err != nil {
283 t.Fatalf("failed to load encryption config: %v", err)
284 }
285
286 d := NewDynamicEncryptionConfiguration(
287 "test-controller",
288 "does not matter",
289 encryptionconfig.NewDynamicTransformers(
290 encryptionConfiguration.Transformers,
291 encryptionConfiguration.HealthChecks[0],
292 closeTransformers,
293 0, // set grace period to 0 so that the time.Sleep in DynamicTransformers.Set finishes quickly
294 ),
295 encryptionConfiguration.EncryptionFileContentHash,
296 "test-apiserver",
297 )
298 d.queue.ShutDown() // we do not use the real queue during tests
299
300 queue := &mockWorkQueue{
301 addCalled: make(chan struct{}),
302 cancel: closeServer,
303 }
304 d.queue = queue
305
306 var hashCalls, loadCalls int
307 d.loadEncryptionConfig = func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
308 loadCalls++
309 queue.ctx = ctx
310 return test.mockLoadEncryptionConfig(ctx, filepath, reload, apiServerID)
311 }
312 d.getEncryptionConfigHash = func(ctx context.Context, filepath string) (string, error) {
313 hashCalls++
314 queue.ctx = ctx
315 return test.mockGetEncryptionConfigHash(ctx, filepath)
316 }
317
318 d.Run(ctxServer) // this should block and run exactly one iteration of the worker loop
319
320 if test.wantECFileHash != d.lastLoadedEncryptionConfigHash {
321 t.Errorf("expected encryption config hash %q but got %q", test.wantECFileHash, d.lastLoadedEncryptionConfigHash)
322 }
323
324 if test.wantLoadCalls != loadCalls {
325 t.Errorf("load calls does not match: want=%v, got=%v", test.wantLoadCalls, loadCalls)
326 }
327
328 if test.wantHashCalls != hashCalls {
329 t.Errorf("hash calls does not match: want=%v, got=%v", test.wantHashCalls, hashCalls)
330 }
331
332 if test.wantTransformerClosed != queue.wasCanceled {
333 t.Errorf("transformer closed does not match: want=%v, got=%v", test.wantTransformerClosed, queue.wasCanceled)
334 }
335
336 if test.wantAddRateLimitedCount != queue.addRateLimitedCount.Load() {
337 t.Errorf("queue addRateLimitedCount does not match: want=%v, got=%v", test.wantAddRateLimitedCount, queue.addRateLimitedCount.Load())
338 }
339
340 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(test.wantMetrics),
341 "apiserver_encryption_config_controller_automatic_reload_success_total",
342 "apiserver_encryption_config_controller_automatic_reload_failures_total",
343 "apiserver_encryption_config_controller_automatic_reloads_total",
344 ); err != nil {
345 t.Errorf("failed to validate metrics: %v", err)
346 }
347 })
348 }
349 }
350
351 type mockWorkQueue struct {
352 workqueue.TypedRateLimitingInterface[string] // will panic if any unexpected method is called
353
354 closeOnce sync.Once
355 addCalled chan struct{}
356
357 count atomic.Uint64
358 ctx context.Context
359 wasCanceled bool
360 cancel func()
361
362 addRateLimitedCount atomic.Uint64
363 }
364
365 func (m *mockWorkQueue) Done(item string) {
366 m.count.Add(1)
367 m.wasCanceled = m.ctx.Err() != nil
368 m.cancel()
369 }
370
371 func (m *mockWorkQueue) Get() (item string, shutdown bool) {
372 <-m.addCalled
373
374 switch m.count.Load() {
375 case 0:
376 return "", false
377 case 1:
378 return "", true
379 default:
380 panic("too many calls to Get")
381 }
382 }
383
384 func (m *mockWorkQueue) Add(item string) {
385 m.closeOnce.Do(func() {
386 close(m.addCalled)
387 })
388 }
389
390 func (m *mockWorkQueue) ShutDown() {}
391 func (m *mockWorkQueue) AddRateLimited(item string) { m.addRateLimitedCount.Add(1) }
392
393 type mockHealthChecker struct {
394 pluginName string
395 err error
396 }
397
398 func (m *mockHealthChecker) Check(req *http.Request) error {
399 return m.err
400 }
401
402 func (m *mockHealthChecker) Name() string {
403 return m.pluginName
404 }