Debian Codegraph @

pkg/server/options/encryptionconfig/config.go

raw
1 /*
2 Copyright 2017 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package encryptionconfig
18
19 import (
20 "context"
21 "crypto/aes"
22 "crypto/cipher"
23 "crypto/sha256"
24 "encoding/base64"
25 "errors"
26 "fmt"
27 "net/http"
28 "os"
29 "sync"
30 "sync/atomic"
31 "time"
32
33 "k8s.io/apimachinery/pkg/runtime"
34 "k8s.io/apimachinery/pkg/runtime/schema"
35 "k8s.io/apimachinery/pkg/runtime/serializer"
36 utilerrors "k8s.io/apimachinery/pkg/util/errors"
37 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
38 "k8s.io/apimachinery/pkg/util/uuid"
39 "k8s.io/apimachinery/pkg/util/wait"
40 "k8s.io/apiserver/pkg/apis/apiserver"
41 apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
42 "k8s.io/apiserver/pkg/apis/apiserver/validation"
43 "k8s.io/apiserver/pkg/endpoints/request"
44 "k8s.io/apiserver/pkg/features"
45 "k8s.io/apiserver/pkg/server/healthz"
46 "k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
47 storagevalue "k8s.io/apiserver/pkg/storage/value"
48 aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
49 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
50 envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
51 kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
52 envelopemetrics "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
53 "k8s.io/apiserver/pkg/storage/value/encrypt/identity"
54 "k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
55 utilfeature "k8s.io/apiserver/pkg/util/feature"
56 "k8s.io/klog/v2"
57 kmsservice "k8s.io/kms/pkg/service"
58 )
59
60 const (
61 aesCBCTransformerPrefixV1 = "k8s:enc:aescbc:v1:"
62 aesGCMTransformerPrefixV1 = "k8s:enc:aesgcm:v1:"
63 secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:"
64 kmsTransformerPrefixV1 = "k8s:enc:kms:v1:"
65 kmsTransformerPrefixV2 = "k8s:enc:kms:v2:"
66
67 // these constants relate to how the KMS v2 plugin status poll logic
68 // and the DEK/seed generation logic behave. In particular, the positive
69 // interval and max TTL are closely related as the difference between
70 // these values defines the worst case window in which the write DEK/seed
71 // could expire due to the plugin going into an error state. The
72 // worst case window divided by the negative interval defines the
73 // minimum amount of times the server will attempt to return to a
74 // healthy state before the DEK/seed expires and writes begin to fail.
75 //
76 // For now, these values are kept small and hardcoded to support being
77 // able to perform a "passive" storage migration while tolerating some
78 // amount of plugin downtime.
79 //
80 // With the current approach, a user can update the key ID their plugin
81 // is using and then can simply schedule a migration for 3 + N + M minutes
82 // later where N is how long it takes their plugin to pick up new config
83 // and M is extra buffer to allow the API server to process the config.
84 // At that point, they are guaranteed to either migrate to the new key
85 // or get errors during the migration.
86 //
87 // If the API server coasted forever on the last DEK/seed, they would need
88 // to actively check if it had observed the new key ID before starting
89 // a migration - otherwise it could keep using the old DEK/seed and their
90 // storage migration would not do what they thought it did.
91 kmsv2PluginHealthzPositiveInterval = 1 * time.Minute
92 kmsv2PluginHealthzNegativeInterval = 10 * time.Second
93 kmsv2PluginWriteDEKSourceMaxTTL = 3 * time.Minute
94
95 kmsPluginHealthzNegativeTTL = 3 * time.Second
96 kmsPluginHealthzPositiveTTL = 20 * time.Second
97 kmsAPIVersionV1 = "v1"
98 kmsAPIVersionV2 = "v2"
99 // this name is used for two different healthz endpoints:
100 // - when one or more KMS v2 plugins are in use and no KMS v1 plugins are in use
101 // in this case, all v2 plugins are probed via this single endpoint
102 // - when automatic reload of encryption config is enabled
103 // in this case, all KMS plugins are probed via this single endpoint
104 // the endpoint is present even if there are no KMS plugins configured (it is a no-op then)
105 kmsReloadHealthCheckName = "kms-providers"
106 )
107
108 var codecs serializer.CodecFactory
109
110 // this map allows us to swap enablement of the KMSv2KDF feature in tests
111 // as the feature gate is now locked to true starting with v1.29
112 // Note: it cannot be set by an end user
113 // KDF enablement is tracked per KMS provider to allow tests to run in parallel.
114 var kdfEnabledPerKMS sync.Map // map[string]bool, KMS name -> KDF enabled
115
116 // this function should only be called in tests to swap enablement of the KMSv2KDF feature
117 // Caller must guarantee that all KMS providers have distinct names across all tests.
118 func SetKDFForTests(kmsName string, b bool) func() {
119 if len(kmsName) == 0 { // guarantee that GetKDF("") returns the default value
120 panic("empty KMS name used in test")
121 }
122 if _, loaded := kdfEnabledPerKMS.LoadOrStore(kmsName, b); loaded {
123 panic("duplicate KMS name used in test")
124 }
125 return func() { kdfEnabledPerKMS.Delete(kmsName) }
126 }
127
128 // this function should be used to determine enablement of the KMSv2KDF feature
129 // instead of getting it from DefaultFeatureGate as the feature gate is now locked
130 // to true starting with v1.29
131 // to allow integration tests to run in parallel, this "feature flag" can be set
132 // per KMS provider as long as all providers use distinct names.
133 func GetKDF(kmsName string) bool {
134 kdfEnabled, ok := kdfEnabledPerKMS.Load(kmsName)
135 if !ok {
136 return true // explicit config is missing, but KDF is enabled by default
137 }
138 return kdfEnabled.(bool) // this will panic if a non-bool ever gets stored, which should never happen
139 }
140
141 func init() {
142 configScheme := runtime.NewScheme()
143 utilruntime.Must(apiserver.AddToScheme(configScheme))
144 utilruntime.Must(apiserverv1.AddToScheme(configScheme))
145 codecs = serializer.NewCodecFactory(configScheme, serializer.EnableStrict)
146 envelopemetrics.RegisterMetrics()
147 storagevalue.RegisterMetrics()
148 metrics.RegisterMetrics()
149 }
150
151 type kmsPluginHealthzResponse struct {
152 err error
153 received time.Time
154 }
155
156 type kmsPluginProbe struct {
157 name string
158 ttl time.Duration
159 service envelope.Service
160 lastResponse *kmsPluginHealthzResponse
161 l *sync.Mutex
162 }
163
164 type kmsv2PluginProbe struct {
165 state atomic.Pointer[envelopekmsv2.State]
166 name string
167 ttl time.Duration
168 service kmsservice.Service
169 lastResponse *kmsPluginHealthzResponse
170 l *sync.Mutex
171 apiServerID string
172 version string
173 }
174
175 type kmsHealthChecker []healthz.HealthChecker
176
177 func (k kmsHealthChecker) Name() string {
178 return kmsReloadHealthCheckName
179 }
180
181 func (k kmsHealthChecker) Check(req *http.Request) error {
182 var errs []error
183
184 for i := range k {
185 checker := k[i]
186 if err := checker.Check(req); err != nil {
187 errs = append(errs, fmt.Errorf("%s: %w", checker.Name(), err))
188 }
189 }
190
191 return utilerrors.Reduce(utilerrors.NewAggregate(errs))
192 }
193
194 func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
195 return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
196 return h.check()
197 })
198 }
199
200 func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
201 return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
202 return h.check(r.Context())
203 })
204 }
205
206 // EncryptionConfiguration represents the parsed and normalized encryption configuration for the apiserver.
207 type EncryptionConfiguration struct {
208 // Transformers is a list of value.Transformer that will be used to encrypt and decrypt data.
209 Transformers map[schema.GroupResource]storagevalue.Transformer
210
211 // HealthChecks is a list of healthz.HealthChecker that will be used to check the health of the encryption providers.
212 HealthChecks []healthz.HealthChecker
213
214 // EncryptionFileContentHash is the hash of the encryption config file.
215 EncryptionFileContentHash string
216
217 // KMSCloseGracePeriod is the duration we will wait before closing old transformers.
218 // We wait for any in-flight requests to finish by using the duration which is longer than their timeout.
219 KMSCloseGracePeriod time.Duration
220 }
221
222 // LoadEncryptionConfig parses and validates the encryption config specified by filepath.
223 // It may launch multiple go routines whose lifecycle is controlled by ctx.
224 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
225 // If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
226 func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool, apiServerID string) (*EncryptionConfiguration, error) {
227 config, contentHash, err := loadConfig(filepath, reload)
228 if err != nil {
229 return nil, fmt.Errorf("error while parsing file: %w", err)
230 }
231
232 transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config, apiServerID)
233 if err != nil {
234 return nil, fmt.Errorf("error while building transformers: %w", err)
235 }
236
237 if reload || (kmsUsed.v2Used && !kmsUsed.v1Used) {
238 kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)}
239 }
240
241 // KMSCloseGracePeriod is the duration we will wait before closing old transformers.
242 // The way we calculate is as follows:
243 // 1. Sum all timeouts across all KMS plugins. (check kmsPrefixTransformer for differences between v1 and v2)
244 // 2. Multiply that by 2 (to allow for some buffer)
245 // The reason we sum all timeout is because kmsHealthChecker() will run all health checks serially
246 return &EncryptionConfiguration{
247 Transformers: transformers,
248 HealthChecks: kmsHealthChecks,
249 EncryptionFileContentHash: contentHash,
250 KMSCloseGracePeriod: 2 * kmsUsed.kmsTimeoutSum,
251 }, nil
252 }
253
254 // getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config.
255 // It may launch multiple go routines whose lifecycle is controlled by ctx.
256 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
257 func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserver.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
258 var kmsHealthChecks []healthz.HealthChecker
259 transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, apiServerID)
260 if err != nil {
261 return nil, nil, nil, err
262 }
263 for i := range probes {
264 probe := probes[i]
265 kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i))
266 }
267
268 return transformers, kmsHealthChecks, kmsUsed, nil
269 }
270
271 type healthChecker interface {
272 toHealthzCheck(idx int) healthz.HealthChecker
273 }
274
275 // getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config.
276 // It may launch multiple go routines whose lifecycle is controlled by ctx.
277 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
278 func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserver.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
279 resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{}
280 var probes []healthChecker
281 var kmsUsed kmsState
282
283 // For each entry in the configuration
284 for _, resourceConfig := range config.Resources {
285 resourceConfig := resourceConfig
286
287 transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig, apiServerID)
288 if err != nil {
289 return nil, nil, nil, err
290 }
291 kmsUsed.accumulate(used)
292
293 // For each resource, create a list of providers to use
294 for _, resource := range resourceConfig.Resources {
295 resource := resource
296 gr := schema.ParseGroupResource(resource)
297
298 // check if resource is masked by *.group rule
299 anyResourceInGroup := schema.GroupResource{Group: gr.Group, Resource: "*"}
300 if _, masked := resourceToPrefixTransformer[anyResourceInGroup]; masked {
301 // an earlier rule already configured a transformer for *.group, masking this rule
302 // return error since this is not allowed
303 return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyResourceInGroup))
304 }
305
306 if _, masked := resourceToPrefixTransformer[anyGroupAnyResource]; masked {
307 // an earlier rule already configured a transformer for *.*, masking this rule
308 // return error since this is not allowed
309 return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyGroupAnyResource))
310 }
311
312 resourceToPrefixTransformer[gr] = append(resourceToPrefixTransformer[gr], transformers...)
313 }
314
315 probes = append(probes, p...)
316 }
317
318 transformers := make(map[schema.GroupResource]storagevalue.Transformer, len(resourceToPrefixTransformer))
319 for gr, transList := range resourceToPrefixTransformer {
320 gr := gr
321 transList := transList
322 transformers[gr] = storagevalue.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
323 }
324
325 return transformers, probes, &kmsUsed, nil
326 }
327
328 // check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
329 func (h *kmsPluginProbe) check() error {
330 h.l.Lock()
331 defer h.l.Unlock()
332
333 if (time.Since(h.lastResponse.received)) < h.ttl {
334 return h.lastResponse.err
335 }
336
337 p, err := h.service.Encrypt([]byte("ping"))
338 if err != nil {
339 h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
340 h.ttl = kmsPluginHealthzNegativeTTL
341 return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
342 }
343
344 if _, err := h.service.Decrypt(p); err != nil {
345 h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
346 h.ttl = kmsPluginHealthzNegativeTTL
347 return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
348 }
349
350 h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
351 h.ttl = kmsPluginHealthzPositiveTTL
352 return nil
353 }
354
355 // check gets the healthz status of the KMSv2-Plugin using the Status() method.
356 func (h *kmsv2PluginProbe) check(ctx context.Context) error {
357 h.l.Lock()
358 defer h.l.Unlock()
359
360 if time.Since(h.lastResponse.received) < h.ttl {
361 return h.lastResponse.err
362 }
363
364 p, err := h.service.Status(ctx)
365 if err != nil {
366 h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
367 h.ttl = kmsPluginHealthzNegativeTTL
368 return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %w", h.name, err)
369 }
370
371 if err := h.isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx, p); err != nil {
372 h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
373 h.ttl = kmsPluginHealthzNegativeTTL
374 return err
375 }
376
377 h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
378 h.ttl = kmsPluginHealthzPositiveTTL
379 return nil
380 }
381
382 // rotateDEKOnKeyIDChange tries to rotate to a new DEK/seed if the key ID returned by Status does not match the
383 // current state. If a successful rotation is performed, the new DEK/seed and keyID overwrite the existing state.
384 // On any failure during rotation (including mismatch between status and encrypt calls), the current state is
385 // preserved and will remain valid to use for encryption until its expiration (the system attempts to coast).
386 // If the key ID returned by Status matches the current state, the expiration of the current state is extended
387 // and no rotation is performed.
388 func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKeyID, uid string) error {
389 // we do not check ValidateEncryptCapability here because it is fine to re-use an old key
390 // that was marked as expired during an unhealthy period. As long as the key ID matches
391 // what we expect then there is no need to rotate here.
392 state, errState := h.getCurrentState()
393
394 // allow reads indefinitely in all cases
395 // allow writes indefinitely as long as there is no error
396 // allow writes for only up to kmsv2PluginWriteDEKSourceMaxTTL from now when there are errors
397 // we start the timer before we make the network call because kmsv2PluginWriteDEKSourceMaxTTL is meant to be the upper bound
398 expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKSourceMaxTTL)
399
400 // dynamically check if we want to use KDF seed to derive DEKs or just a single DEK
401 // this gate can only change during tests, but the check is cheap enough to always make
402 // this allows us to easily exercise both modes without restarting the API server
403 // TODO integration test that this dynamically takes effect
404 useSeed := GetKDF(h.name)
405 stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
406
407 // state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
408 // just move the expiration of the current state forward by the reuse interval
409 // useSeed can only change at runtime during tests, so we check it here to allow us to easily exercise both modes
410 if errState == nil && state.EncryptedObject.KeyID == statusKeyID && stateUseSeed == useSeed {
411 state.ExpirationTimestamp = expirationTimestamp
412 h.state.Store(&state)
413 return nil
414 }
415
416 transformer, encObject, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service, useSeed)
417
418 if encObject == nil {
419 encObject = &kmstypes.EncryptedObject{} // avoid nil panics
420 }
421
422 // happy path, should be the common case
423 // TODO maybe add success metrics?
424 if errGen == nil && encObject.KeyID == statusKeyID {
425 h.state.Store(&envelopekmsv2.State{
426 Transformer: transformer,
427 EncryptedObject: *encObject,
428 UID: uid,
429 ExpirationTimestamp: expirationTimestamp,
430 CacheKey: cacheKey,
431 })
432
433 // it should be logically impossible for the new state to be invalid but check just in case
434 _, errGen = h.getCurrentState()
435 if errGen == nil {
436 klogV6 := klog.V(6)
437 if klogV6.Enabled() {
438 klogV6.InfoS("successfully rotated DEK",
439 "uid", uid,
440 "useSeed", useSeed,
441 "newKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID),
442 "oldKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID),
443 "expirationTimestamp", expirationTimestamp.Format(time.RFC3339),
444 )
445 }
446 return nil
447 }
448 }
449
450 return fmt.Errorf("failed to rotate DEK uid=%q, useSeed=%v, errState=%v, errGen=%v, statusKeyIDHash=%q, encryptKeyIDHash=%q, stateKeyIDHash=%q, expirationTimestamp=%s",
451 uid, useSeed, errState, errGen, envelopekmsv2.GetHashIfNotEmpty(statusKeyID), envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID), envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID), state.ExpirationTimestamp.Format(time.RFC3339))
452 }
453
454 // getCurrentState returns the latest state from the last status and encrypt calls.
455 // If the returned error is nil, the state is considered valid indefinitely for read requests.
456 // For write requests, the caller must also check that state.ValidateEncryptCapability does not error.
457 func (h *kmsv2PluginProbe) getCurrentState() (envelopekmsv2.State, error) {
458 state := *h.state.Load()
459
460 if state.Transformer == nil {
461 return envelopekmsv2.State{}, fmt.Errorf("got unexpected nil transformer")
462 }
463
464 encryptedObjectCopy := state.EncryptedObject
465 if len(encryptedObjectCopy.EncryptedData) != 0 {
466 return envelopekmsv2.State{}, fmt.Errorf("got unexpected non-empty EncryptedData")
467 }
468 encryptedObjectCopy.EncryptedData = []byte{0} // any non-empty value to pass validation
469 if err := envelopekmsv2.ValidateEncryptedObject(&encryptedObjectCopy); err != nil {
470 return envelopekmsv2.State{}, fmt.Errorf("got invalid EncryptedObject: %w", err)
471 }
472
473 if state.ExpirationTimestamp.IsZero() {
474 return envelopekmsv2.State{}, fmt.Errorf("got unexpected zero expirationTimestamp")
475 }
476
477 if len(state.CacheKey) == 0 {
478 return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty cacheKey")
479 }
480
481 return state, nil
482 }
483
484 func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.Context, response *kmsservice.StatusResponse) error {
485 var errs []error
486 if response.Healthz != "ok" {
487 errs = append(errs, fmt.Errorf("got unexpected healthz status: %s", response.Healthz))
488 }
489 if response.Version != envelopekmsv2.KMSAPIVersionv2 && response.Version != envelopekmsv2.KMSAPIVersionv2beta1 {
490 errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersionv2, response.Version))
491 } else {
492 // set version for the first status response
493 if len(h.version) == 0 {
494 h.version = response.Version
495 }
496 if h.version != response.Version {
497 errs = append(errs, fmt.Errorf("KMSv2 API version should not change after the initial status response version %s, got %s", h.version, response.Version))
498 }
499 }
500
501 if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil {
502 envelopemetrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode))
503 errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err))
504 } else {
505 envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID, h.apiServerID)
506 // unconditionally append as we filter out nil errors below
507 errs = append(errs, h.rotateDEKOnKeyIDChange(ctx, response.KeyID, string(uuid.NewUUID())))
508 }
509
510 if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
511 return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", h.name, err)
512 }
513 return nil
514 }
515
516 // loadConfig parses the encryption configuration file at filepath and returns the parsed config and hash of the file.
517 func loadConfig(filepath string, reload bool) (*apiserver.EncryptionConfiguration, string, error) {
518 data, contentHash, err := loadDataAndHash(filepath)
519 if err != nil {
520 return nil, "", fmt.Errorf("error while loading file: %w", err)
521 }
522
523 configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
524 if err != nil {
525 return nil, "", fmt.Errorf("error decoding encryption provider configuration file %q: %w", filepath, err)
526 }
527 config, ok := configObj.(*apiserver.EncryptionConfiguration)
528 if !ok {
529 return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
530 }
531
532 return config, contentHash, validation.ValidateEncryptionConfiguration(config, reload).ToAggregate()
533 }
534
535 func loadDataAndHash(filepath string) ([]byte, string, error) {
536 data, err := os.ReadFile(filepath)
537 if err != nil {
538 return nil, "", fmt.Errorf("error reading encryption provider configuration file %q: %w", filepath, err)
539 }
540 if len(data) == 0 {
541 return nil, "", fmt.Errorf("encryption provider configuration file %q is empty", filepath)
542 }
543
544 return data, computeEncryptionConfigHash(data), nil
545 }
546
547 // GetEncryptionConfigHash reads the encryption configuration file at filepath and returns the hash of the file.
548 // It does not attempt to decode or load the config, and serves as a cheap check to determine if the file has changed.
549 func GetEncryptionConfigHash(filepath string) (string, error) {
550 _, contentHash, err := loadDataAndHash(filepath)
551 return contentHash, err
552 }
553
554 // prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config.
555 // It may launch multiple go routines whose lifecycle is controlled by ctx.
556 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
557 func prefixTransformersAndProbes(ctx context.Context, config apiserver.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
558 var transformers []storagevalue.PrefixTransformer
559 var probes []healthChecker
560 var kmsUsed kmsState
561
562 for _, provider := range config.Providers {
563 provider := provider
564 var (
565 transformer storagevalue.PrefixTransformer
566 transformerErr error
567 probe healthChecker
568 used *kmsState
569 )
570
571 switch {
572 case provider.AESGCM != nil:
573 transformer, transformerErr = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
574
575 case provider.AESCBC != nil:
576 cbcTransformer := func(block cipher.Block) (storagevalue.Transformer, error) {
577 return aestransformer.NewCBCTransformer(block), nil
578 }
579 transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, cbcTransformer, aesCBCTransformerPrefixV1)
580
581 case provider.Secretbox != nil:
582 transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
583
584 case provider.KMS != nil:
585 transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS, apiServerID)
586 if transformerErr == nil {
587 probes = append(probes, probe)
588 kmsUsed.accumulate(used)
589 }
590
591 case provider.Identity != nil:
592 transformer = storagevalue.PrefixTransformer{
593 Transformer: identity.NewEncryptCheckTransformer(),
594 Prefix: []byte{},
595 }
596
597 default:
598 return nil, nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
599 }
600
601 if transformerErr != nil {
602 return nil, nil, nil, transformerErr
603 }
604
605 transformers = append(transformers, transformer)
606 }
607
608 return transformers, probes, &kmsUsed, nil
609 }
610
611 type blockTransformerFunc func(cipher.Block) (storagevalue.Transformer, error)
612
613 func aesPrefixTransformer(config *apiserver.AESConfiguration, fn blockTransformerFunc, prefix string) (storagevalue.PrefixTransformer, error) {
614 var result storagevalue.PrefixTransformer
615
616 if len(config.Keys) == 0 {
617 return result, fmt.Errorf("aes provider has no valid keys")
618 }
619 for _, key := range config.Keys {
620 key := key
621 if key.Name == "" {
622 return result, fmt.Errorf("key with invalid name provided")
623 }
624 if key.Secret == "" {
625 return result, fmt.Errorf("key %v has no provided secret", key.Name)
626 }
627 }
628
629 keyTransformers := []storagevalue.PrefixTransformer{}
630
631 for _, keyData := range config.Keys {
632 keyData := keyData
633 key, err := base64.StdEncoding.DecodeString(keyData.Secret)
634 if err != nil {
635 return result, fmt.Errorf("could not obtain secret for named key %s: %w", keyData.Name, err)
636 }
637 block, err := aes.NewCipher(key)
638 if err != nil {
639 return result, fmt.Errorf("error while creating cipher for named key %s: %w", keyData.Name, err)
640 }
641 transformer, err := fn(block)
642 if err != nil {
643 return result, fmt.Errorf("error while creating transformer for named key %s: %w", keyData.Name, err)
644 }
645
646 // Create a new PrefixTransformer for this key
647 keyTransformers = append(keyTransformers,
648 storagevalue.PrefixTransformer{
649 Transformer: transformer,
650 Prefix: []byte(keyData.Name + ":"),
651 })
652 }
653
654 // Create a prefixTransformer which can choose between these keys
655 keyTransformer := storagevalue.NewPrefixTransformers(
656 fmt.Errorf("no matching key was found for the provided AES transformer"), keyTransformers...)
657
658 // Create a PrefixTransformer which shall later be put in a list with other providers
659 result = storagevalue.PrefixTransformer{
660 Transformer: keyTransformer,
661 Prefix: []byte(prefix),
662 }
663 return result, nil
664 }
665
666 func secretboxPrefixTransformer(config *apiserver.SecretboxConfiguration) (storagevalue.PrefixTransformer, error) {
667 var result storagevalue.PrefixTransformer
668
669 if len(config.Keys) == 0 {
670 return result, fmt.Errorf("secretbox provider has no valid keys")
671 }
672 for _, key := range config.Keys {
673 key := key
674 if key.Name == "" {
675 return result, fmt.Errorf("key with invalid name provided")
676 }
677 if key.Secret == "" {
678 return result, fmt.Errorf("key %v has no provided secret", key.Name)
679 }
680 }
681
682 keyTransformers := []storagevalue.PrefixTransformer{}
683
684 for _, keyData := range config.Keys {
685 keyData := keyData
686 key, err := base64.StdEncoding.DecodeString(keyData.Secret)
687 if err != nil {
688 return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
689 }
690
691 if len(key) != 32 {
692 return result, fmt.Errorf("expected key size 32 for secretbox provider, got %v", len(key))
693 }
694
695 keyArray := [32]byte{}
696 copy(keyArray[:], key)
697
698 // Create a new PrefixTransformer for this key
699 keyTransformers = append(keyTransformers,
700 storagevalue.PrefixTransformer{
701 Transformer: secretbox.NewSecretboxTransformer(keyArray),
702 Prefix: []byte(keyData.Name + ":"),
703 })
704 }
705
706 // Create a prefixTransformer which can choose between these keys
707 keyTransformer := storagevalue.NewPrefixTransformers(
708 fmt.Errorf("no matching key was found for the provided Secretbox transformer"), keyTransformers...)
709
710 // Create a PrefixTransformer which shall later be put in a list with other providers
711 result = storagevalue.PrefixTransformer{
712 Transformer: keyTransformer,
713 Prefix: []byte(secretboxTransformerPrefixV1),
714 }
715 return result, nil
716 }
717
718 var (
719 // The factory to create kms service. This is to make writing test easier.
720 envelopeServiceFactory = envelope.NewGRPCService
721
722 // The factory to create kmsv2 service. Exported for integration tests.
723 EnvelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService
724 )
725
726 type kmsState struct {
727 v1Used, v2Used bool
728 kmsTimeoutSum time.Duration
729 }
730
731 // accumulate computes the KMS state by:
732 // - determining which KMS plugin versions are in use
733 // - calculating kmsTimeoutSum which is used as transformTracker.kmsCloseGracePeriod
734 // DynamicTransformers.Set waits for this period before closing old transformers after a config reload
735 func (s *kmsState) accumulate(other *kmsState) {
736 s.v1Used = s.v1Used || other.v1Used
737 s.v2Used = s.v2Used || other.v2Used
738 s.kmsTimeoutSum += other.kmsTimeoutSum
739 }
740
741 // kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config.
742 // It may launch multiple go routines whose lifecycle is controlled by ctx.
743 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
744 func kmsPrefixTransformer(ctx context.Context, config *apiserver.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
745 kmsName := config.Name
746 switch config.APIVersion {
747 case kmsAPIVersionV1:
748 if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) {
749 return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead. Set --feature-gates=KMSv1=true to use the deprecated KMSv1 feature.")
750 }
751 klog.InfoS("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead.")
752
753 envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
754 if err != nil {
755 return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
756 }
757
758 probe := &kmsPluginProbe{
759 name: kmsName,
760 ttl: kmsPluginHealthzNegativeTTL,
761 service: envelopeService,
762 l: &sync.Mutex{},
763 lastResponse: &kmsPluginHealthzResponse{},
764 }
765
766 transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1)
767
768 return transformer, probe, &kmsState{
769 v1Used: true,
770 // for v1 we will do encrypt and decrypt for health check. Since these are serial operations, we will double the timeout.
771 kmsTimeoutSum: 2 * config.Timeout.Duration,
772 }, nil
773
774 case kmsAPIVersionV2:
775 envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Name, config.Timeout.Duration)
776 if err != nil {
777 return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
778 }
779
780 probe := &kmsv2PluginProbe{
781 name: kmsName,
782 ttl: kmsPluginHealthzNegativeTTL,
783 service: envelopeService,
784 l: &sync.Mutex{},
785 lastResponse: &kmsPluginHealthzResponse{},
786 apiServerID: apiServerID,
787 }
788 // initialize state so that Load always works
789 probe.state.Store(&envelopekmsv2.State{})
790
791 primeAndProbeKMSv2(ctx, probe, kmsName)
792 transformer := storagevalue.PrefixTransformer{
793 Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState, apiServerID),
794 Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"),
795 }
796
797 return transformer, probe, &kmsState{
798 v2Used: true,
799 kmsTimeoutSum: config.Timeout.Duration,
800 }, nil
801
802 default:
803 return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
804 }
805 }
806
807 func primeAndProbeKMSv2(ctx context.Context, probe *kmsv2PluginProbe, kmsName string) {
808 runProbeCheckAndLog := func(ctx context.Context, depth int) error {
809 if err := probe.check(ctx); err != nil {
810 klog.VDepth(1+depth, 2).ErrorS(err, "kms plugin failed health check probe", "name", kmsName)
811 return err
812 }
813 return nil
814 }
815
816 blockAndProbeFastUntilSuccess := func(ctx context.Context) {
817 _ = wait.PollUntilWithContext(
818 ctx,
819 kmsv2PluginHealthzNegativeInterval,
820 func(ctx context.Context) (bool, error) {
821 return runProbeCheckAndLog(ctx, 1) == nil, nil
822 },
823 )
824 }
825
826 // on the happy path where the plugin is healthy and available on server start,
827 // prime keyID and DEK by running the check inline once (this also prevents unit tests from flaking)
828 errPrime := runProbeCheckAndLog(ctx, 0)
829
830 // if our initial attempt to prime failed, start trying to get to a valid state in the background ASAP
831 // this prevents a slow start when the external healthz checker is configured to ignore the KMS healthz endpoint
832 // since we want to support the plugin starting up async with the API server, this error is not fatal
833 if errPrime != nil {
834 go blockAndProbeFastUntilSuccess(ctx) // separate go routine to avoid blocking
835 }
836
837 // make sure that the plugin's key ID is reasonably up-to-date
838 // also, make sure that our DEK is up-to-date to with said key ID (if it expires the server will fail all writes)
839 // if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKSourceMaxTTL
840 go wait.PollUntilWithContext(
841 ctx,
842 kmsv2PluginHealthzPositiveInterval,
843 func(ctx context.Context) (bool, error) {
844 if err := runProbeCheckAndLog(ctx, 0); err == nil {
845 return false, nil
846 }
847
848 // TODO add integration test for quicker error poll on failure
849 // if we fail, block the outer polling and start a new quicker poll inline
850 // this limits the chance that our DEK expires during a transient failure
851 blockAndProbeFastUntilSuccess(ctx)
852
853 return false, nil
854 })
855 }
856
857 func envelopePrefixTransformer(config *apiserver.KMSConfiguration, envelopeService envelope.Service, prefix string) storagevalue.PrefixTransformer {
858 baseTransformerFunc := func(block cipher.Block) (storagevalue.Transformer, error) {
859 gcm, err := aestransformer.NewGCMTransformer(block)
860 if err != nil {
861 return nil, err
862 }
863
864 // v1.24: write using AES-CBC only but support reads via AES-CBC and AES-GCM (so we can move to AES-GCM)
865 // v1.25: write using AES-GCM only but support reads via AES-GCM and fallback to AES-CBC for backwards compatibility
866 // TODO(aramase): Post v1.25: We cannot drop CBC read support until we automate storage migration.
867 // We could have a release note that hard requires users to perform storage migration.
868 return unionTransformers{gcm, aestransformer.NewCBCTransformer(block)}, nil
869 }
870
871 return storagevalue.PrefixTransformer{
872 Transformer: envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc),
873 Prefix: []byte(prefix + config.Name + ":"),
874 }
875 }
876
877 type unionTransformers []storagevalue.Transformer
878
879 func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, stale bool, err error) {
880 var errs []error
881 for i := range u {
882 transformer := u[i]
883 result, stale, err := transformer.TransformFromStorage(ctx, data, dataCtx)
884 if err != nil {
885 errs = append(errs, err)
886 continue
887 }
888 // when i != 0, we have transformed the data from storage using the new transformer,
889 // we want to issue a write to etcd even if the contents of the data haven't changed
890 return result, stale || i != 0, nil
891 }
892 if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
893 return nil, false, err
894 }
895 return nil, false, fmt.Errorf("unionTransformers: unable to transform from storage")
896 }
897
898 func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, err error) {
899 return u[0].TransformToStorage(ctx, data, dataCtx)
900 }
901
902 // computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes.
903 // We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers.
904 // This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees.
905 func computeEncryptionConfigHash(data []byte) string {
906 return fmt.Sprintf("k8s:enc:unstable:1:%x", sha256.Sum256(data))
907 }
908
909 var _ storagevalue.ResourceTransformers = &DynamicTransformers{}
910 var _ healthz.HealthChecker = &DynamicTransformers{}
911
912 // DynamicTransformers holds transformers that may be dynamically updated via a single external actor, likely a controller.
913 // This struct must avoid locks (even read write locks) as it is inline to all calls to storage.
914 type DynamicTransformers struct {
915 transformTracker *atomic.Value
916 }
917
918 type transformTracker struct {
919 transformerOverrides map[schema.GroupResource]storagevalue.Transformer
920 kmsPluginHealthzCheck healthz.HealthChecker
921 closeTransformers context.CancelFunc
922 kmsCloseGracePeriod time.Duration
923 }
924
925 // NewDynamicTransformers returns transformers, health checks for kms providers and an ability to close transformers.
926 func NewDynamicTransformers(
927 transformerOverrides map[schema.GroupResource]storagevalue.Transformer,
928 kmsPluginHealthzCheck healthz.HealthChecker,
929 closeTransformers context.CancelFunc,
930 kmsCloseGracePeriod time.Duration,
931 ) *DynamicTransformers {
932 dynamicTransformers := &DynamicTransformers{
933 transformTracker: &atomic.Value{},
934 }
935
936 tracker := &transformTracker{
937 transformerOverrides: transformerOverrides,
938 kmsPluginHealthzCheck: kmsPluginHealthzCheck,
939 closeTransformers: closeTransformers,
940 kmsCloseGracePeriod: kmsCloseGracePeriod,
941 }
942 dynamicTransformers.transformTracker.Store(tracker)
943
944 return dynamicTransformers
945 }
946
947 // Check implements healthz.HealthChecker
948 func (d *DynamicTransformers) Check(req *http.Request) error {
949 return d.transformTracker.Load().(*transformTracker).kmsPluginHealthzCheck.Check(req)
950 }
951
952 // Name implements healthz.HealthChecker
953 func (d *DynamicTransformers) Name() string {
954 return kmsReloadHealthCheckName
955 }
956
957 // TransformerForResource returns the transformer for the given resource.
958 func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer {
959 return &resourceTransformer{
960 resource: resource,
961 transformTracker: d.transformTracker,
962 }
963 }
964
965 // Set sets the transformer overrides. This method is not go routine safe and must only be called by the same, single caller throughout the lifetime of this object.
966 func (d *DynamicTransformers) Set(
967 transformerOverrides map[schema.GroupResource]storagevalue.Transformer,
968 closeTransformers context.CancelFunc,
969 kmsPluginHealthzCheck healthz.HealthChecker,
970 kmsCloseGracePeriod time.Duration,
971 ) {
972 // store new values
973 newTransformTracker := &transformTracker{
974 transformerOverrides: transformerOverrides,
975 closeTransformers: closeTransformers,
976 kmsPluginHealthzCheck: kmsPluginHealthzCheck,
977 kmsCloseGracePeriod: kmsCloseGracePeriod,
978 }
979
980 // update new transformer overrides
981 oldTransformTracker := d.transformTracker.Swap(newTransformTracker).(*transformTracker)
982
983 // close old transformers once we wait for grpc request to finish any in-flight requests.
984 // by the time we spawn this go routine, the new transformers have already been set and will be used for new requests.
985 // if the server starts shutting down during sleep duration then the transformers will correctly closed early because their lifetime is tied to the api-server drain notifier.
986 go func() {
987 time.Sleep(oldTransformTracker.kmsCloseGracePeriod)
988 oldTransformTracker.closeTransformers()
989 }()
990 }
991
992 var _ storagevalue.Transformer = &resourceTransformer{}
993
994 type resourceTransformer struct {
995 resource schema.GroupResource
996 transformTracker *atomic.Value
997 }
998
999 func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, bool, error) {
1000 return r.transformer().TransformFromStorage(ctx, data, dataCtx)
1001 }
1002
1003 func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, error) {
1004 return r.transformer().TransformToStorage(ctx, data, dataCtx)
1005 }
1006
1007 func (r *resourceTransformer) transformer() storagevalue.Transformer {
1008 return transformerFromOverrides(r.transformTracker.Load().(*transformTracker).transformerOverrides, r.resource)
1009 }
1010
1011 var _ storagevalue.ResourceTransformers = &StaticTransformers{}
1012
1013 type StaticTransformers map[schema.GroupResource]storagevalue.Transformer
1014
1015 func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer {
1016 return transformerFromOverrides(s, resource)
1017 }
1018
1019 var anyGroupAnyResource = schema.GroupResource{
1020 Group: "*",
1021 Resource: "*",
1022 }
1023
1024 func transformerFromOverrides(transformerOverrides map[schema.GroupResource]storagevalue.Transformer, resource schema.GroupResource) (out storagevalue.Transformer) {
1025 defer func() { out = newRequestInfoTransformer(resource, out) }()
1026
1027 if transformer := transformerOverrides[resource]; transformer != nil {
1028 return transformer
1029 }
1030
1031 if transformer := transformerOverrides[schema.GroupResource{
1032 Group: resource.Group,
1033 Resource: "*",
1034 }]; transformer != nil {
1035 return transformer
1036 }
1037
1038 if transformer := transformerOverrides[anyGroupAnyResource]; transformer != nil {
1039 return transformer
1040 }
1041
1042 return identity.NewEncryptCheckTransformer()
1043 }
1044
1045 func grYAMLString(gr schema.GroupResource) string {
1046 if gr.Group == "" && gr.Resource == "*" {
1047 return "*."
1048 }
1049
1050 return gr.String()
1051 }
1052
1053 var _ storagevalue.Transformer = &requestInfoTransformer{}
1054
1055 type requestInfoTransformer struct {
1056 baseValueCtx context.Context
1057 delegate storagevalue.Transformer
1058 }
1059
1060 func newRequestInfoTransformer(resource schema.GroupResource, delegate storagevalue.Transformer) *requestInfoTransformer {
1061 return &requestInfoTransformer{
1062 baseValueCtx: request.WithRequestInfo(context.Background(), &request.RequestInfo{IsResourceRequest: true, APIGroup: resource.Group, Resource: resource.Resource}),
1063 delegate: delegate,
1064 }
1065 }
1066
1067 func (l *requestInfoTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, bool, error) {
1068 return l.delegate.TransformFromStorage(l.withBaseValueCtx(ctx), data, dataCtx)
1069 }
1070
1071 func (l *requestInfoTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, error) {
1072 return l.delegate.TransformToStorage(l.withBaseValueCtx(ctx), data, dataCtx)
1073 }
1074
1075 func (l *requestInfoTransformer) withBaseValueCtx(ctx context.Context) context.Context {
1076 return &joinValueContext{Context: ctx, baseValueCtx: l.baseValueCtx}
1077 }
1078
1079 type joinValueContext struct {
1080 context.Context
1081 baseValueCtx context.Context
1082 }
1083
1084 func (j *joinValueContext) Value(key any) any {
1085 if val := j.Context.Value(key); val != nil {
1086 return val
1087 }
1088 return j.baseValueCtx.Value(key)
1089 }