| 1 |
/* |
| 2 |
Copyright 2017 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package encryptionconfig |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"crypto/aes" |
| 22 |
"crypto/cipher" |
| 23 |
"crypto/sha256" |
| 24 |
"encoding/base64" |
| 25 |
"errors" |
| 26 |
"fmt" |
| 27 |
"net/http" |
| 28 |
"os" |
| 29 |
"sync" |
| 30 |
"sync/atomic" |
| 31 |
"time" |
| 32 |
|
| 33 |
"k8s.io/apimachinery/pkg/runtime" |
| 34 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 35 |
"k8s.io/apimachinery/pkg/runtime/serializer" |
| 36 |
utilerrors "k8s.io/apimachinery/pkg/util/errors" |
| 37 |
utilruntime "k8s.io/apimachinery/pkg/util/runtime" |
| 38 |
"k8s.io/apimachinery/pkg/util/uuid" |
| 39 |
"k8s.io/apimachinery/pkg/util/wait" |
| 40 |
"k8s.io/apiserver/pkg/apis/apiserver" |
| 41 |
apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" |
| 42 |
"k8s.io/apiserver/pkg/apis/apiserver/validation" |
| 43 |
"k8s.io/apiserver/pkg/endpoints/request" |
| 44 |
"k8s.io/apiserver/pkg/features" |
| 45 |
"k8s.io/apiserver/pkg/server/healthz" |
| 46 |
"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics" |
| 47 |
storagevalue "k8s.io/apiserver/pkg/storage/value" |
| 48 |
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" |
| 49 |
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope" |
| 50 |
envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2" |
| 51 |
kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2" |
| 52 |
envelopemetrics "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics" |
| 53 |
"k8s.io/apiserver/pkg/storage/value/encrypt/identity" |
| 54 |
"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox" |
| 55 |
utilfeature "k8s.io/apiserver/pkg/util/feature" |
| 56 |
"k8s.io/klog/v2" |
| 57 |
kmsservice "k8s.io/kms/pkg/service" |
| 58 |
) |
| 59 |
|
| 60 |
const ( |
| 61 |
aesCBCTransformerPrefixV1 = "k8s:enc:aescbc:v1:" |
| 62 |
aesGCMTransformerPrefixV1 = "k8s:enc:aesgcm:v1:" |
| 63 |
secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:" |
| 64 |
kmsTransformerPrefixV1 = "k8s:enc:kms:v1:" |
| 65 |
kmsTransformerPrefixV2 = "k8s:enc:kms:v2:" |
| 66 |
|
| 67 |
// these constants relate to how the KMS v2 plugin status poll logic |
| 68 |
// and the DEK/seed generation logic behave. In particular, the positive |
| 69 |
// interval and max TTL are closely related as the difference between |
| 70 |
// these values defines the worst case window in which the write DEK/seed |
| 71 |
// could expire due to the plugin going into an error state. The |
| 72 |
// worst case window divided by the negative interval defines the |
| 73 |
// minimum amount of times the server will attempt to return to a |
| 74 |
// healthy state before the DEK/seed expires and writes begin to fail. |
| 75 |
// |
| 76 |
// For now, these values are kept small and hardcoded to support being |
| 77 |
// able to perform a "passive" storage migration while tolerating some |
| 78 |
// amount of plugin downtime. |
| 79 |
// |
| 80 |
// With the current approach, a user can update the key ID their plugin |
| 81 |
// is using and then can simply schedule a migration for 3 + N + M minutes |
| 82 |
// later where N is how long it takes their plugin to pick up new config |
| 83 |
// and M is extra buffer to allow the API server to process the config. |
| 84 |
// At that point, they are guaranteed to either migrate to the new key |
| 85 |
// or get errors during the migration. |
| 86 |
// |
| 87 |
// If the API server coasted forever on the last DEK/seed, they would need |
| 88 |
// to actively check if it had observed the new key ID before starting |
| 89 |
// a migration - otherwise it could keep using the old DEK/seed and their |
| 90 |
// storage migration would not do what they thought it did. |
| 91 |
kmsv2PluginHealthzPositiveInterval = 1 * time.Minute |
| 92 |
kmsv2PluginHealthzNegativeInterval = 10 * time.Second |
| 93 |
kmsv2PluginWriteDEKSourceMaxTTL = 3 * time.Minute |
| 94 |
|
| 95 |
kmsPluginHealthzNegativeTTL = 3 * time.Second |
| 96 |
kmsPluginHealthzPositiveTTL = 20 * time.Second |
| 97 |
kmsAPIVersionV1 = "v1" |
| 98 |
kmsAPIVersionV2 = "v2" |
| 99 |
// this name is used for two different healthz endpoints: |
| 100 |
// - when one or more KMS v2 plugins are in use and no KMS v1 plugins are in use |
| 101 |
// in this case, all v2 plugins are probed via this single endpoint |
| 102 |
// - when automatic reload of encryption config is enabled |
| 103 |
// in this case, all KMS plugins are probed via this single endpoint |
| 104 |
// the endpoint is present even if there are no KMS plugins configured (it is a no-op then) |
| 105 |
kmsReloadHealthCheckName = "kms-providers" |
| 106 |
) |
| 107 |
|
| 108 |
var codecs serializer.CodecFactory |
| 109 |
|
| 110 |
// this map allows us to swap enablement of the KMSv2KDF feature in tests |
| 111 |
// as the feature gate is now locked to true starting with v1.29 |
| 112 |
// Note: it cannot be set by an end user |
| 113 |
// KDF enablement is tracked per KMS provider to allow tests to run in parallel. |
| 114 |
var kdfEnabledPerKMS sync.Map // map[string]bool, KMS name -> KDF enabled |
| 115 |
|
| 116 |
// this function should only be called in tests to swap enablement of the KMSv2KDF feature |
| 117 |
// Caller must guarantee that all KMS providers have distinct names across all tests. |
| 118 |
func SetKDFForTests(kmsName string, b bool) func() { |
| 119 |
if len(kmsName) == 0 { // guarantee that GetKDF("") returns the default value |
| 120 |
panic("empty KMS name used in test") |
| 121 |
} |
| 122 |
if _, loaded := kdfEnabledPerKMS.LoadOrStore(kmsName, b); loaded { |
| 123 |
panic("duplicate KMS name used in test") |
| 124 |
} |
| 125 |
return func() { kdfEnabledPerKMS.Delete(kmsName) } |
| 126 |
} |
| 127 |
|
| 128 |
// this function should be used to determine enablement of the KMSv2KDF feature |
| 129 |
// instead of getting it from DefaultFeatureGate as the feature gate is now locked |
| 130 |
// to true starting with v1.29 |
| 131 |
// to allow integration tests to run in parallel, this "feature flag" can be set |
| 132 |
// per KMS provider as long as all providers use distinct names. |
| 133 |
func GetKDF(kmsName string) bool { |
| 134 |
kdfEnabled, ok := kdfEnabledPerKMS.Load(kmsName) |
| 135 |
if !ok { |
| 136 |
return true // explicit config is missing, but KDF is enabled by default |
| 137 |
} |
| 138 |
return kdfEnabled.(bool) // this will panic if a non-bool ever gets stored, which should never happen |
| 139 |
} |
| 140 |
|
| 141 |
func init() { |
| 142 |
configScheme := runtime.NewScheme() |
| 143 |
utilruntime.Must(apiserver.AddToScheme(configScheme)) |
| 144 |
utilruntime.Must(apiserverv1.AddToScheme(configScheme)) |
| 145 |
codecs = serializer.NewCodecFactory(configScheme, serializer.EnableStrict) |
| 146 |
envelopemetrics.RegisterMetrics() |
| 147 |
storagevalue.RegisterMetrics() |
| 148 |
metrics.RegisterMetrics() |
| 149 |
} |
| 150 |
|
| 151 |
type kmsPluginHealthzResponse struct { |
| 152 |
err error |
| 153 |
received time.Time |
| 154 |
} |
| 155 |
|
| 156 |
type kmsPluginProbe struct { |
| 157 |
name string |
| 158 |
ttl time.Duration |
| 159 |
service envelope.Service |
| 160 |
lastResponse *kmsPluginHealthzResponse |
| 161 |
l *sync.Mutex |
| 162 |
} |
| 163 |
|
| 164 |
type kmsv2PluginProbe struct { |
| 165 |
state atomic.Pointer[envelopekmsv2.State] |
| 166 |
name string |
| 167 |
ttl time.Duration |
| 168 |
service kmsservice.Service |
| 169 |
lastResponse *kmsPluginHealthzResponse |
| 170 |
l *sync.Mutex |
| 171 |
apiServerID string |
| 172 |
version string |
| 173 |
} |
| 174 |
|
| 175 |
type kmsHealthChecker []healthz.HealthChecker |
| 176 |
|
| 177 |
func (k kmsHealthChecker) Name() string { |
| 178 |
return kmsReloadHealthCheckName |
| 179 |
} |
| 180 |
|
| 181 |
func (k kmsHealthChecker) Check(req *http.Request) error { |
| 182 |
var errs []error |
| 183 |
|
| 184 |
for i := range k { |
| 185 |
checker := k[i] |
| 186 |
if err := checker.Check(req); err != nil { |
| 187 |
errs = append(errs, fmt.Errorf("%s: %w", checker.Name(), err)) |
| 188 |
} |
| 189 |
} |
| 190 |
|
| 191 |
return utilerrors.Reduce(utilerrors.NewAggregate(errs)) |
| 192 |
} |
| 193 |
|
| 194 |
func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { |
| 195 |
return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error { |
| 196 |
return h.check() |
| 197 |
}) |
| 198 |
} |
| 199 |
|
| 200 |
func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { |
| 201 |
return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error { |
| 202 |
return h.check(r.Context()) |
| 203 |
}) |
| 204 |
} |
| 205 |
|
| 206 |
// EncryptionConfiguration represents the parsed and normalized encryption configuration for the apiserver. |
| 207 |
type EncryptionConfiguration struct { |
| 208 |
// Transformers is a list of value.Transformer that will be used to encrypt and decrypt data. |
| 209 |
Transformers map[schema.GroupResource]storagevalue.Transformer |
| 210 |
|
| 211 |
// HealthChecks is a list of healthz.HealthChecker that will be used to check the health of the encryption providers. |
| 212 |
HealthChecks []healthz.HealthChecker |
| 213 |
|
| 214 |
// EncryptionFileContentHash is the hash of the encryption config file. |
| 215 |
EncryptionFileContentHash string |
| 216 |
|
| 217 |
// KMSCloseGracePeriod is the duration we will wait before closing old transformers. |
| 218 |
// We wait for any in-flight requests to finish by using the duration which is longer than their timeout. |
| 219 |
KMSCloseGracePeriod time.Duration |
| 220 |
} |
| 221 |
|
| 222 |
// LoadEncryptionConfig parses and validates the encryption config specified by filepath. |
| 223 |
// It may launch multiple go routines whose lifecycle is controlled by ctx. |
| 224 |
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. |
| 225 |
// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1. |
| 226 |
func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool, apiServerID string) (*EncryptionConfiguration, error) { |
| 227 |
config, contentHash, err := loadConfig(filepath, reload) |
| 228 |
if err != nil { |
| 229 |
return nil, fmt.Errorf("error while parsing file: %w", err) |
| 230 |
} |
| 231 |
|
| 232 |
transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config, apiServerID) |
| 233 |
if err != nil { |
| 234 |
return nil, fmt.Errorf("error while building transformers: %w", err) |
| 235 |
} |
| 236 |
|
| 237 |
if reload || (kmsUsed.v2Used && !kmsUsed.v1Used) { |
| 238 |
kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)} |
| 239 |
} |
| 240 |
|
| 241 |
// KMSCloseGracePeriod is the duration we will wait before closing old transformers. |
| 242 |
// The way we calculate is as follows: |
| 243 |
// 1. Sum all timeouts across all KMS plugins. (check kmsPrefixTransformer for differences between v1 and v2) |
| 244 |
// 2. Multiply that by 2 (to allow for some buffer) |
| 245 |
// The reason we sum all timeout is because kmsHealthChecker() will run all health checks serially |
| 246 |
return &EncryptionConfiguration{ |
| 247 |
Transformers: transformers, |
| 248 |
HealthChecks: kmsHealthChecks, |
| 249 |
EncryptionFileContentHash: contentHash, |
| 250 |
KMSCloseGracePeriod: 2 * kmsUsed.kmsTimeoutSum, |
| 251 |
}, nil |
| 252 |
} |
| 253 |
|
| 254 |
// getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config. |
| 255 |
// It may launch multiple go routines whose lifecycle is controlled by ctx. |
| 256 |
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. |
| 257 |
func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserver.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) { |
| 258 |
var kmsHealthChecks []healthz.HealthChecker |
| 259 |
transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, apiServerID) |
| 260 |
if err != nil { |
| 261 |
return nil, nil, nil, err |
| 262 |
} |
| 263 |
for i := range probes { |
| 264 |
probe := probes[i] |
| 265 |
kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i)) |
| 266 |
} |
| 267 |
|
| 268 |
return transformers, kmsHealthChecks, kmsUsed, nil |
| 269 |
} |
| 270 |
|
| 271 |
type healthChecker interface { |
| 272 |
toHealthzCheck(idx int) healthz.HealthChecker |
| 273 |
} |
| 274 |
|
| 275 |
// getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config. |
| 276 |
// It may launch multiple go routines whose lifecycle is controlled by ctx. |
| 277 |
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. |
| 278 |
func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserver.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) { |
| 279 |
resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{} |
| 280 |
var probes []healthChecker |
| 281 |
var kmsUsed kmsState |
| 282 |
|
| 283 |
// For each entry in the configuration |
| 284 |
for _, resourceConfig := range config.Resources { |
| 285 |
resourceConfig := resourceConfig |
| 286 |
|
| 287 |
transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig, apiServerID) |
| 288 |
if err != nil { |
| 289 |
return nil, nil, nil, err |
| 290 |
} |
| 291 |
kmsUsed.accumulate(used) |
| 292 |
|
| 293 |
// For each resource, create a list of providers to use |
| 294 |
for _, resource := range resourceConfig.Resources { |
| 295 |
resource := resource |
| 296 |
gr := schema.ParseGroupResource(resource) |
| 297 |
|
| 298 |
// check if resource is masked by *.group rule |
| 299 |
anyResourceInGroup := schema.GroupResource{Group: gr.Group, Resource: "*"} |
| 300 |
if _, masked := resourceToPrefixTransformer[anyResourceInGroup]; masked { |
| 301 |
// an earlier rule already configured a transformer for *.group, masking this rule |
| 302 |
// return error since this is not allowed |
| 303 |
return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyResourceInGroup)) |
| 304 |
} |
| 305 |
|
| 306 |
if _, masked := resourceToPrefixTransformer[anyGroupAnyResource]; masked { |
| 307 |
// an earlier rule already configured a transformer for *.*, masking this rule |
| 308 |
// return error since this is not allowed |
| 309 |
return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyGroupAnyResource)) |
| 310 |
} |
| 311 |
|
| 312 |
resourceToPrefixTransformer[gr] = append(resourceToPrefixTransformer[gr], transformers...) |
| 313 |
} |
| 314 |
|
| 315 |
probes = append(probes, p...) |
| 316 |
} |
| 317 |
|
| 318 |
transformers := make(map[schema.GroupResource]storagevalue.Transformer, len(resourceToPrefixTransformer)) |
| 319 |
for gr, transList := range resourceToPrefixTransformer { |
| 320 |
gr := gr |
| 321 |
transList := transList |
| 322 |
transformers[gr] = storagevalue.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...) |
| 323 |
} |
| 324 |
|
| 325 |
return transformers, probes, &kmsUsed, nil |
| 326 |
} |
| 327 |
|
| 328 |
// check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint. |
| 329 |
func (h *kmsPluginProbe) check() error { |
| 330 |
h.l.Lock() |
| 331 |
defer h.l.Unlock() |
| 332 |
|
| 333 |
if (time.Since(h.lastResponse.received)) < h.ttl { |
| 334 |
return h.lastResponse.err |
| 335 |
} |
| 336 |
|
| 337 |
p, err := h.service.Encrypt([]byte("ping")) |
| 338 |
if err != nil { |
| 339 |
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} |
| 340 |
h.ttl = kmsPluginHealthzNegativeTTL |
| 341 |
return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err) |
| 342 |
} |
| 343 |
|
| 344 |
if _, err := h.service.Decrypt(p); err != nil { |
| 345 |
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} |
| 346 |
h.ttl = kmsPluginHealthzNegativeTTL |
| 347 |
return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err) |
| 348 |
} |
| 349 |
|
| 350 |
h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()} |
| 351 |
h.ttl = kmsPluginHealthzPositiveTTL |
| 352 |
return nil |
| 353 |
} |
| 354 |
|
| 355 |
// check gets the healthz status of the KMSv2-Plugin using the Status() method. |
| 356 |
func (h *kmsv2PluginProbe) check(ctx context.Context) error { |
| 357 |
h.l.Lock() |
| 358 |
defer h.l.Unlock() |
| 359 |
|
| 360 |
if time.Since(h.lastResponse.received) < h.ttl { |
| 361 |
return h.lastResponse.err |
| 362 |
} |
| 363 |
|
| 364 |
p, err := h.service.Status(ctx) |
| 365 |
if err != nil { |
| 366 |
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} |
| 367 |
h.ttl = kmsPluginHealthzNegativeTTL |
| 368 |
return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %w", h.name, err) |
| 369 |
} |
| 370 |
|
| 371 |
if err := h.isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx, p); err != nil { |
| 372 |
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} |
| 373 |
h.ttl = kmsPluginHealthzNegativeTTL |
| 374 |
return err |
| 375 |
} |
| 376 |
|
| 377 |
h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()} |
| 378 |
h.ttl = kmsPluginHealthzPositiveTTL |
| 379 |
return nil |
| 380 |
} |
| 381 |
|
| 382 |
// rotateDEKOnKeyIDChange tries to rotate to a new DEK/seed if the key ID returned by Status does not match the |
| 383 |
// current state. If a successful rotation is performed, the new DEK/seed and keyID overwrite the existing state. |
| 384 |
// On any failure during rotation (including mismatch between status and encrypt calls), the current state is |
| 385 |
// preserved and will remain valid to use for encryption until its expiration (the system attempts to coast). |
| 386 |
// If the key ID returned by Status matches the current state, the expiration of the current state is extended |
| 387 |
// and no rotation is performed. |
| 388 |
func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKeyID, uid string) error { |
| 389 |
// we do not check ValidateEncryptCapability here because it is fine to re-use an old key |
| 390 |
// that was marked as expired during an unhealthy period. As long as the key ID matches |
| 391 |
// what we expect then there is no need to rotate here. |
| 392 |
state, errState := h.getCurrentState() |
| 393 |
|
| 394 |
// allow reads indefinitely in all cases |
| 395 |
// allow writes indefinitely as long as there is no error |
| 396 |
// allow writes for only up to kmsv2PluginWriteDEKSourceMaxTTL from now when there are errors |
| 397 |
// we start the timer before we make the network call because kmsv2PluginWriteDEKSourceMaxTTL is meant to be the upper bound |
| 398 |
expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKSourceMaxTTL) |
| 399 |
|
| 400 |
// dynamically check if we want to use KDF seed to derive DEKs or just a single DEK |
| 401 |
// this gate can only change during tests, but the check is cheap enough to always make |
| 402 |
// this allows us to easily exercise both modes without restarting the API server |
| 403 |
// TODO integration test that this dynamically takes effect |
| 404 |
useSeed := GetKDF(h.name) |
| 405 |
stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED |
| 406 |
|
| 407 |
// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it |
| 408 |
// just move the expiration of the current state forward by the reuse interval |
| 409 |
// useSeed can only change at runtime during tests, so we check it here to allow us to easily exercise both modes |
| 410 |
if errState == nil && state.EncryptedObject.KeyID == statusKeyID && stateUseSeed == useSeed { |
| 411 |
state.ExpirationTimestamp = expirationTimestamp |
| 412 |
h.state.Store(&state) |
| 413 |
return nil |
| 414 |
} |
| 415 |
|
| 416 |
transformer, encObject, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service, useSeed) |
| 417 |
|
| 418 |
if encObject == nil { |
| 419 |
encObject = &kmstypes.EncryptedObject{} // avoid nil panics |
| 420 |
} |
| 421 |
|
| 422 |
// happy path, should be the common case |
| 423 |
// TODO maybe add success metrics? |
| 424 |
if errGen == nil && encObject.KeyID == statusKeyID { |
| 425 |
h.state.Store(&envelopekmsv2.State{ |
| 426 |
Transformer: transformer, |
| 427 |
EncryptedObject: *encObject, |
| 428 |
UID: uid, |
| 429 |
ExpirationTimestamp: expirationTimestamp, |
| 430 |
CacheKey: cacheKey, |
| 431 |
}) |
| 432 |
|
| 433 |
// it should be logically impossible for the new state to be invalid but check just in case |
| 434 |
_, errGen = h.getCurrentState() |
| 435 |
if errGen == nil { |
| 436 |
klogV6 := klog.V(6) |
| 437 |
if klogV6.Enabled() { |
| 438 |
klogV6.InfoS("successfully rotated DEK", |
| 439 |
"uid", uid, |
| 440 |
"useSeed", useSeed, |
| 441 |
"newKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID), |
| 442 |
"oldKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID), |
| 443 |
"expirationTimestamp", expirationTimestamp.Format(time.RFC3339), |
| 444 |
) |
| 445 |
} |
| 446 |
return nil |
| 447 |
} |
| 448 |
} |
| 449 |
|
| 450 |
return fmt.Errorf("failed to rotate DEK uid=%q, useSeed=%v, errState=%v, errGen=%v, statusKeyIDHash=%q, encryptKeyIDHash=%q, stateKeyIDHash=%q, expirationTimestamp=%s", |
| 451 |
uid, useSeed, errState, errGen, envelopekmsv2.GetHashIfNotEmpty(statusKeyID), envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID), envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID), state.ExpirationTimestamp.Format(time.RFC3339)) |
| 452 |
} |
| 453 |
|
| 454 |
// getCurrentState returns the latest state from the last status and encrypt calls. |
| 455 |
// If the returned error is nil, the state is considered valid indefinitely for read requests. |
| 456 |
// For write requests, the caller must also check that state.ValidateEncryptCapability does not error. |
| 457 |
func (h *kmsv2PluginProbe) getCurrentState() (envelopekmsv2.State, error) { |
| 458 |
state := *h.state.Load() |
| 459 |
|
| 460 |
if state.Transformer == nil { |
| 461 |
return envelopekmsv2.State{}, fmt.Errorf("got unexpected nil transformer") |
| 462 |
} |
| 463 |
|
| 464 |
encryptedObjectCopy := state.EncryptedObject |
| 465 |
if len(encryptedObjectCopy.EncryptedData) != 0 { |
| 466 |
return envelopekmsv2.State{}, fmt.Errorf("got unexpected non-empty EncryptedData") |
| 467 |
} |
| 468 |
encryptedObjectCopy.EncryptedData = []byte{0} // any non-empty value to pass validation |
| 469 |
if err := envelopekmsv2.ValidateEncryptedObject(&encryptedObjectCopy); err != nil { |
| 470 |
return envelopekmsv2.State{}, fmt.Errorf("got invalid EncryptedObject: %w", err) |
| 471 |
} |
| 472 |
|
| 473 |
if state.ExpirationTimestamp.IsZero() { |
| 474 |
return envelopekmsv2.State{}, fmt.Errorf("got unexpected zero expirationTimestamp") |
| 475 |
} |
| 476 |
|
| 477 |
if len(state.CacheKey) == 0 { |
| 478 |
return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty cacheKey") |
| 479 |
} |
| 480 |
|
| 481 |
return state, nil |
| 482 |
} |
| 483 |
|
| 484 |
func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.Context, response *kmsservice.StatusResponse) error { |
| 485 |
var errs []error |
| 486 |
if response.Healthz != "ok" { |
| 487 |
errs = append(errs, fmt.Errorf("got unexpected healthz status: %s", response.Healthz)) |
| 488 |
} |
| 489 |
if response.Version != envelopekmsv2.KMSAPIVersionv2 && response.Version != envelopekmsv2.KMSAPIVersionv2beta1 { |
| 490 |
errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersionv2, response.Version)) |
| 491 |
} else { |
| 492 |
// set version for the first status response |
| 493 |
if len(h.version) == 0 { |
| 494 |
h.version = response.Version |
| 495 |
} |
| 496 |
if h.version != response.Version { |
| 497 |
errs = append(errs, fmt.Errorf("KMSv2 API version should not change after the initial status response version %s, got %s", h.version, response.Version)) |
| 498 |
} |
| 499 |
} |
| 500 |
|
| 501 |
if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil { |
| 502 |
envelopemetrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode)) |
| 503 |
errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err)) |
| 504 |
} else { |
| 505 |
envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID, h.apiServerID) |
| 506 |
// unconditionally append as we filter out nil errors below |
| 507 |
errs = append(errs, h.rotateDEKOnKeyIDChange(ctx, response.KeyID, string(uuid.NewUUID()))) |
| 508 |
} |
| 509 |
|
| 510 |
if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil { |
| 511 |
return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", h.name, err) |
| 512 |
} |
| 513 |
return nil |
| 514 |
} |
| 515 |
|
| 516 |
// loadConfig parses the encryption configuration file at filepath and returns the parsed config and hash of the file. |
| 517 |
func loadConfig(filepath string, reload bool) (*apiserver.EncryptionConfiguration, string, error) { |
| 518 |
data, contentHash, err := loadDataAndHash(filepath) |
| 519 |
if err != nil { |
| 520 |
return nil, "", fmt.Errorf("error while loading file: %w", err) |
| 521 |
} |
| 522 |
|
| 523 |
configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil) |
| 524 |
if err != nil { |
| 525 |
return nil, "", fmt.Errorf("error decoding encryption provider configuration file %q: %w", filepath, err) |
| 526 |
} |
| 527 |
config, ok := configObj.(*apiserver.EncryptionConfiguration) |
| 528 |
if !ok { |
| 529 |
return nil, "", fmt.Errorf("got unexpected config type: %v", gvk) |
| 530 |
} |
| 531 |
|
| 532 |
return config, contentHash, validation.ValidateEncryptionConfiguration(config, reload).ToAggregate() |
| 533 |
} |
| 534 |
|
| 535 |
func loadDataAndHash(filepath string) ([]byte, string, error) { |
| 536 |
data, err := os.ReadFile(filepath) |
| 537 |
if err != nil { |
| 538 |
return nil, "", fmt.Errorf("error reading encryption provider configuration file %q: %w", filepath, err) |
| 539 |
} |
| 540 |
if len(data) == 0 { |
| 541 |
return nil, "", fmt.Errorf("encryption provider configuration file %q is empty", filepath) |
| 542 |
} |
| 543 |
|
| 544 |
return data, computeEncryptionConfigHash(data), nil |
| 545 |
} |
| 546 |
|
| 547 |
// GetEncryptionConfigHash reads the encryption configuration file at filepath and returns the hash of the file. |
| 548 |
// It does not attempt to decode or load the config, and serves as a cheap check to determine if the file has changed. |
| 549 |
func GetEncryptionConfigHash(filepath string) (string, error) { |
| 550 |
_, contentHash, err := loadDataAndHash(filepath) |
| 551 |
return contentHash, err |
| 552 |
} |
| 553 |
|
| 554 |
// prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config. |
| 555 |
// It may launch multiple go routines whose lifecycle is controlled by ctx. |
| 556 |
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. |
| 557 |
func prefixTransformersAndProbes(ctx context.Context, config apiserver.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) { |
| 558 |
var transformers []storagevalue.PrefixTransformer |
| 559 |
var probes []healthChecker |
| 560 |
var kmsUsed kmsState |
| 561 |
|
| 562 |
for _, provider := range config.Providers { |
| 563 |
provider := provider |
| 564 |
var ( |
| 565 |
transformer storagevalue.PrefixTransformer |
| 566 |
transformerErr error |
| 567 |
probe healthChecker |
| 568 |
used *kmsState |
| 569 |
) |
| 570 |
|
| 571 |
switch { |
| 572 |
case provider.AESGCM != nil: |
| 573 |
transformer, transformerErr = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1) |
| 574 |
|
| 575 |
case provider.AESCBC != nil: |
| 576 |
cbcTransformer := func(block cipher.Block) (storagevalue.Transformer, error) { |
| 577 |
return aestransformer.NewCBCTransformer(block), nil |
| 578 |
} |
| 579 |
transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, cbcTransformer, aesCBCTransformerPrefixV1) |
| 580 |
|
| 581 |
case provider.Secretbox != nil: |
| 582 |
transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox) |
| 583 |
|
| 584 |
case provider.KMS != nil: |
| 585 |
transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS, apiServerID) |
| 586 |
if transformerErr == nil { |
| 587 |
probes = append(probes, probe) |
| 588 |
kmsUsed.accumulate(used) |
| 589 |
} |
| 590 |
|
| 591 |
case provider.Identity != nil: |
| 592 |
transformer = storagevalue.PrefixTransformer{ |
| 593 |
Transformer: identity.NewEncryptCheckTransformer(), |
| 594 |
Prefix: []byte{}, |
| 595 |
} |
| 596 |
|
| 597 |
default: |
| 598 |
return nil, nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity") |
| 599 |
} |
| 600 |
|
| 601 |
if transformerErr != nil { |
| 602 |
return nil, nil, nil, transformerErr |
| 603 |
} |
| 604 |
|
| 605 |
transformers = append(transformers, transformer) |
| 606 |
} |
| 607 |
|
| 608 |
return transformers, probes, &kmsUsed, nil |
| 609 |
} |
| 610 |
|
| 611 |
type blockTransformerFunc func(cipher.Block) (storagevalue.Transformer, error) |
| 612 |
|
| 613 |
func aesPrefixTransformer(config *apiserver.AESConfiguration, fn blockTransformerFunc, prefix string) (storagevalue.PrefixTransformer, error) { |
| 614 |
var result storagevalue.PrefixTransformer |
| 615 |
|
| 616 |
if len(config.Keys) == 0 { |
| 617 |
return result, fmt.Errorf("aes provider has no valid keys") |
| 618 |
} |
| 619 |
for _, key := range config.Keys { |
| 620 |
key := key |
| 621 |
if key.Name == "" { |
| 622 |
return result, fmt.Errorf("key with invalid name provided") |
| 623 |
} |
| 624 |
if key.Secret == "" { |
| 625 |
return result, fmt.Errorf("key %v has no provided secret", key.Name) |
| 626 |
} |
| 627 |
} |
| 628 |
|
| 629 |
keyTransformers := []storagevalue.PrefixTransformer{} |
| 630 |
|
| 631 |
for _, keyData := range config.Keys { |
| 632 |
keyData := keyData |
| 633 |
key, err := base64.StdEncoding.DecodeString(keyData.Secret) |
| 634 |
if err != nil { |
| 635 |
return result, fmt.Errorf("could not obtain secret for named key %s: %w", keyData.Name, err) |
| 636 |
} |
| 637 |
block, err := aes.NewCipher(key) |
| 638 |
if err != nil { |
| 639 |
return result, fmt.Errorf("error while creating cipher for named key %s: %w", keyData.Name, err) |
| 640 |
} |
| 641 |
transformer, err := fn(block) |
| 642 |
if err != nil { |
| 643 |
return result, fmt.Errorf("error while creating transformer for named key %s: %w", keyData.Name, err) |
| 644 |
} |
| 645 |
|
| 646 |
// Create a new PrefixTransformer for this key |
| 647 |
keyTransformers = append(keyTransformers, |
| 648 |
storagevalue.PrefixTransformer{ |
| 649 |
Transformer: transformer, |
| 650 |
Prefix: []byte(keyData.Name + ":"), |
| 651 |
}) |
| 652 |
} |
| 653 |
|
| 654 |
// Create a prefixTransformer which can choose between these keys |
| 655 |
keyTransformer := storagevalue.NewPrefixTransformers( |
| 656 |
fmt.Errorf("no matching key was found for the provided AES transformer"), keyTransformers...) |
| 657 |
|
| 658 |
// Create a PrefixTransformer which shall later be put in a list with other providers |
| 659 |
result = storagevalue.PrefixTransformer{ |
| 660 |
Transformer: keyTransformer, |
| 661 |
Prefix: []byte(prefix), |
| 662 |
} |
| 663 |
return result, nil |
| 664 |
} |
| 665 |
|
| 666 |
func secretboxPrefixTransformer(config *apiserver.SecretboxConfiguration) (storagevalue.PrefixTransformer, error) { |
| 667 |
var result storagevalue.PrefixTransformer |
| 668 |
|
| 669 |
if len(config.Keys) == 0 { |
| 670 |
return result, fmt.Errorf("secretbox provider has no valid keys") |
| 671 |
} |
| 672 |
for _, key := range config.Keys { |
| 673 |
key := key |
| 674 |
if key.Name == "" { |
| 675 |
return result, fmt.Errorf("key with invalid name provided") |
| 676 |
} |
| 677 |
if key.Secret == "" { |
| 678 |
return result, fmt.Errorf("key %v has no provided secret", key.Name) |
| 679 |
} |
| 680 |
} |
| 681 |
|
| 682 |
keyTransformers := []storagevalue.PrefixTransformer{} |
| 683 |
|
| 684 |
for _, keyData := range config.Keys { |
| 685 |
keyData := keyData |
| 686 |
key, err := base64.StdEncoding.DecodeString(keyData.Secret) |
| 687 |
if err != nil { |
| 688 |
return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err) |
| 689 |
} |
| 690 |
|
| 691 |
if len(key) != 32 { |
| 692 |
return result, fmt.Errorf("expected key size 32 for secretbox provider, got %v", len(key)) |
| 693 |
} |
| 694 |
|
| 695 |
keyArray := [32]byte{} |
| 696 |
copy(keyArray[:], key) |
| 697 |
|
| 698 |
// Create a new PrefixTransformer for this key |
| 699 |
keyTransformers = append(keyTransformers, |
| 700 |
storagevalue.PrefixTransformer{ |
| 701 |
Transformer: secretbox.NewSecretboxTransformer(keyArray), |
| 702 |
Prefix: []byte(keyData.Name + ":"), |
| 703 |
}) |
| 704 |
} |
| 705 |
|
| 706 |
// Create a prefixTransformer which can choose between these keys |
| 707 |
keyTransformer := storagevalue.NewPrefixTransformers( |
| 708 |
fmt.Errorf("no matching key was found for the provided Secretbox transformer"), keyTransformers...) |
| 709 |
|
| 710 |
// Create a PrefixTransformer which shall later be put in a list with other providers |
| 711 |
result = storagevalue.PrefixTransformer{ |
| 712 |
Transformer: keyTransformer, |
| 713 |
Prefix: []byte(secretboxTransformerPrefixV1), |
| 714 |
} |
| 715 |
return result, nil |
| 716 |
} |
| 717 |
|
| 718 |
var ( |
| 719 |
// The factory to create kms service. This is to make writing test easier. |
| 720 |
envelopeServiceFactory = envelope.NewGRPCService |
| 721 |
|
| 722 |
// The factory to create kmsv2 service. Exported for integration tests. |
| 723 |
EnvelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService |
| 724 |
) |
| 725 |
|
| 726 |
type kmsState struct { |
| 727 |
v1Used, v2Used bool |
| 728 |
kmsTimeoutSum time.Duration |
| 729 |
} |
| 730 |
|
| 731 |
// accumulate computes the KMS state by: |
| 732 |
// - determining which KMS plugin versions are in use |
| 733 |
// - calculating kmsTimeoutSum which is used as transformTracker.kmsCloseGracePeriod |
| 734 |
// DynamicTransformers.Set waits for this period before closing old transformers after a config reload |
| 735 |
func (s *kmsState) accumulate(other *kmsState) { |
| 736 |
s.v1Used = s.v1Used || other.v1Used |
| 737 |
s.v2Used = s.v2Used || other.v2Used |
| 738 |
s.kmsTimeoutSum += other.kmsTimeoutSum |
| 739 |
} |
| 740 |
|
| 741 |
// kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config. |
| 742 |
// It may launch multiple go routines whose lifecycle is controlled by ctx. |
| 743 |
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. |
| 744 |
func kmsPrefixTransformer(ctx context.Context, config *apiserver.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) { |
| 745 |
kmsName := config.Name |
| 746 |
switch config.APIVersion { |
| 747 |
case kmsAPIVersionV1: |
| 748 |
if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) { |
| 749 |
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead. Set --feature-gates=KMSv1=true to use the deprecated KMSv1 feature.") |
| 750 |
} |
| 751 |
klog.InfoS("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead.") |
| 752 |
|
| 753 |
envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration) |
| 754 |
if err != nil { |
| 755 |
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err) |
| 756 |
} |
| 757 |
|
| 758 |
probe := &kmsPluginProbe{ |
| 759 |
name: kmsName, |
| 760 |
ttl: kmsPluginHealthzNegativeTTL, |
| 761 |
service: envelopeService, |
| 762 |
l: &sync.Mutex{}, |
| 763 |
lastResponse: &kmsPluginHealthzResponse{}, |
| 764 |
} |
| 765 |
|
| 766 |
transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1) |
| 767 |
|
| 768 |
return transformer, probe, &kmsState{ |
| 769 |
v1Used: true, |
| 770 |
// for v1 we will do encrypt and decrypt for health check. Since these are serial operations, we will double the timeout. |
| 771 |
kmsTimeoutSum: 2 * config.Timeout.Duration, |
| 772 |
}, nil |
| 773 |
|
| 774 |
case kmsAPIVersionV2: |
| 775 |
envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Name, config.Timeout.Duration) |
| 776 |
if err != nil { |
| 777 |
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err) |
| 778 |
} |
| 779 |
|
| 780 |
probe := &kmsv2PluginProbe{ |
| 781 |
name: kmsName, |
| 782 |
ttl: kmsPluginHealthzNegativeTTL, |
| 783 |
service: envelopeService, |
| 784 |
l: &sync.Mutex{}, |
| 785 |
lastResponse: &kmsPluginHealthzResponse{}, |
| 786 |
apiServerID: apiServerID, |
| 787 |
} |
| 788 |
// initialize state so that Load always works |
| 789 |
probe.state.Store(&envelopekmsv2.State{}) |
| 790 |
|
| 791 |
primeAndProbeKMSv2(ctx, probe, kmsName) |
| 792 |
transformer := storagevalue.PrefixTransformer{ |
| 793 |
Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState, apiServerID), |
| 794 |
Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"), |
| 795 |
} |
| 796 |
|
| 797 |
return transformer, probe, &kmsState{ |
| 798 |
v2Used: true, |
| 799 |
kmsTimeoutSum: config.Timeout.Duration, |
| 800 |
}, nil |
| 801 |
|
| 802 |
default: |
| 803 |
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion) |
| 804 |
} |
| 805 |
} |
| 806 |
|
| 807 |
func primeAndProbeKMSv2(ctx context.Context, probe *kmsv2PluginProbe, kmsName string) { |
| 808 |
runProbeCheckAndLog := func(ctx context.Context, depth int) error { |
| 809 |
if err := probe.check(ctx); err != nil { |
| 810 |
klog.VDepth(1+depth, 2).ErrorS(err, "kms plugin failed health check probe", "name", kmsName) |
| 811 |
return err |
| 812 |
} |
| 813 |
return nil |
| 814 |
} |
| 815 |
|
| 816 |
blockAndProbeFastUntilSuccess := func(ctx context.Context) { |
| 817 |
_ = wait.PollUntilWithContext( |
| 818 |
ctx, |
| 819 |
kmsv2PluginHealthzNegativeInterval, |
| 820 |
func(ctx context.Context) (bool, error) { |
| 821 |
return runProbeCheckAndLog(ctx, 1) == nil, nil |
| 822 |
}, |
| 823 |
) |
| 824 |
} |
| 825 |
|
| 826 |
// on the happy path where the plugin is healthy and available on server start, |
| 827 |
// prime keyID and DEK by running the check inline once (this also prevents unit tests from flaking) |
| 828 |
errPrime := runProbeCheckAndLog(ctx, 0) |
| 829 |
|
| 830 |
// if our initial attempt to prime failed, start trying to get to a valid state in the background ASAP |
| 831 |
// this prevents a slow start when the external healthz checker is configured to ignore the KMS healthz endpoint |
| 832 |
// since we want to support the plugin starting up async with the API server, this error is not fatal |
| 833 |
if errPrime != nil { |
| 834 |
go blockAndProbeFastUntilSuccess(ctx) // separate go routine to avoid blocking |
| 835 |
} |
| 836 |
|
| 837 |
// make sure that the plugin's key ID is reasonably up-to-date |
| 838 |
// also, make sure that our DEK is up-to-date to with said key ID (if it expires the server will fail all writes) |
| 839 |
// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKSourceMaxTTL |
| 840 |
go wait.PollUntilWithContext( |
| 841 |
ctx, |
| 842 |
kmsv2PluginHealthzPositiveInterval, |
| 843 |
func(ctx context.Context) (bool, error) { |
| 844 |
if err := runProbeCheckAndLog(ctx, 0); err == nil { |
| 845 |
return false, nil |
| 846 |
} |
| 847 |
|
| 848 |
// TODO add integration test for quicker error poll on failure |
| 849 |
// if we fail, block the outer polling and start a new quicker poll inline |
| 850 |
// this limits the chance that our DEK expires during a transient failure |
| 851 |
blockAndProbeFastUntilSuccess(ctx) |
| 852 |
|
| 853 |
return false, nil |
| 854 |
}) |
| 855 |
} |
| 856 |
|
| 857 |
func envelopePrefixTransformer(config *apiserver.KMSConfiguration, envelopeService envelope.Service, prefix string) storagevalue.PrefixTransformer { |
| 858 |
baseTransformerFunc := func(block cipher.Block) (storagevalue.Transformer, error) { |
| 859 |
gcm, err := aestransformer.NewGCMTransformer(block) |
| 860 |
if err != nil { |
| 861 |
return nil, err |
| 862 |
} |
| 863 |
|
| 864 |
// v1.24: write using AES-CBC only but support reads via AES-CBC and AES-GCM (so we can move to AES-GCM) |
| 865 |
// v1.25: write using AES-GCM only but support reads via AES-GCM and fallback to AES-CBC for backwards compatibility |
| 866 |
// TODO(aramase): Post v1.25: We cannot drop CBC read support until we automate storage migration. |
| 867 |
// We could have a release note that hard requires users to perform storage migration. |
| 868 |
return unionTransformers{gcm, aestransformer.NewCBCTransformer(block)}, nil |
| 869 |
} |
| 870 |
|
| 871 |
return storagevalue.PrefixTransformer{ |
| 872 |
Transformer: envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc), |
| 873 |
Prefix: []byte(prefix + config.Name + ":"), |
| 874 |
} |
| 875 |
} |
| 876 |
|
| 877 |
type unionTransformers []storagevalue.Transformer |
| 878 |
|
| 879 |
func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, stale bool, err error) { |
| 880 |
var errs []error |
| 881 |
for i := range u { |
| 882 |
transformer := u[i] |
| 883 |
result, stale, err := transformer.TransformFromStorage(ctx, data, dataCtx) |
| 884 |
if err != nil { |
| 885 |
errs = append(errs, err) |
| 886 |
continue |
| 887 |
} |
| 888 |
// when i != 0, we have transformed the data from storage using the new transformer, |
| 889 |
// we want to issue a write to etcd even if the contents of the data haven't changed |
| 890 |
return result, stale || i != 0, nil |
| 891 |
} |
| 892 |
if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil { |
| 893 |
return nil, false, err |
| 894 |
} |
| 895 |
return nil, false, fmt.Errorf("unionTransformers: unable to transform from storage") |
| 896 |
} |
| 897 |
|
| 898 |
func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, err error) { |
| 899 |
return u[0].TransformToStorage(ctx, data, dataCtx) |
| 900 |
} |
| 901 |
|
| 902 |
// computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes. |
| 903 |
// We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers. |
| 904 |
// This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees. |
| 905 |
func computeEncryptionConfigHash(data []byte) string { |
| 906 |
return fmt.Sprintf("k8s:enc:unstable:1:%x", sha256.Sum256(data)) |
| 907 |
} |
| 908 |
|
| 909 |
var _ storagevalue.ResourceTransformers = &DynamicTransformers{} |
| 910 |
var _ healthz.HealthChecker = &DynamicTransformers{} |
| 911 |
|
| 912 |
// DynamicTransformers holds transformers that may be dynamically updated via a single external actor, likely a controller. |
| 913 |
// This struct must avoid locks (even read write locks) as it is inline to all calls to storage. |
| 914 |
type DynamicTransformers struct { |
| 915 |
transformTracker *atomic.Value |
| 916 |
} |
| 917 |
|
| 918 |
type transformTracker struct { |
| 919 |
transformerOverrides map[schema.GroupResource]storagevalue.Transformer |
| 920 |
kmsPluginHealthzCheck healthz.HealthChecker |
| 921 |
closeTransformers context.CancelFunc |
| 922 |
kmsCloseGracePeriod time.Duration |
| 923 |
} |
| 924 |
|
| 925 |
// NewDynamicTransformers returns transformers, health checks for kms providers and an ability to close transformers. |
| 926 |
func NewDynamicTransformers( |
| 927 |
transformerOverrides map[schema.GroupResource]storagevalue.Transformer, |
| 928 |
kmsPluginHealthzCheck healthz.HealthChecker, |
| 929 |
closeTransformers context.CancelFunc, |
| 930 |
kmsCloseGracePeriod time.Duration, |
| 931 |
) *DynamicTransformers { |
| 932 |
dynamicTransformers := &DynamicTransformers{ |
| 933 |
transformTracker: &atomic.Value{}, |
| 934 |
} |
| 935 |
|
| 936 |
tracker := &transformTracker{ |
| 937 |
transformerOverrides: transformerOverrides, |
| 938 |
kmsPluginHealthzCheck: kmsPluginHealthzCheck, |
| 939 |
closeTransformers: closeTransformers, |
| 940 |
kmsCloseGracePeriod: kmsCloseGracePeriod, |
| 941 |
} |
| 942 |
dynamicTransformers.transformTracker.Store(tracker) |
| 943 |
|
| 944 |
return dynamicTransformers |
| 945 |
} |
| 946 |
|
| 947 |
// Check implements healthz.HealthChecker |
| 948 |
func (d *DynamicTransformers) Check(req *http.Request) error { |
| 949 |
return d.transformTracker.Load().(*transformTracker).kmsPluginHealthzCheck.Check(req) |
| 950 |
} |
| 951 |
|
| 952 |
// Name implements healthz.HealthChecker |
| 953 |
func (d *DynamicTransformers) Name() string { |
| 954 |
return kmsReloadHealthCheckName |
| 955 |
} |
| 956 |
|
| 957 |
// TransformerForResource returns the transformer for the given resource. |
| 958 |
func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer { |
| 959 |
return &resourceTransformer{ |
| 960 |
resource: resource, |
| 961 |
transformTracker: d.transformTracker, |
| 962 |
} |
| 963 |
} |
| 964 |
|
| 965 |
// Set sets the transformer overrides. This method is not go routine safe and must only be called by the same, single caller throughout the lifetime of this object. |
| 966 |
func (d *DynamicTransformers) Set( |
| 967 |
transformerOverrides map[schema.GroupResource]storagevalue.Transformer, |
| 968 |
closeTransformers context.CancelFunc, |
| 969 |
kmsPluginHealthzCheck healthz.HealthChecker, |
| 970 |
kmsCloseGracePeriod time.Duration, |
| 971 |
) { |
| 972 |
// store new values |
| 973 |
newTransformTracker := &transformTracker{ |
| 974 |
transformerOverrides: transformerOverrides, |
| 975 |
closeTransformers: closeTransformers, |
| 976 |
kmsPluginHealthzCheck: kmsPluginHealthzCheck, |
| 977 |
kmsCloseGracePeriod: kmsCloseGracePeriod, |
| 978 |
} |
| 979 |
|
| 980 |
// update new transformer overrides |
| 981 |
oldTransformTracker := d.transformTracker.Swap(newTransformTracker).(*transformTracker) |
| 982 |
|
| 983 |
// close old transformers once we wait for grpc request to finish any in-flight requests. |
| 984 |
// by the time we spawn this go routine, the new transformers have already been set and will be used for new requests. |
| 985 |
// if the server starts shutting down during sleep duration then the transformers will correctly closed early because their lifetime is tied to the api-server drain notifier. |
| 986 |
go func() { |
| 987 |
time.Sleep(oldTransformTracker.kmsCloseGracePeriod) |
| 988 |
oldTransformTracker.closeTransformers() |
| 989 |
}() |
| 990 |
} |
| 991 |
|
| 992 |
var _ storagevalue.Transformer = &resourceTransformer{} |
| 993 |
|
| 994 |
type resourceTransformer struct { |
| 995 |
resource schema.GroupResource |
| 996 |
transformTracker *atomic.Value |
| 997 |
} |
| 998 |
|
| 999 |
func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, bool, error) { |
| 1000 |
return r.transformer().TransformFromStorage(ctx, data, dataCtx) |
| 1001 |
} |
| 1002 |
|
| 1003 |
func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, error) { |
| 1004 |
return r.transformer().TransformToStorage(ctx, data, dataCtx) |
| 1005 |
} |
| 1006 |
|
| 1007 |
func (r *resourceTransformer) transformer() storagevalue.Transformer { |
| 1008 |
return transformerFromOverrides(r.transformTracker.Load().(*transformTracker).transformerOverrides, r.resource) |
| 1009 |
} |
| 1010 |
|
| 1011 |
var _ storagevalue.ResourceTransformers = &StaticTransformers{} |
| 1012 |
|
| 1013 |
type StaticTransformers map[schema.GroupResource]storagevalue.Transformer |
| 1014 |
|
| 1015 |
func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer { |
| 1016 |
return transformerFromOverrides(s, resource) |
| 1017 |
} |
| 1018 |
|
| 1019 |
var anyGroupAnyResource = schema.GroupResource{ |
| 1020 |
Group: "*", |
| 1021 |
Resource: "*", |
| 1022 |
} |
| 1023 |
|
| 1024 |
func transformerFromOverrides(transformerOverrides map[schema.GroupResource]storagevalue.Transformer, resource schema.GroupResource) (out storagevalue.Transformer) { |
| 1025 |
defer func() { out = newRequestInfoTransformer(resource, out) }() |
| 1026 |
|
| 1027 |
if transformer := transformerOverrides[resource]; transformer != nil { |
| 1028 |
return transformer |
| 1029 |
} |
| 1030 |
|
| 1031 |
if transformer := transformerOverrides[schema.GroupResource{ |
| 1032 |
Group: resource.Group, |
| 1033 |
Resource: "*", |
| 1034 |
}]; transformer != nil { |
| 1035 |
return transformer |
| 1036 |
} |
| 1037 |
|
| 1038 |
if transformer := transformerOverrides[anyGroupAnyResource]; transformer != nil { |
| 1039 |
return transformer |
| 1040 |
} |
| 1041 |
|
| 1042 |
return identity.NewEncryptCheckTransformer() |
| 1043 |
} |
| 1044 |
|
| 1045 |
func grYAMLString(gr schema.GroupResource) string { |
| 1046 |
if gr.Group == "" && gr.Resource == "*" { |
| 1047 |
return "*." |
| 1048 |
} |
| 1049 |
|
| 1050 |
return gr.String() |
| 1051 |
} |
| 1052 |
|
| 1053 |
var _ storagevalue.Transformer = &requestInfoTransformer{} |
| 1054 |
|
| 1055 |
type requestInfoTransformer struct { |
| 1056 |
baseValueCtx context.Context |
| 1057 |
delegate storagevalue.Transformer |
| 1058 |
} |
| 1059 |
|
| 1060 |
func newRequestInfoTransformer(resource schema.GroupResource, delegate storagevalue.Transformer) *requestInfoTransformer { |
| 1061 |
return &requestInfoTransformer{ |
| 1062 |
baseValueCtx: request.WithRequestInfo(context.Background(), &request.RequestInfo{IsResourceRequest: true, APIGroup: resource.Group, Resource: resource.Resource}), |
| 1063 |
delegate: delegate, |
| 1064 |
} |
| 1065 |
} |
| 1066 |
|
| 1067 |
func (l *requestInfoTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, bool, error) { |
| 1068 |
return l.delegate.TransformFromStorage(l.withBaseValueCtx(ctx), data, dataCtx) |
| 1069 |
} |
| 1070 |
|
| 1071 |
func (l *requestInfoTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, error) { |
| 1072 |
return l.delegate.TransformToStorage(l.withBaseValueCtx(ctx), data, dataCtx) |
| 1073 |
} |
| 1074 |
|
| 1075 |
func (l *requestInfoTransformer) withBaseValueCtx(ctx context.Context) context.Context { |
| 1076 |
return &joinValueContext{Context: ctx, baseValueCtx: l.baseValueCtx} |
| 1077 |
} |
| 1078 |
|
| 1079 |
type joinValueContext struct { |
| 1080 |
context.Context |
| 1081 |
baseValueCtx context.Context |
| 1082 |
} |
| 1083 |
|
| 1084 |
func (j *joinValueContext) Value(key any) any { |
| 1085 |
if val := j.Context.Value(key); val != nil { |
| 1086 |
return val |
| 1087 |
} |
| 1088 |
return j.baseValueCtx.Value(key) |
| 1089 |
} |