Debian Codegraph @

pkg/storage/value/encrypt/envelope/kmsv2/envelope.go

raw
1 /*
2 Copyright 2022 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 // Package kmsv2 transforms values for storage at rest using a Envelope v2 provider
18 package kmsv2
19
20 import (
21 "context"
22 "crypto/aes"
23 "crypto/cipher"
24 "crypto/sha256"
25 "fmt"
26 "sort"
27 "time"
28 "unsafe"
29
30 "github.com/gogo/protobuf/proto"
31 "go.opentelemetry.io/otel/attribute"
32 "golang.org/x/crypto/cryptobyte"
33
34 utilerrors "k8s.io/apimachinery/pkg/util/errors"
35 "k8s.io/apimachinery/pkg/util/uuid"
36 "k8s.io/apimachinery/pkg/util/validation"
37 "k8s.io/apimachinery/pkg/util/validation/field"
38 genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
39 "k8s.io/apiserver/pkg/storage/value"
40 aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
41 kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
42 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
43 "k8s.io/component-base/tracing"
44 "k8s.io/klog/v2"
45 kmsservice "k8s.io/kms/pkg/service"
46 "k8s.io/utils/clock"
47 )
48
49 func init() {
50 value.RegisterMetrics()
51 metrics.RegisterMetrics()
52 }
53
54 const (
55 // KMSAPIVersionv2 is a version of the KMS API.
56 KMSAPIVersionv2 = "v2"
57 // KMSAPIVersionv2beta1 is a version of the KMS API.
58 KMSAPIVersionv2beta1 = "v2beta1"
59 // annotationsMaxSize is the maximum size of the annotations.
60 annotationsMaxSize = 32 * 1024 // 32 kB
61 // KeyIDMaxSize is the maximum size of the keyID.
62 KeyIDMaxSize = 1 * 1024 // 1 kB
63 // encryptedDEKSourceMaxSize is the maximum size of the encrypted DEK source.
64 encryptedDEKSourceMaxSize = 1 * 1024 // 1 kB
65 // cacheTTL is the default time-to-live for the cache entry.
66 // this allows the cache to grow to an infinite size for up to a day.
67 // there is unlikely to be any meaningful memory impact on the server
68 // because the cache will likely never have more than a few thousand entries.
69 // each entry can be large due to an internal cache that maps the DEK seed to individual
70 // DEK entries, but that cache has an aggressive TTL to keep the size under control.
71 // with DEK/seed reuse and no storage migration, the number of entries in this cache
72 // would be approximated by unique key IDs used by the KMS plugin
73 // combined with the number of server restarts. If storage migration
74 // is performed after key ID changes, and the number of restarts
75 // is limited, this cache size may be as small as the number of API
76 // servers in use (once old entries expire out from the TTL).
77 cacheTTL = 24 * time.Hour
78 // key ID related error codes for metrics
79 errKeyIDOKCode ErrCodeKeyID = "ok"
80 errKeyIDEmptyCode ErrCodeKeyID = "empty"
81 errKeyIDTooLongCode ErrCodeKeyID = "too_long"
82 )
83
84 // NowFunc is exported so tests can override it.
85 var NowFunc = time.Now
86
87 type StateFunc func() (State, error)
88 type ErrCodeKeyID string
89
90 type State struct {
91 Transformer value.Transformer
92
93 EncryptedObject kmstypes.EncryptedObject
94
95 UID string
96
97 ExpirationTimestamp time.Time
98
99 // CacheKey is the key used to cache the DEK/seed in envelopeTransformer.cache.
100 CacheKey []byte
101 }
102
103 func (s *State) ValidateEncryptCapability() error {
104 if now := NowFunc(); now.After(s.ExpirationTimestamp) {
105 return fmt.Errorf("encryptedDEKSource with keyID hash %q expired at %s (current time is %s)",
106 GetHashIfNotEmpty(s.EncryptedObject.KeyID), s.ExpirationTimestamp.Format(time.RFC3339), now.Format(time.RFC3339))
107 }
108 return nil
109 }
110
111 type envelopeTransformer struct {
112 envelopeService kmsservice.Service
113 providerName string
114 stateFunc StateFunc
115
116 // cache is a thread-safe expiring lru cache which caches decrypted DEKs indexed by their encrypted form.
117 cache *simpleCache
118 apiServerID string
119 }
120
121 // NewEnvelopeTransformer returns a transformer which implements a KEK-DEK based envelope encryption scheme.
122 // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
123 // the data items they encrypt.
124 func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string) value.Transformer {
125 return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, apiServerID, cacheTTL, clock.RealClock{})
126 }
127
128 func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string, cacheTTL time.Duration, clock clock.Clock) value.Transformer {
129 return &envelopeTransformer{
130 envelopeService: envelopeService,
131 providerName: providerName,
132 stateFunc: stateFunc,
133 cache: newSimpleCache(clock, cacheTTL, providerName),
134 apiServerID: apiServerID,
135 }
136 }
137
138 // TransformFromStorage decrypts data encrypted by this transformer using envelope encryption.
139 func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
140 ctx, span := tracing.Start(ctx, "TransformFromStorage with envelopeTransformer",
141 attribute.String("transformer.provider.name", t.providerName),
142 // The service.instance_id of the apiserver is already available in the trace
143 /*
144 {
145 "key": "service.instance.id",
146 "type": "string",
147 "value": "apiserver-zsteyir5lyrtdcmqqmd5kzze6m"
148 }
149 */
150 )
151 defer span.End(500 * time.Millisecond)
152
153 span.AddEvent("About to decode encrypted object")
154 // Deserialize the EncryptedObject from the data.
155 encryptedObject, err := t.doDecode(data)
156 if err != nil {
157 span.AddEvent("Decoding encrypted object failed")
158 span.RecordError(err)
159 return nil, false, err
160 }
161 span.AddEvent("Decoded encrypted object")
162
163 useSeed := encryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
164
165 // TODO: consider marking state.EncryptedDEK != encryptedObject.EncryptedDEK as a stale read to support DEK defragmentation
166 // at a minimum we should have a metric that helps the user understand if DEK fragmentation is high
167 state, err := t.stateFunc() // no need to call state.ValidateEncryptCapability on reads
168 if err != nil {
169 return nil, false, err
170 }
171
172 encryptedObjectCacheKey, err := generateCacheKey(encryptedObject.EncryptedDEKSourceType, encryptedObject.EncryptedDEKSource, encryptedObject.KeyID, encryptedObject.Annotations)
173 if err != nil {
174 return nil, false, err
175 }
176
177 // Look up the decrypted DEK from cache first
178 transformer := t.cache.get(encryptedObjectCacheKey)
179
180 // fallback to the envelope service if we do not have the transformer locally
181 if transformer == nil {
182 span.AddEvent("About to decrypt DEK using remote service")
183 value.RecordCacheMiss()
184
185 requestInfo := getRequestInfoFromContext(ctx)
186 uid := string(uuid.NewUUID())
187 klog.V(6).InfoS("decrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData()),
188 "group", requestInfo.APIGroup, "version", requestInfo.APIVersion, "resource", requestInfo.Resource, "subresource", requestInfo.Subresource,
189 "verb", requestInfo.Verb, "namespace", requestInfo.Namespace, "name", requestInfo.Name)
190
191 key, err := t.envelopeService.Decrypt(ctx, uid, &kmsservice.DecryptRequest{
192 Ciphertext: encryptedObject.EncryptedDEKSource,
193 KeyID: encryptedObject.KeyID,
194 Annotations: encryptedObject.Annotations,
195 })
196 if err != nil {
197 span.AddEvent("DEK decryption failed")
198 span.RecordError(err)
199 return nil, false, fmt.Errorf("failed to decrypt DEK, error: %w", err)
200 }
201 span.AddEvent("DEK decryption succeeded")
202
203 transformer, err = t.addTransformerForDecryption(encryptedObjectCacheKey, key, useSeed)
204 if err != nil {
205 return nil, false, err
206 }
207 }
208 metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID, t.apiServerID)
209
210 span.AddEvent("About to decrypt data using DEK")
211 out, stale, err := transformer.TransformFromStorage(ctx, encryptedObject.EncryptedData, dataCtx)
212 if err != nil {
213 span.AddEvent("Data decryption failed")
214 span.RecordError(err)
215 return nil, false, err
216 }
217
218 span.AddEvent("Data decryption succeeded")
219 // data is considered stale if the key ID does not match our current write transformer
220 return out,
221 stale ||
222 encryptedObject.KeyID != state.EncryptedObject.KeyID ||
223 encryptedObject.EncryptedDEKSourceType != state.EncryptedObject.EncryptedDEKSourceType,
224 nil
225 }
226
227 // TransformToStorage encrypts data to be written to disk using envelope encryption.
228 func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
229 ctx, span := tracing.Start(ctx, "TransformToStorage with envelopeTransformer",
230 attribute.String("transformer.provider.name", t.providerName),
231 // The service.instance_id of the apiserver is already available in the trace
232 /*
233 {
234 "key": "service.instance.id",
235 "type": "string",
236 "value": "apiserver-zsteyir5lyrtdcmqqmd5kzze6m"
237 }
238 */
239 )
240 defer span.End(500 * time.Millisecond)
241
242 state, err := t.stateFunc()
243 if err != nil {
244 return nil, err
245 }
246 if err := state.ValidateEncryptCapability(); err != nil {
247 return nil, err
248 }
249
250 // this prevents a cache miss every time the DEK rotates
251 // this has the side benefit of causing the cache to perform a GC
252 // TODO see if we can do this inside the stateFunc control loop
253 t.cache.set(state.CacheKey, state.Transformer)
254
255 requestInfo := getRequestInfoFromContext(ctx)
256 klog.V(6).InfoS("encrypting content using DEK", "uid", state.UID, "key", string(dataCtx.AuthenticatedData()),
257 "group", requestInfo.APIGroup, "version", requestInfo.APIVersion, "resource", requestInfo.Resource, "subresource", requestInfo.Subresource,
258 "verb", requestInfo.Verb, "namespace", requestInfo.Namespace, "name", requestInfo.Name)
259
260 span.AddEvent("About to encrypt data using DEK")
261 result, err := state.Transformer.TransformToStorage(ctx, data, dataCtx)
262 if err != nil {
263 span.AddEvent("Data encryption failed")
264 span.RecordError(err)
265 return nil, err
266 }
267 span.AddEvent("Data encryption succeeded")
268
269 metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID, t.apiServerID)
270
271 encObjectCopy := state.EncryptedObject
272 encObjectCopy.EncryptedData = result
273
274 span.AddEvent("About to encode encrypted object")
275 // Serialize the EncryptedObject to a byte array.
276 out, err := t.doEncode(&encObjectCopy)
277 if err != nil {
278 span.AddEvent("Encoding encrypted object failed")
279 span.RecordError(err)
280 return nil, err
281 }
282 span.AddEvent("Encoded encrypted object")
283
284 return out, nil
285 }
286
287 // addTransformerForDecryption inserts a new transformer to the Envelope cache of DEKs for future reads.
288 func (t *envelopeTransformer) addTransformerForDecryption(cacheKey []byte, key []byte, useSeed bool) (value.Read, error) {
289 var transformer value.Read
290 var err error
291 if useSeed {
292 // the input key is considered safe to use here because it is coming from the KMS plugin / etcd
293 transformer, err = aestransformer.NewHKDFExtendedNonceGCMTransformer(key)
294 } else {
295 var block cipher.Block
296 block, err = aes.NewCipher(key)
297 if err != nil {
298 return nil, err
299 }
300 // this is compatible with NewGCMTransformerWithUniqueKeyUnsafe for decryption
301 // it would use random nonces for encryption but we never do that
302 transformer, err = aestransformer.NewGCMTransformer(block)
303 }
304 if err != nil {
305 return nil, err
306 }
307 t.cache.set(cacheKey, transformer)
308 return transformer, nil
309 }
310
311 // doEncode encodes the EncryptedObject to a byte array.
312 func (t *envelopeTransformer) doEncode(request *kmstypes.EncryptedObject) ([]byte, error) {
313 if err := ValidateEncryptedObject(request); err != nil {
314 return nil, err
315 }
316 return proto.Marshal(request)
317 }
318
319 // doDecode decodes the byte array to an EncryptedObject.
320 func (t *envelopeTransformer) doDecode(originalData []byte) (*kmstypes.EncryptedObject, error) {
321 o := &kmstypes.EncryptedObject{}
322 if err := proto.Unmarshal(originalData, o); err != nil {
323 return nil, err
324 }
325 if err := ValidateEncryptedObject(o); err != nil {
326 return nil, err
327 }
328
329 return o, nil
330 }
331
332 // GenerateTransformer generates a new transformer and encrypts the DEK/seed using the envelope service.
333 // It returns the transformer, the encrypted DEK/seed, cache key and error.
334 func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service, useSeed bool) (value.Transformer, *kmstypes.EncryptedObject, []byte, error) {
335 newTransformerFunc := func() (value.Transformer, []byte, error) {
336 seed, err := aestransformer.GenerateKey(aestransformer.MinSeedSizeExtendedNonceGCM)
337 if err != nil {
338 return nil, nil, err
339 }
340 transformer, err := aestransformer.NewHKDFExtendedNonceGCMTransformer(seed)
341 if err != nil {
342 return nil, nil, err
343 }
344 return transformer, seed, nil
345 }
346 if !useSeed {
347 newTransformerFunc = aestransformer.NewGCMTransformerWithUniqueKeyUnsafe
348 }
349 transformer, newKey, err := newTransformerFunc()
350 if err != nil {
351 return nil, nil, nil, err
352 }
353
354 klog.V(6).InfoS("encrypting content using envelope service", "uid", uid)
355
356 resp, err := envelopeService.Encrypt(ctx, uid, newKey)
357 if err != nil {
358 return nil, nil, nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)
359 }
360
361 o := &kmstypes.EncryptedObject{
362 KeyID: resp.KeyID,
363 EncryptedDEKSource: resp.Ciphertext,
364 EncryptedData: []byte{0}, // any non-empty value to pass validation
365 Annotations: resp.Annotations,
366 }
367
368 if useSeed {
369 o.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
370 } else {
371 o.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_AES_GCM_KEY
372 }
373
374 if err := ValidateEncryptedObject(o); err != nil {
375 return nil, nil, nil, err
376 }
377
378 cacheKey, err := generateCacheKey(o.EncryptedDEKSourceType, resp.Ciphertext, resp.KeyID, resp.Annotations)
379 if err != nil {
380 return nil, nil, nil, err
381 }
382
383 o.EncryptedData = nil // make sure that later code that uses this encrypted object sets this field
384
385 return transformer, o, cacheKey, nil
386 }
387
388 func ValidateEncryptedObject(o *kmstypes.EncryptedObject) error {
389 if o == nil {
390 return fmt.Errorf("encrypted object is nil")
391 }
392 switch t := o.EncryptedDEKSourceType; t {
393 case kmstypes.EncryptedDEKSourceType_AES_GCM_KEY:
394 case kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED:
395 default:
396 return fmt.Errorf("unknown encryptedDEKSourceType: %d", t)
397 }
398 if len(o.EncryptedData) == 0 {
399 return fmt.Errorf("encrypted data is empty")
400 }
401 if err := validateEncryptedDEKSource(o.EncryptedDEKSource); err != nil {
402 return fmt.Errorf("failed to validate encrypted DEK source: %w", err)
403 }
404 if _, err := ValidateKeyID(o.KeyID); err != nil {
405 return fmt.Errorf("failed to validate key id: %w", err)
406 }
407 if err := validateAnnotations(o.Annotations); err != nil {
408 return fmt.Errorf("failed to validate annotations: %w", err)
409 }
410 return nil
411 }
412
413 // validateEncryptedDEKSource tests the following:
414 // 1. The encrypted DEK source is not empty.
415 // 2. The size of encrypted DEK source is less than 1 kB.
416 func validateEncryptedDEKSource(encryptedDEKSource []byte) error {
417 if len(encryptedDEKSource) == 0 {
418 return fmt.Errorf("encrypted DEK source is empty")
419 }
420 if len(encryptedDEKSource) > encryptedDEKSourceMaxSize {
421 return fmt.Errorf("encrypted DEK source is %d bytes, which exceeds the max size of %d", len(encryptedDEKSource), encryptedDEKSourceMaxSize)
422 }
423 return nil
424 }
425
426 // validateAnnotations tests the following:
427 // 1. checks if the annotation key is fully qualified
428 // 2. The size of annotations keys + values is less than 32 kB.
429 func validateAnnotations(annotations map[string][]byte) error {
430 var errs []error
431 var totalSize uint64
432 for k, v := range annotations {
433 if fieldErr := validation.IsFullyQualifiedDomainName(field.NewPath("annotations"), k); fieldErr != nil {
434 errs = append(errs, fieldErr.ToAggregate())
435 }
436 totalSize += uint64(len(k)) + uint64(len(v))
437 }
438 if totalSize > annotationsMaxSize {
439 errs = append(errs, fmt.Errorf("total size of annotations is %d, which exceeds the max size of %d", totalSize, annotationsMaxSize))
440 }
441 return utilerrors.NewAggregate(errs)
442 }
443
444 // ValidateKeyID tests the following:
445 // 1. The keyID is not empty.
446 // 2. The size of keyID is less than 1 kB.
447 func ValidateKeyID(keyID string) (ErrCodeKeyID, error) {
448 if len(keyID) == 0 {
449 return errKeyIDEmptyCode, fmt.Errorf("keyID is empty")
450 }
451 if len(keyID) > KeyIDMaxSize {
452 return errKeyIDTooLongCode, fmt.Errorf("keyID is %d bytes, which exceeds the max size of %d", len(keyID), KeyIDMaxSize)
453 }
454 return errKeyIDOKCode, nil
455 }
456
457 func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestInfo {
458 if reqInfo, found := genericapirequest.RequestInfoFrom(ctx); found {
459 return reqInfo
460 }
461 return &genericapirequest.RequestInfo{}
462 }
463
464 // generateCacheKey returns a key for the cache.
465 // The key is a concatenation of:
466 // 0. encryptedDEKSourceType
467 // 1. encryptedDEKSource
468 // 2. keyID
469 // 3. length of annotations
470 // 4. annotations (sorted by key) - each annotation is a concatenation of:
471 // a. annotation key
472 // b. annotation value
473 func generateCacheKey(encryptedDEKSourceType kmstypes.EncryptedDEKSourceType, encryptedDEKSource []byte, keyID string, annotations map[string][]byte) ([]byte, error) {
474 // TODO(aramase): use sync pool buffer to avoid allocations
475 b := cryptobyte.NewBuilder(nil)
476 b.AddUint32(uint32(encryptedDEKSourceType))
477 b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
478 b.AddBytes(encryptedDEKSource)
479 })
480 b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
481 b.AddBytes(toBytes(keyID))
482 })
483 if len(annotations) == 0 {
484 return b.Bytes()
485 }
486
487 // add the length of annotations to the cache key
488 b.AddUint32(uint32(len(annotations)))
489
490 // Sort the annotations by key.
491 keys := make([]string, 0, len(annotations))
492 for k := range annotations {
493 k := k
494 keys = append(keys, k)
495 }
496 sort.Strings(keys)
497 for _, k := range keys {
498 // The maximum size of annotations is annotationsMaxSize (32 kB) so we can safely
499 // assume that the length of the key and value will fit in a uint16.
500 b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
501 b.AddBytes(toBytes(k))
502 })
503 b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
504 b.AddBytes(annotations[k])
505 })
506 }
507
508 return b.Bytes()
509 }
510
511 // toBytes performs unholy acts to avoid allocations
512 func toBytes(s string) []byte {
513 // unsafe.StringData is unspecified for the empty string, so we provide a strict interpretation
514 if len(s) == 0 {
515 return nil
516 }
517 // Copied from go 1.20.1 os.File.WriteString
518 // https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/os/file.go#L246
519 return unsafe.Slice(unsafe.StringData(s), len(s))
520 }
521
522 // GetHashIfNotEmpty returns the sha256 hash of the data if it is not empty.
523 func GetHashIfNotEmpty(data string) string {
524 if len(data) > 0 {
525 return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data)))
526 }
527 return ""
528 }