Debian Codegraph @

pkg/storage/etcd3/store.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package etcd3
18
19 import (
20 "bytes"
21 "context"
22 "fmt"
23 "path"
24 "reflect"
25 "strconv"
26 "strings"
27 "time"
28
29 "go.etcd.io/etcd/api/v3/mvccpb"
30 clientv3 "go.etcd.io/etcd/client/v3"
31 "go.etcd.io/etcd/client/v3/kubernetes"
32 "go.opentelemetry.io/otel/attribute"
33
34 apierrors "k8s.io/apimachinery/pkg/api/errors"
35 "k8s.io/apimachinery/pkg/api/meta"
36 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
37 "k8s.io/apimachinery/pkg/conversion"
38 "k8s.io/apimachinery/pkg/fields"
39 "k8s.io/apimachinery/pkg/labels"
40 "k8s.io/apimachinery/pkg/runtime"
41 "k8s.io/apimachinery/pkg/runtime/schema"
42 "k8s.io/apimachinery/pkg/watch"
43 "k8s.io/apiserver/pkg/audit"
44 "k8s.io/apiserver/pkg/features"
45 "k8s.io/apiserver/pkg/storage"
46 "k8s.io/apiserver/pkg/storage/etcd3/metrics"
47 etcdfeature "k8s.io/apiserver/pkg/storage/feature"
48 "k8s.io/apiserver/pkg/storage/value"
49 utilfeature "k8s.io/apiserver/pkg/util/feature"
50 "k8s.io/component-base/tracing"
51 "k8s.io/klog/v2"
52 )
53
54 const (
55 // maxLimit is a maximum page limit increase used when fetching objects from etcd.
56 // This limit is used only for increasing page size by kube-apiserver. If request
57 // specifies larger limit initially, it won't be changed.
58 maxLimit = 10000
59 )
60
61 // authenticatedDataString satisfies the value.Context interface. It uses the key to
62 // authenticate the stored data. This does not defend against reuse of previously
63 // encrypted values under the same key, but will prevent an attacker from using an
64 // encrypted value from a different key. A stronger authenticated data segment would
65 // include the etcd3 Version field (which is incremented on each write to a key and
66 // reset when the key is deleted), but an attacker with write access to etcd can
67 // force deletion and recreation of keys to weaken that angle.
68 type authenticatedDataString string
69
70 // AuthenticatedData implements the value.Context interface.
71 func (d authenticatedDataString) AuthenticatedData() []byte {
72 return []byte(string(d))
73 }
74
75 var _ value.Context = authenticatedDataString("")
76
77 type store struct {
78 client *kubernetes.Client
79 codec runtime.Codec
80 versioner storage.Versioner
81 transformer value.Transformer
82 pathPrefix string
83 groupResource schema.GroupResource
84 groupResourceString string
85 watcher *watcher
86 leaseManager *leaseManager
87 decoder Decoder
88 listErrAggrFactory func() ListErrorAggregator
89
90 resourcePrefix string
91 newListFunc func() runtime.Object
92 }
93
94 func (s *store) RequestWatchProgress(ctx context.Context) error {
95 // Use watchContext to match ctx metadata provided when creating the watch.
96 // In best case scenario we would use the same context that watch was created, but there is no way access it from watchCache.
97 return s.client.RequestProgress(s.watchContext(ctx))
98 }
99
100 type objState struct {
101 obj runtime.Object
102 meta *storage.ResponseMeta
103 rev int64
104 data []byte
105 stale bool
106 }
107
108 // ListErrorAggregator aggregates the error(s) that the LIST operation
109 // encounters while retrieving object(s) from the storage
110 type ListErrorAggregator interface {
111 // Aggregate aggregates the given error from list operation
112 // key: it identifies the given object in the storage.
113 // err: it represents the error the list operation encountered while
114 // retrieving the given object from the storage.
115 // done: true if the aggregation is done and the list operation should
116 // abort, otherwise the list operation will continue
117 Aggregate(key string, err error) bool
118
119 // Err returns the aggregated error
120 Err() error
121 }
122
123 // defaultListErrorAggregatorFactory returns the default list error
124 // aggregator that maintains backward compatibility, which is abort
125 // the list operation as soon as it encounters the first error
126 func defaultListErrorAggregatorFactory() ListErrorAggregator { return &abortOnFirstError{} }
127
128 // LIST aborts on the first error it encounters (backward compatible)
129 type abortOnFirstError struct {
130 err error
131 }
132
133 func (a *abortOnFirstError) Aggregate(key string, err error) bool {
134 a.err = err
135 return true
136 }
137 func (a *abortOnFirstError) Err() error { return a.err }
138
139 // New returns an etcd3 implementation of storage.Interface.
140 func New(c *kubernetes.Client, codec runtime.Codec, newFunc, newListFunc func() runtime.Object, prefix, resourcePrefix string, groupResource schema.GroupResource, transformer value.Transformer, leaseManagerConfig LeaseManagerConfig, decoder Decoder, versioner storage.Versioner) storage.Interface {
141 if utilfeature.DefaultFeatureGate.Enabled(features.AllowUnsafeMalformedObjectDeletion) {
142 transformer = WithCorruptObjErrorHandlingTransformer(transformer)
143 decoder = WithCorruptObjErrorHandlingDecoder(decoder)
144 }
145 var store storage.Interface
146 store = newStore(c, codec, newFunc, newListFunc, prefix, resourcePrefix, groupResource, transformer, leaseManagerConfig, decoder, versioner)
147 if utilfeature.DefaultFeatureGate.Enabled(features.AllowUnsafeMalformedObjectDeletion) {
148 store = NewStoreWithUnsafeCorruptObjectDeletion(store, groupResource)
149 }
150 return store
151 }
152
153 func newStore(c *kubernetes.Client, codec runtime.Codec, newFunc, newListFunc func() runtime.Object, prefix, resourcePrefix string, groupResource schema.GroupResource, transformer value.Transformer, leaseManagerConfig LeaseManagerConfig, decoder Decoder, versioner storage.Versioner) *store {
154 // for compatibility with etcd2 impl.
155 // no-op for default prefix of '/registry'.
156 // keeps compatibility with etcd2 impl for custom prefixes that don't start with '/'
157 pathPrefix := path.Join("/", prefix)
158 if !strings.HasSuffix(pathPrefix, "/") {
159 // Ensure the pathPrefix ends in "/" here to simplify key concatenation later.
160 pathPrefix += "/"
161 }
162
163 listErrAggrFactory := defaultListErrorAggregatorFactory
164 if utilfeature.DefaultFeatureGate.Enabled(features.AllowUnsafeMalformedObjectDeletion) {
165 listErrAggrFactory = corruptObjErrAggregatorFactory(100)
166 }
167
168 w := &watcher{
169 client: c.Client,
170 codec: codec,
171 newFunc: newFunc,
172 groupResource: groupResource,
173 versioner: versioner,
174 transformer: transformer,
175 }
176 if newFunc == nil {
177 w.objectType = "<unknown>"
178 } else {
179 w.objectType = reflect.TypeOf(newFunc()).String()
180 }
181 s := &store{
182 client: c,
183 codec: codec,
184 versioner: versioner,
185 transformer: transformer,
186 pathPrefix: pathPrefix,
187 groupResource: groupResource,
188 groupResourceString: groupResource.String(),
189 watcher: w,
190 leaseManager: newDefaultLeaseManager(c.Client, leaseManagerConfig),
191 decoder: decoder,
192 listErrAggrFactory: listErrAggrFactory,
193
194 resourcePrefix: resourcePrefix,
195 newListFunc: newListFunc,
196 }
197
198 w.getCurrentStorageRV = func(ctx context.Context) (uint64, error) {
199 return s.GetCurrentResourceVersion(ctx)
200 }
201 if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCache) || utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
202 etcdfeature.DefaultFeatureSupportChecker.CheckClient(c.Ctx(), c, storage.RequestWatchProgress)
203 }
204 return s
205 }
206
207 // Versioner implements storage.Interface.Versioner.
208 func (s *store) Versioner() storage.Versioner {
209 return s.versioner
210 }
211
212 // Get implements storage.Interface.Get.
213 func (s *store) Get(ctx context.Context, key string, opts storage.GetOptions, out runtime.Object) error {
214 preparedKey, err := s.prepareKey(key)
215 if err != nil {
216 return err
217 }
218 startTime := time.Now()
219 getResp, err := s.client.Kubernetes.Get(ctx, preparedKey, kubernetes.GetOptions{})
220 metrics.RecordEtcdRequest("get", s.groupResourceString, err, startTime)
221 if err != nil {
222 return err
223 }
224 if err = s.validateMinimumResourceVersion(opts.ResourceVersion, uint64(getResp.Revision)); err != nil {
225 return err
226 }
227
228 if getResp.KV == nil {
229 if opts.IgnoreNotFound {
230 return runtime.SetZeroValue(out)
231 }
232 return storage.NewKeyNotFoundError(preparedKey, 0)
233 }
234
235 data, _, err := s.transformer.TransformFromStorage(ctx, getResp.KV.Value, authenticatedDataString(preparedKey))
236 if err != nil {
237 return storage.NewInternalError(err)
238 }
239
240 err = s.decoder.Decode(data, out, getResp.KV.ModRevision)
241 if err != nil {
242 recordDecodeError(s.groupResourceString, preparedKey)
243 return err
244 }
245 return nil
246 }
247
248 // Create implements storage.Interface.Create.
249 func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error {
250 preparedKey, err := s.prepareKey(key)
251 if err != nil {
252 return err
253 }
254 ctx, span := tracing.Start(ctx, "Create etcd3",
255 attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)),
256 attribute.String("key", key),
257 attribute.String("type", getTypeName(obj)),
258 attribute.String("resource", s.groupResourceString),
259 )
260 defer span.End(500 * time.Millisecond)
261 if version, err := s.versioner.ObjectResourceVersion(obj); err == nil && version != 0 {
262 return storage.ErrResourceVersionSetOnCreate
263 }
264 if err := s.versioner.PrepareObjectForStorage(obj); err != nil {
265 return fmt.Errorf("PrepareObjectForStorage failed: %v", err)
266 }
267 span.AddEvent("About to Encode")
268 data, err := runtime.Encode(s.codec, obj)
269 if err != nil {
270 span.AddEvent("Encode failed", attribute.Int("len", len(data)), attribute.String("err", err.Error()))
271 return err
272 }
273 span.AddEvent("Encode succeeded", attribute.Int("len", len(data)))
274
275 var lease clientv3.LeaseID
276 if ttl != 0 {
277 lease, err = s.leaseManager.GetLease(ctx, int64(ttl))
278 if err != nil {
279 return err
280 }
281 }
282
283 newData, err := s.transformer.TransformToStorage(ctx, data, authenticatedDataString(preparedKey))
284 if err != nil {
285 span.AddEvent("TransformToStorage failed", attribute.String("err", err.Error()))
286 return storage.NewInternalError(err)
287 }
288 span.AddEvent("TransformToStorage succeeded")
289
290 startTime := time.Now()
291 txnResp, err := s.client.Kubernetes.OptimisticPut(ctx, preparedKey, newData, 0, kubernetes.PutOptions{LeaseID: lease})
292 metrics.RecordEtcdRequest("create", s.groupResourceString, err, startTime)
293 if err != nil {
294 span.AddEvent("Txn call failed", attribute.String("err", err.Error()))
295 return err
296 }
297 span.AddEvent("Txn call succeeded")
298
299 if !txnResp.Succeeded {
300 return storage.NewKeyExistsError(preparedKey, 0)
301 }
302
303 if out != nil {
304 err = s.decoder.Decode(data, out, txnResp.Revision)
305 if err != nil {
306 span.AddEvent("decode failed", attribute.Int("len", len(data)), attribute.String("err", err.Error()))
307 recordDecodeError(s.groupResourceString, preparedKey)
308 return err
309 }
310 span.AddEvent("decode succeeded", attribute.Int("len", len(data)))
311 }
312 return nil
313 }
314
315 // Delete implements storage.Interface.Delete.
316 func (s *store) Delete(
317 ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions,
318 validateDeletion storage.ValidateObjectFunc, cachedExistingObject runtime.Object, opts storage.DeleteOptions) error {
319 preparedKey, err := s.prepareKey(key)
320 if err != nil {
321 return err
322 }
323 v, err := conversion.EnforcePtr(out)
324 if err != nil {
325 return fmt.Errorf("unable to convert output object to pointer: %v", err)
326 }
327
328 skipTransformDecode := false
329 if utilfeature.DefaultFeatureGate.Enabled(features.AllowUnsafeMalformedObjectDeletion) {
330 skipTransformDecode = opts.IgnoreStoreReadError
331 }
332 return s.conditionalDelete(ctx, preparedKey, out, v, preconditions, validateDeletion, cachedExistingObject, skipTransformDecode)
333 }
334
335 func (s *store) conditionalDelete(
336 ctx context.Context, key string, out runtime.Object, v reflect.Value, preconditions *storage.Preconditions,
337 validateDeletion storage.ValidateObjectFunc, cachedExistingObject runtime.Object, skipTransformDecode bool) error {
338 getCurrentState := s.getCurrentState(ctx, key, v, false, skipTransformDecode)
339
340 var origState *objState
341 var err error
342 var origStateIsCurrent bool
343 if cachedExistingObject != nil {
344 origState, err = s.getStateFromObject(cachedExistingObject)
345 } else {
346 origState, err = getCurrentState()
347 origStateIsCurrent = true
348 }
349 if err != nil {
350 return err
351 }
352
353 for {
354 if preconditions != nil {
355 if err := preconditions.Check(key, origState.obj); err != nil {
356 if origStateIsCurrent {
357 return err
358 }
359
360 // It's possible we're working with stale data.
361 // Remember the revision of the potentially stale data and the resulting update error
362 cachedRev := origState.rev
363 cachedUpdateErr := err
364
365 // Actually fetch
366 origState, err = getCurrentState()
367 if err != nil {
368 return err
369 }
370 origStateIsCurrent = true
371
372 // it turns out our cached data was not stale, return the error
373 if cachedRev == origState.rev {
374 return cachedUpdateErr
375 }
376
377 // Retry
378 continue
379 }
380 }
381 if err := validateDeletion(ctx, origState.obj); err != nil {
382 if origStateIsCurrent {
383 return err
384 }
385
386 // It's possible we're working with stale data.
387 // Remember the revision of the potentially stale data and the resulting update error
388 cachedRev := origState.rev
389 cachedUpdateErr := err
390
391 // Actually fetch
392 origState, err = getCurrentState()
393 if err != nil {
394 return err
395 }
396 origStateIsCurrent = true
397
398 // it turns out our cached data was not stale, return the error
399 if cachedRev == origState.rev {
400 return cachedUpdateErr
401 }
402
403 // Retry
404 continue
405 }
406
407 startTime := time.Now()
408 txnResp, err := s.client.Kubernetes.OptimisticDelete(ctx, key, origState.rev, kubernetes.DeleteOptions{
409 GetOnFailure: true,
410 })
411 metrics.RecordEtcdRequest("delete", s.groupResourceString, err, startTime)
412 if err != nil {
413 return err
414 }
415 if !txnResp.Succeeded {
416 klog.V(4).Infof("deletion of %s failed because of a conflict, going to retry", key)
417 origState, err = s.getState(ctx, txnResp.KV, key, v, false, skipTransformDecode)
418 if err != nil {
419 return err
420 }
421 origStateIsCurrent = true
422 continue
423 }
424
425 if !skipTransformDecode {
426 err = s.decoder.Decode(origState.data, out, txnResp.Revision)
427 if err != nil {
428 recordDecodeError(s.groupResourceString, key)
429 return err
430 }
431 }
432 return nil
433 }
434 }
435
436 // GuaranteedUpdate implements storage.Interface.GuaranteedUpdate.
437 func (s *store) GuaranteedUpdate(
438 ctx context.Context, key string, destination runtime.Object, ignoreNotFound bool,
439 preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, cachedExistingObject runtime.Object) error {
440 preparedKey, err := s.prepareKey(key)
441 if err != nil {
442 return err
443 }
444 ctx, span := tracing.Start(ctx, "GuaranteedUpdate etcd3",
445 attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)),
446 attribute.String("key", key),
447 attribute.String("type", getTypeName(destination)),
448 attribute.String("resource", s.groupResourceString))
449 defer span.End(500 * time.Millisecond)
450
451 v, err := conversion.EnforcePtr(destination)
452 if err != nil {
453 return fmt.Errorf("unable to convert output object to pointer: %v", err)
454 }
455
456 skipTransformDecode := false
457 getCurrentState := s.getCurrentState(ctx, preparedKey, v, ignoreNotFound, skipTransformDecode)
458
459 var origState *objState
460 var origStateIsCurrent bool
461 if cachedExistingObject != nil {
462 origState, err = s.getStateFromObject(cachedExistingObject)
463 } else {
464 origState, err = getCurrentState()
465 origStateIsCurrent = true
466 }
467 if err != nil {
468 return err
469 }
470 span.AddEvent("initial value restored")
471
472 transformContext := authenticatedDataString(preparedKey)
473 for {
474 if err := preconditions.Check(preparedKey, origState.obj); err != nil {
475 // If our data is already up to date, return the error
476 if origStateIsCurrent {
477 return err
478 }
479
480 // It's possible we were working with stale data
481 // Actually fetch
482 origState, err = getCurrentState()
483 if err != nil {
484 return err
485 }
486 origStateIsCurrent = true
487 // Retry
488 continue
489 }
490
491 ret, ttl, err := s.updateState(origState, tryUpdate)
492 if err != nil {
493 // If our data is already up to date, return the error
494 if origStateIsCurrent {
495 return err
496 }
497
498 // It's possible we were working with stale data
499 // Remember the revision of the potentially stale data and the resulting update error
500 cachedRev := origState.rev
501 cachedUpdateErr := err
502
503 // Actually fetch
504 origState, err = getCurrentState()
505 if err != nil {
506 return err
507 }
508 origStateIsCurrent = true
509
510 // it turns out our cached data was not stale, return the error
511 if cachedRev == origState.rev {
512 return cachedUpdateErr
513 }
514
515 // Retry
516 continue
517 }
518
519 span.AddEvent("About to Encode")
520 data, err := runtime.Encode(s.codec, ret)
521 if err != nil {
522 span.AddEvent("Encode failed", attribute.Int("len", len(data)), attribute.String("err", err.Error()))
523 return err
524 }
525 span.AddEvent("Encode succeeded", attribute.Int("len", len(data)))
526 if !origState.stale && bytes.Equal(data, origState.data) {
527 // if we skipped the original Get in this loop, we must refresh from
528 // etcd in order to be sure the data in the store is equivalent to
529 // our desired serialization
530 if !origStateIsCurrent {
531 origState, err = getCurrentState()
532 if err != nil {
533 return err
534 }
535 origStateIsCurrent = true
536 if !bytes.Equal(data, origState.data) {
537 // original data changed, restart loop
538 continue
539 }
540 }
541 // recheck that the data from etcd is not stale before short-circuiting a write
542 if !origState.stale {
543 err = s.decoder.Decode(origState.data, destination, origState.rev)
544 if err != nil {
545 recordDecodeError(s.groupResourceString, preparedKey)
546 return err
547 }
548 return nil
549 }
550 }
551
552 newData, err := s.transformer.TransformToStorage(ctx, data, transformContext)
553 if err != nil {
554 span.AddEvent("TransformToStorage failed", attribute.String("err", err.Error()))
555 return storage.NewInternalError(err)
556 }
557 span.AddEvent("TransformToStorage succeeded")
558
559 var lease clientv3.LeaseID
560 if ttl != 0 {
561 lease, err = s.leaseManager.GetLease(ctx, int64(ttl))
562 if err != nil {
563 return err
564 }
565 }
566 span.AddEvent("Transaction prepared")
567
568 startTime := time.Now()
569
570 txnResp, err := s.client.Kubernetes.OptimisticPut(ctx, preparedKey, newData, origState.rev, kubernetes.PutOptions{
571 GetOnFailure: true,
572 LeaseID: lease,
573 })
574 metrics.RecordEtcdRequest("update", s.groupResourceString, err, startTime)
575 if err != nil {
576 span.AddEvent("Txn call failed", attribute.String("err", err.Error()))
577 return err
578 }
579 span.AddEvent("Txn call completed")
580 span.AddEvent("Transaction committed")
581 if !txnResp.Succeeded {
582 klog.V(4).Infof("GuaranteedUpdate of %s failed because of a conflict, going to retry", preparedKey)
583 origState, err = s.getState(ctx, txnResp.KV, preparedKey, v, ignoreNotFound, skipTransformDecode)
584 if err != nil {
585 return err
586 }
587 span.AddEvent("Retry value restored")
588 origStateIsCurrent = true
589 continue
590 }
591
592 err = s.decoder.Decode(data, destination, txnResp.Revision)
593 if err != nil {
594 span.AddEvent("decode failed", attribute.Int("len", len(data)), attribute.String("err", err.Error()))
595 recordDecodeError(s.groupResourceString, preparedKey)
596 return err
597 }
598 span.AddEvent("decode succeeded", attribute.Int("len", len(data)))
599 return nil
600 }
601 }
602
603 func getNewItemFunc(listObj runtime.Object, v reflect.Value) func() runtime.Object {
604 // For unstructured lists with a target group/version, preserve the group/version in the instantiated list items
605 if unstructuredList, isUnstructured := listObj.(*unstructured.UnstructuredList); isUnstructured {
606 if apiVersion := unstructuredList.GetAPIVersion(); len(apiVersion) > 0 {
607 return func() runtime.Object {
608 return &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": apiVersion}}
609 }
610 }
611 }
612
613 // Otherwise just instantiate an empty item
614 elem := v.Type().Elem()
615 return func() runtime.Object {
616 return reflect.New(elem).Interface().(runtime.Object)
617 }
618 }
619
620 func (s *store) Count(key string) (int64, error) {
621 preparedKey, err := s.prepareKey(key)
622 if err != nil {
623 return 0, err
624 }
625
626 // We need to make sure the key ended with "/" so that we only get children "directories".
627 // e.g. if we have key "/a", "/a/b", "/ab", getting keys with prefix "/a" will return all three,
628 // while with prefix "/a/" will return only "/a/b" which is the correct answer.
629 if !strings.HasSuffix(preparedKey, "/") {
630 preparedKey += "/"
631 }
632
633 startTime := time.Now()
634 count, err := s.client.Kubernetes.Count(context.Background(), preparedKey, kubernetes.CountOptions{})
635 metrics.RecordEtcdRequest("listWithCount", preparedKey, err, startTime)
636 if err != nil {
637 return 0, err
638 }
639 return count, nil
640 }
641
642 // ReadinessCheck implements storage.Interface.
643 func (s *store) ReadinessCheck() error {
644 return nil
645 }
646
647 func (s *store) GetCurrentResourceVersion(ctx context.Context) (uint64, error) {
648 emptyList := s.newListFunc()
649 pred := storage.SelectionPredicate{
650 Label: labels.Everything(),
651 Field: fields.Everything(),
652 Limit: 1, // just in case we actually hit something
653 }
654
655 err := s.GetList(ctx, s.resourcePrefix, storage.ListOptions{Predicate: pred}, emptyList)
656 if err != nil {
657 return 0, err
658 }
659 emptyListAccessor, err := meta.ListAccessor(emptyList)
660 if err != nil {
661 return 0, err
662 }
663 if emptyListAccessor == nil {
664 return 0, fmt.Errorf("unable to extract a list accessor from %T", emptyList)
665 }
666
667 currentResourceVersion, err := strconv.Atoi(emptyListAccessor.GetResourceVersion())
668 if err != nil {
669 return 0, err
670 }
671
672 if currentResourceVersion == 0 {
673 return 0, fmt.Errorf("the current resource version must be greater than 0")
674 }
675 return uint64(currentResourceVersion), nil
676 }
677
678 // GetList implements storage.Interface.
679 func (s *store) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
680 keyPrefix, err := s.prepareKey(key)
681 if err != nil {
682 return err
683 }
684 ctx, span := tracing.Start(ctx, fmt.Sprintf("List(recursive=%v) etcd3", opts.Recursive),
685 attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)),
686 attribute.String("key", key),
687 attribute.String("resourceVersion", opts.ResourceVersion),
688 attribute.String("resourceVersionMatch", string(opts.ResourceVersionMatch)),
689 attribute.Int("limit", int(opts.Predicate.Limit)),
690 attribute.String("continue", opts.Predicate.Continue))
691 defer span.End(500 * time.Millisecond)
692 listPtr, err := meta.GetItemsPtr(listObj)
693 if err != nil {
694 return err
695 }
696 v, err := conversion.EnforcePtr(listPtr)
697 if err != nil || v.Kind() != reflect.Slice {
698 return fmt.Errorf("need ptr to slice: %v", err)
699 }
700
701 // For recursive lists, we need to make sure the key ended with "/" so that we only
702 // get children "directories". e.g. if we have key "/a", "/a/b", "/ab", getting keys
703 // with prefix "/a" will return all three, while with prefix "/a/" will return only
704 // "/a/b" which is the correct answer.
705 if opts.Recursive && !strings.HasSuffix(keyPrefix, "/") {
706 keyPrefix += "/"
707 }
708
709 // set the appropriate clientv3 options to filter the returned data set
710 limit := opts.Predicate.Limit
711 paging := opts.Predicate.Limit > 0
712 newItemFunc := getNewItemFunc(listObj, v)
713
714 withRev, continueKey, err := storage.ValidateListOptions(keyPrefix, s.versioner, opts)
715 if err != nil {
716 return err
717 }
718
719 // loop until we have filled the requested limit from etcd or there are no more results
720 var lastKey []byte
721 var hasMore bool
722 var getResp kubernetes.ListResponse
723 var numFetched int
724 var numEvald int
725 // Because these metrics are for understanding the costs of handling LIST requests,
726 // get them recorded even in error cases.
727 defer func() {
728 numReturn := v.Len()
729 metrics.RecordStorageListMetrics(s.groupResourceString, numFetched, numEvald, numReturn)
730 }()
731
732 metricsOp := "get"
733 if opts.Recursive {
734 metricsOp = "list"
735 }
736
737 aggregator := s.listErrAggrFactory()
738 for {
739 startTime := time.Now()
740 getResp, err = s.getList(ctx, keyPrefix, opts.Recursive, kubernetes.ListOptions{
741 Revision: withRev,
742 Limit: limit,
743 Continue: continueKey,
744 })
745 metrics.RecordEtcdRequest(metricsOp, s.groupResourceString, err, startTime)
746 if err != nil {
747 return interpretListError(err, len(opts.Predicate.Continue) > 0, continueKey, keyPrefix)
748 }
749 numFetched += len(getResp.Kvs)
750 if err = s.validateMinimumResourceVersion(opts.ResourceVersion, uint64(getResp.Revision)); err != nil {
751 return err
752 }
753 hasMore = int64(len(getResp.Kvs)) < getResp.Count
754
755 if len(getResp.Kvs) == 0 && hasMore {
756 return fmt.Errorf("no results were found, but etcd indicated there were more values remaining")
757 }
758 // indicate to the client which resource version was returned, and use the same resource version for subsequent requests.
759 if withRev == 0 {
760 withRev = getResp.Revision
761 }
762
763 // avoid small allocations for the result slice, since this can be called in many
764 // different contexts and we don't know how significantly the result will be filtered
765 if opts.Predicate.Empty() {
766 growSlice(v, len(getResp.Kvs))
767 } else {
768 growSlice(v, 2048, len(getResp.Kvs))
769 }
770
771 // take items from the response until the bucket is full, filtering as we go
772 for i, kv := range getResp.Kvs {
773 if paging && int64(v.Len()) >= opts.Predicate.Limit {
774 hasMore = true
775 break
776 }
777 lastKey = kv.Key
778
779 data, _, err := s.transformer.TransformFromStorage(ctx, kv.Value, authenticatedDataString(kv.Key))
780 if err != nil {
781 if done := aggregator.Aggregate(string(kv.Key), storage.NewInternalError(fmt.Errorf("unable to transform key %q: %w", kv.Key, err))); done {
782 return aggregator.Err()
783 }
784 continue
785 }
786
787 // Check if the request has already timed out before decode object
788 select {
789 case <-ctx.Done():
790 // parent context is canceled or timed out, no point in continuing
791 return storage.NewTimeoutError(string(kv.Key), "request did not complete within requested timeout")
792 default:
793 }
794
795 obj, err := s.decoder.DecodeListItem(ctx, data, uint64(kv.ModRevision), newItemFunc)
796 if err != nil {
797 recordDecodeError(s.groupResourceString, string(kv.Key))
798 if done := aggregator.Aggregate(string(kv.Key), err); done {
799 return aggregator.Err()
800 }
801 continue
802 }
803
804 // being unable to set the version does not prevent the object from being extracted
805 if matched, err := opts.Predicate.Matches(obj); err == nil && matched {
806 v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
807 }
808
809 numEvald++
810
811 // free kv early. Long lists can take O(seconds) to decode.
812 getResp.Kvs[i] = nil
813 }
814 continueKey = string(lastKey) + "\x00"
815
816 // no more results remain or we didn't request paging
817 if !hasMore || !paging {
818 break
819 }
820 // we're paging but we have filled our bucket
821 if int64(v.Len()) >= opts.Predicate.Limit {
822 break
823 }
824
825 if limit < maxLimit {
826 // We got incomplete result due to field/label selector dropping the object.
827 // Double page size to reduce total number of calls to etcd.
828 limit *= 2
829 if limit > maxLimit {
830 limit = maxLimit
831 }
832 }
833 }
834
835 if err := aggregator.Err(); err != nil {
836 return err
837 }
838
839 if v.IsNil() {
840 // Ensure that we never return a nil Items pointer in the result for consistency.
841 v.Set(reflect.MakeSlice(v.Type(), 0, 0))
842 }
843
844 continueValue, remainingItemCount, err := storage.PrepareContinueToken(string(lastKey), keyPrefix, withRev, getResp.Count, hasMore, opts)
845 if err != nil {
846 return err
847 }
848 return s.versioner.UpdateList(listObj, uint64(withRev), continueValue, remainingItemCount)
849 }
850
851 func (s *store) getList(ctx context.Context, keyPrefix string, recursive bool, options kubernetes.ListOptions) (kubernetes.ListResponse, error) {
852 if recursive {
853 return s.client.Kubernetes.List(ctx, keyPrefix, options)
854 }
855 getResp, err := s.client.Kubernetes.Get(ctx, keyPrefix, kubernetes.GetOptions{
856 Revision: options.Revision,
857 })
858 var resp kubernetes.ListResponse
859 if getResp.KV != nil {
860 resp.Kvs = []*mvccpb.KeyValue{getResp.KV}
861 resp.Count = 1
862 resp.Revision = getResp.Revision
863 } else {
864 resp.Kvs = []*mvccpb.KeyValue{}
865 resp.Count = 0
866 resp.Revision = getResp.Revision
867 }
868 return resp, err
869 }
870
871 // growSlice takes a slice value and grows its capacity up
872 // to the maximum of the passed sizes or maxCapacity, whichever
873 // is smaller. Above maxCapacity decisions about allocation are left
874 // to the Go runtime on append. This allows a caller to make an
875 // educated guess about the potential size of the total list while
876 // still avoiding overly aggressive initial allocation. If sizes
877 // is empty maxCapacity will be used as the size to grow.
878 func growSlice(v reflect.Value, maxCapacity int, sizes ...int) {
879 cap := v.Cap()
880 max := cap
881 for _, size := range sizes {
882 if size > max {
883 max = size
884 }
885 }
886 if len(sizes) == 0 || max > maxCapacity {
887 max = maxCapacity
888 }
889 if max <= cap {
890 return
891 }
892 if v.Len() > 0 {
893 extra := reflect.MakeSlice(v.Type(), v.Len(), max)
894 reflect.Copy(extra, v)
895 v.Set(extra)
896 } else {
897 extra := reflect.MakeSlice(v.Type(), 0, max)
898 v.Set(extra)
899 }
900 }
901
902 // Watch implements storage.Interface.Watch.
903 func (s *store) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
904 preparedKey, err := s.prepareKey(key)
905 if err != nil {
906 return nil, err
907 }
908 rev, err := s.versioner.ParseResourceVersion(opts.ResourceVersion)
909 if err != nil {
910 return nil, err
911 }
912 return s.watcher.Watch(s.watchContext(ctx), preparedKey, int64(rev), opts)
913 }
914
915 func (s *store) watchContext(ctx context.Context) context.Context {
916 // The etcd server waits until it cannot find a leader for 3 election
917 // timeouts to cancel existing streams. 3 is currently a hard coded
918 // constant. The election timeout defaults to 1000ms. If the cluster is
919 // healthy, when the leader is stopped, the leadership transfer should be
920 // smooth. (leader transfers its leadership before stopping). If leader is
921 // hard killed, other servers will take an election timeout to realize
922 // leader lost and start campaign.
923 return clientv3.WithRequireLeader(ctx)
924 }
925
926 func (s *store) getCurrentState(ctx context.Context, key string, v reflect.Value, ignoreNotFound bool, skipTransformDecode bool) func() (*objState, error) {
927 return func() (*objState, error) {
928 startTime := time.Now()
929 getResp, err := s.client.Kubernetes.Get(ctx, key, kubernetes.GetOptions{})
930 metrics.RecordEtcdRequest("get", s.groupResourceString, err, startTime)
931 if err != nil {
932 return nil, err
933 }
934 return s.getState(ctx, getResp.KV, key, v, ignoreNotFound, skipTransformDecode)
935 }
936 }
937
938 // getState constructs a new objState from the given response from the storage.
939 // skipTransformDecode: if true, the function will neither transform the data
940 // from the storage nor decode it into an object; otherwise, data from the
941 // storage will be transformed and decoded.
942 // NOTE: when skipTransformDecode is true, the 'data', and the 'obj' fields
943 // of the objState will be nil, and 'stale' will be set to true.
944 func (s *store) getState(ctx context.Context, kv *mvccpb.KeyValue, key string, v reflect.Value, ignoreNotFound bool, skipTransformDecode bool) (*objState, error) {
945 state := &objState{
946 meta: &storage.ResponseMeta{},
947 }
948
949 if u, ok := v.Addr().Interface().(runtime.Unstructured); ok {
950 state.obj = u.NewEmptyInstance()
951 } else {
952 state.obj = reflect.New(v.Type()).Interface().(runtime.Object)
953 }
954
955 if kv == nil {
956 if !ignoreNotFound {
957 return nil, storage.NewKeyNotFoundError(key, 0)
958 }
959 if err := runtime.SetZeroValue(state.obj); err != nil {
960 return nil, err
961 }
962 } else {
963 state.rev = kv.ModRevision
964 state.meta.ResourceVersion = uint64(state.rev)
965
966 if skipTransformDecode {
967 // be explicit that we don't have the object
968 state.obj = nil
969 state.stale = true // this seems a more sane value here
970 return state, nil
971 }
972
973 data, stale, err := s.transformer.TransformFromStorage(ctx, kv.Value, authenticatedDataString(key))
974 if err != nil {
975 return nil, storage.NewInternalError(err)
976 }
977
978 state.data = data
979 state.stale = stale
980
981 if err := s.decoder.Decode(state.data, state.obj, state.rev); err != nil {
982 recordDecodeError(s.groupResourceString, key)
983 return nil, err
984 }
985 }
986 return state, nil
987 }
988
989 func (s *store) getStateFromObject(obj runtime.Object) (*objState, error) {
990 state := &objState{
991 obj: obj,
992 meta: &storage.ResponseMeta{},
993 }
994
995 rv, err := s.versioner.ObjectResourceVersion(obj)
996 if err != nil {
997 return nil, fmt.Errorf("couldn't get resource version: %v", err)
998 }
999 state.rev = int64(rv)
1000 state.meta.ResourceVersion = uint64(state.rev)
1001
1002 // Compute the serialized form - for that we need to temporarily clean
1003 // its resource version field (those are not stored in etcd).
1004 if err := s.versioner.PrepareObjectForStorage(obj); err != nil {
1005 return nil, fmt.Errorf("PrepareObjectForStorage failed: %v", err)
1006 }
1007 state.data, err = runtime.Encode(s.codec, obj)
1008 if err != nil {
1009 return nil, err
1010 }
1011 if err := s.versioner.UpdateObject(state.obj, uint64(rv)); err != nil {
1012 klog.Errorf("failed to update object version: %v", err)
1013 }
1014 return state, nil
1015 }
1016
1017 func (s *store) updateState(st *objState, userUpdate storage.UpdateFunc) (runtime.Object, uint64, error) {
1018 ret, ttlPtr, err := userUpdate(st.obj, *st.meta)
1019 if err != nil {
1020 return nil, 0, err
1021 }
1022
1023 if err := s.versioner.PrepareObjectForStorage(ret); err != nil {
1024 return nil, 0, fmt.Errorf("PrepareObjectForStorage failed: %v", err)
1025 }
1026 var ttl uint64
1027 if ttlPtr != nil {
1028 ttl = *ttlPtr
1029 }
1030 return ret, ttl, nil
1031 }
1032
1033 // validateMinimumResourceVersion returns a 'too large resource' version error when the provided minimumResourceVersion is
1034 // greater than the most recent actualRevision available from storage.
1035 func (s *store) validateMinimumResourceVersion(minimumResourceVersion string, actualRevision uint64) error {
1036 if minimumResourceVersion == "" {
1037 return nil
1038 }
1039 minimumRV, err := s.versioner.ParseResourceVersion(minimumResourceVersion)
1040 if err != nil {
1041 return apierrors.NewBadRequest(fmt.Sprintf("invalid resource version: %v", err))
1042 }
1043 // Enforce the storage.Interface guarantee that the resource version of the returned data
1044 // "will be at least 'resourceVersion'".
1045 if minimumRV > actualRevision {
1046 return storage.NewTooLargeResourceVersionError(minimumRV, actualRevision, 0)
1047 }
1048 return nil
1049 }
1050
1051 func (s *store) prepareKey(key string) (string, error) {
1052 if key == ".." ||
1053 strings.HasPrefix(key, "../") ||
1054 strings.HasSuffix(key, "/..") ||
1055 strings.Contains(key, "/../") {
1056 return "", fmt.Errorf("invalid key: %q", key)
1057 }
1058 if key == "." ||
1059 strings.HasPrefix(key, "./") ||
1060 strings.HasSuffix(key, "/.") ||
1061 strings.Contains(key, "/./") {
1062 return "", fmt.Errorf("invalid key: %q", key)
1063 }
1064 if key == "" || key == "/" {
1065 return "", fmt.Errorf("empty key: %q", key)
1066 }
1067 // We ensured that pathPrefix ends in '/' in construction, so skip any leading '/' in the key now.
1068 startIndex := 0
1069 if key[0] == '/' {
1070 startIndex = 1
1071 }
1072 return s.pathPrefix + key[startIndex:], nil
1073 }
1074
1075 // recordDecodeError record decode error split by object type.
1076 func recordDecodeError(resource string, key string) {
1077 metrics.RecordDecodeError(resource)
1078 klog.V(4).Infof("Decoding %s \"%s\" failed", resource, key)
1079 }
1080
1081 // getTypeName returns type name of an object for reporting purposes.
1082 func getTypeName(obj interface{}) string {
1083 return reflect.TypeOf(obj).String()
1084 }