Debian Codegraph @

pkg/storage/etcd3/watcher.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package etcd3
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "os"
24 "strconv"
25 "strings"
26 "sync"
27 "time"
28
29 clientv3 "go.etcd.io/etcd/client/v3"
30 grpccodes "google.golang.org/grpc/codes"
31 grpcstatus "google.golang.org/grpc/status"
32
33 apierrors "k8s.io/apimachinery/pkg/api/errors"
34 "k8s.io/apimachinery/pkg/runtime"
35 "k8s.io/apimachinery/pkg/runtime/schema"
36 "k8s.io/apimachinery/pkg/util/wait"
37 "k8s.io/apimachinery/pkg/watch"
38 "k8s.io/apiserver/pkg/features"
39 "k8s.io/apiserver/pkg/storage"
40 "k8s.io/apiserver/pkg/storage/etcd3/metrics"
41 "k8s.io/apiserver/pkg/storage/value"
42 utilfeature "k8s.io/apiserver/pkg/util/feature"
43 utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
44 "k8s.io/klog/v2"
45 )
46
47 const (
48 // We have set a buffer in order to reduce times of context switches.
49 incomingBufSize = 100
50 outgoingBufSize = 100
51 processEventConcurrency = 10
52 )
53
54 // defaultWatcherMaxLimit is used to facilitate construction tests
55 var defaultWatcherMaxLimit int64 = maxLimit
56
57 // fatalOnDecodeError is used during testing to panic the server if watcher encounters a decoding error
58 var fatalOnDecodeError = false
59
60 func init() {
61 // check to see if we are running in a test environment
62 TestOnlySetFatalOnDecodeError(true)
63 fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR"))
64 }
65
66 // TestOnlySetFatalOnDecodeError should only be used for cases where decode errors are expected and need to be tested. e.g. conversion webhooks.
67 func TestOnlySetFatalOnDecodeError(b bool) {
68 fatalOnDecodeError = b
69 }
70
71 type watcher struct {
72 client *clientv3.Client
73 codec runtime.Codec
74 newFunc func() runtime.Object
75 objectType string
76 groupResource schema.GroupResource
77 versioner storage.Versioner
78 transformer value.Transformer
79 getCurrentStorageRV func(context.Context) (uint64, error)
80 }
81
82 // watchChan implements watch.Interface.
83 type watchChan struct {
84 watcher *watcher
85 key string
86 initialRev int64
87 recursive bool
88 progressNotify bool
89 internalPred storage.SelectionPredicate
90 ctx context.Context
91 cancel context.CancelFunc
92 incomingEventChan chan *event
93 resultChan chan watch.Event
94 errChan chan error
95 }
96
97 // Watch watches on a key and returns a watch.Interface that transfers relevant notifications.
98 // If rev is zero, it will return the existing object(s) and then start watching from
99 // the maximum revision+1 from returned objects.
100 // If rev is non-zero, it will watch events happened after given revision.
101 // If opts.Recursive is false, it watches on given key.
102 // If opts.Recursive is true, it watches any children and directories under the key, excluding the root key itself.
103 // pred must be non-nil. Only if opts.Predicate matches the change, it will be returned.
104 func (w *watcher) Watch(ctx context.Context, key string, rev int64, opts storage.ListOptions) (watch.Interface, error) {
105 if opts.Recursive && !strings.HasSuffix(key, "/") {
106 key += "/"
107 }
108 if opts.ProgressNotify && w.newFunc == nil {
109 return nil, apierrors.NewInternalError(errors.New("progressNotify for watch is unsupported by the etcd storage because no newFunc was provided"))
110 }
111 startWatchRV, err := w.getStartWatchResourceVersion(ctx, rev, opts)
112 if err != nil {
113 return nil, err
114 }
115 wc := w.createWatchChan(ctx, key, startWatchRV, opts.Recursive, opts.ProgressNotify, opts.Predicate)
116 go wc.run(isInitialEventsEndBookmarkRequired(opts), areInitialEventsRequired(rev, opts))
117
118 // For etcd watch we don't have an easy way to answer whether the watch
119 // has already caught up. So in the initial version (given that watchcache
120 // is by default enabled for all resources but Events), we just deliver
121 // the initialization signal immediately. Improving this will be explored
122 // in the future.
123 utilflowcontrol.WatchInitialized(ctx)
124
125 return wc, nil
126 }
127
128 func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive, progressNotify bool, pred storage.SelectionPredicate) *watchChan {
129 wc := &watchChan{
130 watcher: w,
131 key: key,
132 initialRev: rev,
133 recursive: recursive,
134 progressNotify: progressNotify,
135 internalPred: pred,
136 incomingEventChan: make(chan *event, incomingBufSize),
137 resultChan: make(chan watch.Event, outgoingBufSize),
138 errChan: make(chan error, 1),
139 }
140 if pred.Empty() {
141 // The filter doesn't filter out any object.
142 wc.internalPred = storage.Everything
143 }
144 wc.ctx, wc.cancel = context.WithCancel(ctx)
145 return wc
146 }
147
148 // getStartWatchResourceVersion returns a ResourceVersion
149 // the watch will be started from.
150 // Depending on the input parameters the semantics of the returned ResourceVersion are:
151 // - start at Exact (return resourceVersion)
152 // - start at Most Recent (return an RV from etcd)
153 func (w *watcher) getStartWatchResourceVersion(ctx context.Context, resourceVersion int64, opts storage.ListOptions) (int64, error) {
154 if resourceVersion > 0 {
155 return resourceVersion, nil
156 }
157 if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
158 return 0, nil
159 }
160 if opts.SendInitialEvents == nil || *opts.SendInitialEvents {
161 // note that when opts.SendInitialEvents=true
162 // we will be issuing a consistent LIST request
163 // against etcd followed by the special bookmark event
164 return 0, nil
165 }
166 // at this point the clients is interested
167 // only in getting a stream of events
168 // starting at the MostRecent point in time (RV)
169 currentStorageRV, err := w.getCurrentStorageRV(ctx)
170 if err != nil {
171 return 0, err
172 }
173 // currentStorageRV is taken from resp.Header.Revision (int64)
174 // and cast to uint64, so it is safe to do reverse
175 // at some point we should unify the interface but that
176 // would require changing Versioner.UpdateList
177 return int64(currentStorageRV), nil
178 }
179
180 // isInitialEventsEndBookmarkRequired since there is no way to directly set
181 // opts.ProgressNotify from the API and the etcd3 impl doesn't support
182 // notification for external clients we simply return initialEventsEndBookmarkRequired
183 // to only send the bookmark event after the initial list call.
184 //
185 // see: https://github.com/kubernetes/kubernetes/issues/120348
186 func isInitialEventsEndBookmarkRequired(opts storage.ListOptions) bool {
187 if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
188 return false
189 }
190 return opts.SendInitialEvents != nil && *opts.SendInitialEvents && opts.Predicate.AllowWatchBookmarks
191 }
192
193 // areInitialEventsRequired returns true if all events from the etcd should be returned.
194 func areInitialEventsRequired(resourceVersion int64, opts storage.ListOptions) bool {
195 if opts.SendInitialEvents == nil && resourceVersion == 0 {
196 return true // legacy case
197 }
198 if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
199 return false
200 }
201 return opts.SendInitialEvents != nil && *opts.SendInitialEvents
202 }
203
204 type etcdError interface {
205 Code() grpccodes.Code
206 Error() string
207 }
208
209 type grpcError interface {
210 GRPCStatus() *grpcstatus.Status
211 }
212
213 func isCancelError(err error) bool {
214 if err == nil {
215 return false
216 }
217 if err == context.Canceled {
218 return true
219 }
220 if etcdErr, ok := err.(etcdError); ok && etcdErr.Code() == grpccodes.Canceled {
221 return true
222 }
223 if grpcErr, ok := err.(grpcError); ok && grpcErr.GRPCStatus().Code() == grpccodes.Canceled {
224 return true
225 }
226 return false
227 }
228
229 func (wc *watchChan) run(initialEventsEndBookmarkRequired, forceInitialEvents bool) {
230 watchClosedCh := make(chan struct{})
231 go wc.startWatching(watchClosedCh, initialEventsEndBookmarkRequired, forceInitialEvents)
232
233 var resultChanWG sync.WaitGroup
234 wc.processEvents(&resultChanWG)
235
236 select {
237 case err := <-wc.errChan:
238 if isCancelError(err) {
239 break
240 }
241 errResult := transformErrorToEvent(err)
242 if errResult != nil {
243 // error result is guaranteed to be received by user before closing ResultChan.
244 select {
245 case wc.resultChan <- *errResult:
246 case <-wc.ctx.Done(): // user has given up all results
247 }
248 }
249 case <-watchClosedCh:
250 case <-wc.ctx.Done(): // user cancel
251 }
252
253 // We use wc.ctx to reap all goroutines. Under whatever condition, we should stop them all.
254 // It's fine to double cancel.
255 wc.cancel()
256
257 // we need to wait until resultChan wouldn't be used anymore
258 resultChanWG.Wait()
259 close(wc.resultChan)
260 }
261
262 func (wc *watchChan) Stop() {
263 wc.cancel()
264 }
265
266 func (wc *watchChan) ResultChan() <-chan watch.Event {
267 return wc.resultChan
268 }
269
270 func (wc *watchChan) RequestWatchProgress() error {
271 return wc.watcher.client.RequestProgress(wc.ctx)
272 }
273
274 // sync tries to retrieve existing data and send them to process.
275 // The revision to watch will be set to the revision in response.
276 // All events sent will have isCreated=true
277 func (wc *watchChan) sync() error {
278 opts := []clientv3.OpOption{}
279 if wc.recursive {
280 opts = append(opts, clientv3.WithLimit(defaultWatcherMaxLimit))
281 rangeEnd := clientv3.GetPrefixRangeEnd(wc.key)
282 opts = append(opts, clientv3.WithRange(rangeEnd))
283 }
284
285 var err error
286 var lastKey []byte
287 var withRev int64
288 var getResp *clientv3.GetResponse
289
290 metricsOp := "get"
291 if wc.recursive {
292 metricsOp = "list"
293 }
294
295 preparedKey := wc.key
296
297 for {
298 startTime := time.Now()
299 getResp, err = wc.watcher.client.KV.Get(wc.ctx, preparedKey, opts...)
300 metrics.RecordEtcdRequest(metricsOp, wc.watcher.groupResource.String(), err, startTime)
301 if err != nil {
302 return interpretListError(err, true, preparedKey, wc.key)
303 }
304
305 if len(getResp.Kvs) == 0 && getResp.More {
306 return fmt.Errorf("no results were found, but etcd indicated there were more values remaining")
307 }
308
309 // send items from the response until no more results
310 for i, kv := range getResp.Kvs {
311 lastKey = kv.Key
312 wc.sendEvent(parseKV(kv))
313 // free kv early. Long lists can take O(seconds) to decode.
314 getResp.Kvs[i] = nil
315 }
316
317 if withRev == 0 {
318 wc.initialRev = getResp.Header.Revision
319 }
320
321 // no more results remain
322 if !getResp.More {
323 return nil
324 }
325
326 preparedKey = string(lastKey) + "\x00"
327 if withRev == 0 {
328 withRev = getResp.Header.Revision
329 opts = append(opts, clientv3.WithRev(withRev))
330 }
331 }
332 }
333
334 func logWatchChannelErr(err error) {
335 switch {
336 case strings.Contains(err.Error(), "mvcc: required revision has been compacted"):
337 // mvcc revision compaction which is regarded as warning, not error
338 klog.Warningf("watch chan error: %v", err)
339 case isCancelError(err):
340 // expected when watches close, no need to log
341 default:
342 klog.Errorf("watch chan error: %v", err)
343 }
344 }
345
346 // startWatching does:
347 // - get current objects if initialRev=0; set initialRev to current rev
348 // - watch on given key and send events to process.
349 //
350 // initialEventsEndBookmarkSent helps us keep track
351 // of whether we have sent an annotated bookmark event.
352 //
353 // it's important to note that we don't
354 // need to track the actual RV because
355 // we only send the bookmark event
356 // after the initial list call.
357 //
358 // when this variable is set to false,
359 // it means we don't have any specific
360 // preferences for delivering bookmark events.
361 func (wc *watchChan) startWatching(watchClosedCh chan struct{}, initialEventsEndBookmarkRequired, forceInitialEvents bool) {
362 if wc.initialRev > 0 && forceInitialEvents {
363 currentStorageRV, err := wc.watcher.getCurrentStorageRV(wc.ctx)
364 if err != nil {
365 wc.sendError(err)
366 return
367 }
368 if uint64(wc.initialRev) > currentStorageRV {
369 wc.sendError(storage.NewTooLargeResourceVersionError(uint64(wc.initialRev), currentStorageRV, int(wait.Jitter(1*time.Second, 3).Seconds())))
370 return
371 }
372 }
373 if forceInitialEvents {
374 if err := wc.sync(); err != nil {
375 klog.Errorf("failed to sync with latest state: %v", err)
376 wc.sendError(err)
377 return
378 }
379 }
380 if initialEventsEndBookmarkRequired {
381 wc.sendEvent(func() *event {
382 e := progressNotifyEvent(wc.initialRev)
383 e.isInitialEventsEndBookmark = true
384 return e
385 }())
386 }
387 opts := []clientv3.OpOption{clientv3.WithRev(wc.initialRev + 1), clientv3.WithPrevKV()}
388 if wc.recursive {
389 opts = append(opts, clientv3.WithPrefix())
390 }
391 if wc.progressNotify {
392 opts = append(opts, clientv3.WithProgressNotify())
393 }
394 wch := wc.watcher.client.Watch(wc.ctx, wc.key, opts...)
395 for wres := range wch {
396 if wres.Err() != nil {
397 err := wres.Err()
398 // If there is an error on server (e.g. compaction), the channel will return it before closed.
399 logWatchChannelErr(err)
400 wc.sendError(err)
401 return
402 }
403 if wres.IsProgressNotify() {
404 wc.sendEvent(progressNotifyEvent(wres.Header.GetRevision()))
405 metrics.RecordEtcdBookmark(wc.watcher.groupResource.String())
406 continue
407 }
408
409 for _, e := range wres.Events {
410 metrics.RecordEtcdEvent(wc.watcher.groupResource.String())
411 parsedEvent, err := parseEvent(e)
412 if err != nil {
413 logWatchChannelErr(err)
414 wc.sendError(err)
415 return
416 }
417 wc.sendEvent(parsedEvent)
418 }
419 }
420 // When we come to this point, it's only possible that client side ends the watch.
421 // e.g. cancel the context, close the client.
422 // If this watch chan is broken and context isn't cancelled, other goroutines will still hang.
423 // We should notify the main thread that this goroutine has exited.
424 close(watchClosedCh)
425 }
426
427 // processEvents processes events from etcd watcher and sends results to resultChan.
428 func (wc *watchChan) processEvents(wg *sync.WaitGroup) {
429 if utilfeature.DefaultFeatureGate.Enabled(features.ConcurrentWatchObjectDecode) {
430 wc.concurrentProcessEvents(wg)
431 } else {
432 wg.Add(1)
433 go wc.serialProcessEvents(wg)
434 }
435 }
436 func (wc *watchChan) serialProcessEvents(wg *sync.WaitGroup) {
437 defer wg.Done()
438 for {
439 select {
440 case e := <-wc.incomingEventChan:
441 res, err := wc.transform(e)
442 if err != nil {
443 wc.sendError(err)
444 return
445 }
446
447 if res == nil {
448 continue
449 }
450 if len(wc.resultChan) == cap(wc.resultChan) {
451 klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow dispatching events to watchers", "outgoingEvents", outgoingBufSize, "objectType", wc.watcher.objectType, "groupResource", wc.watcher.groupResource)
452 }
453 // If user couldn't receive results fast enough, we also block incoming events from watcher.
454 // Because storing events in local will cause more memory usage.
455 // The worst case would be closing the fast watcher.
456 select {
457 case wc.resultChan <- *res:
458 case <-wc.ctx.Done():
459 return
460 }
461 case <-wc.ctx.Done():
462 return
463 }
464 }
465 }
466
467 func (wc *watchChan) concurrentProcessEvents(wg *sync.WaitGroup) {
468 p := concurrentOrderedEventProcessing{
469 wc: wc,
470 processingQueue: make(chan chan *processingResult, processEventConcurrency-1),
471
472 objectType: wc.watcher.objectType,
473 groupResource: wc.watcher.groupResource,
474 }
475 wg.Add(1)
476 go func() {
477 defer wg.Done()
478 p.scheduleEventProcessing(wc.ctx, wg)
479 }()
480 wg.Add(1)
481 go func() {
482 defer wg.Done()
483 p.collectEventProcessing(wc.ctx)
484 }()
485 }
486
487 type processingResult struct {
488 event *watch.Event
489 err error
490 }
491
492 type concurrentOrderedEventProcessing struct {
493 wc *watchChan
494
495 processingQueue chan chan *processingResult
496 // Metadata for logging
497 objectType string
498 groupResource schema.GroupResource
499 }
500
501 func (p *concurrentOrderedEventProcessing) scheduleEventProcessing(ctx context.Context, wg *sync.WaitGroup) {
502 var e *event
503 for {
504 select {
505 case <-ctx.Done():
506 return
507 case e = <-p.wc.incomingEventChan:
508 }
509 processingResponse := make(chan *processingResult, 1)
510 select {
511 case <-ctx.Done():
512 return
513 case p.processingQueue <- processingResponse:
514 }
515 wg.Add(1)
516 go func(e *event, response chan<- *processingResult) {
517 defer wg.Done()
518 responseEvent, err := p.wc.transform(e)
519 select {
520 case <-ctx.Done():
521 case response <- &processingResult{event: responseEvent, err: err}:
522 }
523 }(e, processingResponse)
524 }
525 }
526
527 func (p *concurrentOrderedEventProcessing) collectEventProcessing(ctx context.Context) {
528 var processingResponse chan *processingResult
529 var r *processingResult
530 for {
531 select {
532 case <-ctx.Done():
533 return
534 case processingResponse = <-p.processingQueue:
535 }
536 select {
537 case <-ctx.Done():
538 return
539 case r = <-processingResponse:
540 }
541 if r.err != nil {
542 p.wc.sendError(r.err)
543 return
544 }
545 if r.event == nil {
546 continue
547 }
548 if len(p.wc.resultChan) == cap(p.wc.resultChan) {
549 klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow dispatching events to watchers", "outgoingEvents", outgoingBufSize, "objectType", p.wc.watcher.objectType, "groupResource", p.wc.watcher.groupResource)
550 }
551 // If user couldn't receive results fast enough, we also block incoming events from watcher.
552 // Because storing events in local will cause more memory usage.
553 // The worst case would be closing the fast watcher.
554 select {
555 case p.wc.resultChan <- *r.event:
556 case <-p.wc.ctx.Done():
557 return
558 }
559 }
560 }
561
562 func (wc *watchChan) filter(obj runtime.Object) bool {
563 if wc.internalPred.Empty() {
564 return true
565 }
566 matched, err := wc.internalPred.Matches(obj)
567 return err == nil && matched
568 }
569
570 func (wc *watchChan) acceptAll() bool {
571 return wc.internalPred.Empty()
572 }
573
574 // transform transforms an event into a result for user if not filtered.
575 func (wc *watchChan) transform(e *event) (res *watch.Event, err error) {
576 curObj, oldObj, err := wc.prepareObjs(e)
577 if err != nil {
578 klog.Errorf("failed to prepare current and previous objects: %v", err)
579 return nil, err
580 }
581
582 switch {
583 case e.isProgressNotify:
584 object := wc.watcher.newFunc()
585 if err := wc.watcher.versioner.UpdateObject(object, uint64(e.rev)); err != nil {
586 klog.Errorf("failed to propagate object version: %v", err)
587 return nil, fmt.Errorf("failed to propagate object resource version: %w", err)
588 }
589 if e.isInitialEventsEndBookmark {
590 if err := storage.AnnotateInitialEventsEndBookmark(object); err != nil {
591 return nil, fmt.Errorf("error while accessing object's metadata gr: %v, type: %v, obj: %#v, err: %w", wc.watcher.groupResource, wc.watcher.objectType, object, err)
592 }
593 }
594 res = &watch.Event{
595 Type: watch.Bookmark,
596 Object: object,
597 }
598 case e.isDeleted:
599 if !wc.filter(oldObj) {
600 return nil, nil
601 }
602 res = &watch.Event{
603 Type: watch.Deleted,
604 Object: oldObj,
605 }
606 case e.isCreated:
607 if !wc.filter(curObj) {
608 return nil, nil
609 }
610 res = &watch.Event{
611 Type: watch.Added,
612 Object: curObj,
613 }
614 default:
615 if wc.acceptAll() {
616 res = &watch.Event{
617 Type: watch.Modified,
618 Object: curObj,
619 }
620 return res, nil
621 }
622 curObjPasses := wc.filter(curObj)
623 oldObjPasses := wc.filter(oldObj)
624 switch {
625 case curObjPasses && oldObjPasses:
626 res = &watch.Event{
627 Type: watch.Modified,
628 Object: curObj,
629 }
630 case curObjPasses && !oldObjPasses:
631 res = &watch.Event{
632 Type: watch.Added,
633 Object: curObj,
634 }
635 case !curObjPasses && oldObjPasses:
636 res = &watch.Event{
637 Type: watch.Deleted,
638 Object: oldObj,
639 }
640 }
641 }
642 return res, nil
643 }
644
645 func transformErrorToEvent(err error) *watch.Event {
646 err = interpretWatchError(err)
647 if _, ok := err.(apierrors.APIStatus); !ok {
648 err = apierrors.NewInternalError(err)
649 }
650 status := err.(apierrors.APIStatus).Status()
651 return &watch.Event{
652 Type: watch.Error,
653 Object: &status,
654 }
655 }
656
657 func (wc *watchChan) sendError(err error) {
658 select {
659 case wc.errChan <- err:
660 case <-wc.ctx.Done():
661 }
662 }
663
664 func (wc *watchChan) sendEvent(e *event) {
665 if len(wc.incomingEventChan) == incomingBufSize {
666 klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow decoding, user not receiving fast, or other processing logic", "incomingEvents", incomingBufSize, "objectType", wc.watcher.objectType, "groupResource", wc.watcher.groupResource)
667 }
668 select {
669 case wc.incomingEventChan <- e:
670 case <-wc.ctx.Done():
671 }
672 }
673
674 func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtime.Object, err error) {
675 if e.isProgressNotify {
676 // progressNotify events doesn't contain neither current nor previous object version,
677 return nil, nil, nil
678 }
679
680 if !e.isDeleted {
681 data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.value, authenticatedDataString(e.key))
682 if err != nil {
683 return nil, nil, err
684 }
685 curObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
686 if err != nil {
687 return nil, nil, err
688 }
689 }
690 // We need to decode prevValue, only if this is deletion event or
691 // the underlying filter doesn't accept all objects (otherwise we
692 // know that the filter for previous object will return true and
693 // we need the object only to compute whether it was filtered out
694 // before).
695 if len(e.prevValue) > 0 && (e.isDeleted || !wc.acceptAll()) {
696 data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.prevValue, authenticatedDataString(e.key))
697 if err != nil {
698 return nil, nil, wc.watcher.transformIfCorruptObjectError(e, err)
699 }
700 // Note that this sends the *old* object with the etcd revision for the time at
701 // which it gets deleted.
702 oldObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev)
703 if err != nil {
704 return nil, nil, wc.watcher.transformIfCorruptObjectError(e, err)
705 }
706 }
707 return curObj, oldObj, nil
708 }
709
710 type corruptObjectDeletedError struct {
711 err error
712 }
713
714 func (e *corruptObjectDeletedError) Error() string {
715 return fmt.Sprintf("saw a DELETED event, but object data is corrupt - %v", e.err)
716 }
717 func (e *corruptObjectDeletedError) Unwrap() error { return e.err }
718
719 func (w *watcher) transformIfCorruptObjectError(e *event, err error) error {
720 var corruptObjErr *corruptObjectError
721 if !e.isDeleted || !errors.As(err, &corruptObjErr) {
722 return err
723 }
724
725 // if we are here it means we received a DELETED event but the object
726 // associated with it is corrupt because we failed to transform or
727 // decode the data associated with the object.
728 // wrap the original error so we can send a proper watch Error event.
729 return &corruptObjectDeletedError{err: corruptObjErr}
730 }
731
732 func decodeObj(codec runtime.Codec, versioner storage.Versioner, data []byte, rev int64) (_ runtime.Object, err error) {
733 obj, err := runtime.Decode(codec, []byte(data))
734 if err != nil {
735 if fatalOnDecodeError {
736 // we are running in a test environment and thus an
737 // error here is due to a coder mistake if the defer
738 // does not catch it
739 panic(err)
740 }
741 return nil, err
742 }
743 // ensure resource version is set on the object we load from etcd
744 if err := versioner.UpdateObject(obj, uint64(rev)); err != nil {
745 return nil, fmt.Errorf("failure to version api object (%d) %#v: %v", rev, obj, err)
746 }
747 return obj, nil
748 }