Debian Codegraph @

pkg/storage/cacher/cacher.go

raw
1 /*
2 Copyright 2015 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package cacher
18
19 import (
20 "context"
21 "fmt"
22 "net/http"
23 "reflect"
24 "strings"
25 "sync"
26 "time"
27
28 "go.opentelemetry.io/otel/attribute"
29 "google.golang.org/grpc/metadata"
30
31 "k8s.io/apimachinery/pkg/api/errors"
32 "k8s.io/apimachinery/pkg/api/meta"
33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34 "k8s.io/apimachinery/pkg/conversion"
35 "k8s.io/apimachinery/pkg/fields"
36 "k8s.io/apimachinery/pkg/labels"
37 "k8s.io/apimachinery/pkg/runtime"
38 "k8s.io/apimachinery/pkg/runtime/schema"
39 "k8s.io/apimachinery/pkg/util/wait"
40 "k8s.io/apimachinery/pkg/watch"
41 "k8s.io/apiserver/pkg/audit"
42 "k8s.io/apiserver/pkg/endpoints/request"
43 "k8s.io/apiserver/pkg/features"
44 "k8s.io/apiserver/pkg/storage"
45 "k8s.io/apiserver/pkg/storage/cacher/delegator"
46 "k8s.io/apiserver/pkg/storage/cacher/metrics"
47 "k8s.io/apiserver/pkg/storage/cacher/progress"
48 etcdfeature "k8s.io/apiserver/pkg/storage/feature"
49 utilfeature "k8s.io/apiserver/pkg/util/feature"
50 "k8s.io/client-go/tools/cache"
51 "k8s.io/component-base/tracing"
52 "k8s.io/klog/v2"
53 "k8s.io/utils/clock"
54 "k8s.io/utils/ptr"
55 )
56
57 var (
58 emptyFunc = func(bool) {}
59 coreNamespaceResource = schema.GroupResource{Group: "", Resource: "namespaces"}
60 )
61
62 const (
63 // storageWatchListPageSize is the cacher's request chunk size of
64 // initial and resync watch lists to storage.
65 storageWatchListPageSize = int64(10000)
66
67 // DefaultEventFreshDuration is the default time duration of events
68 // we want to keep.
69 // We set it to defaultBookmarkFrequency plus epsilon to maximize
70 // chances that last bookmark was sent within kept history, at the
71 // same time, minimizing the needed memory usage.
72 DefaultEventFreshDuration = defaultBookmarkFrequency + 15*time.Second
73
74 // defaultBookmarkFrequency defines how frequently watch bookmarks should be send
75 // in addition to sending a bookmark right before watch deadline.
76 defaultBookmarkFrequency = time.Minute
77 )
78
79 // Config contains the configuration for a given Cache.
80 type Config struct {
81 // An underlying storage.Interface.
82 Storage storage.Interface
83
84 // An underlying storage.Versioner.
85 Versioner storage.Versioner
86
87 // The GroupResource the cacher is caching. Used for disambiguating *unstructured.Unstructured (CRDs) in logging
88 // and metrics.
89 GroupResource schema.GroupResource
90
91 // EventsHistoryWindow specifies minimum history duration that storage is keeping.
92 // If lower than DefaultEventFreshDuration, the cache creation will fail.
93 EventsHistoryWindow time.Duration
94
95 // The Cache will be caching objects of a given Type and assumes that they
96 // are all stored under ResourcePrefix directory in the underlying database.
97 ResourcePrefix string
98
99 // KeyFunc is used to get a key in the underlying storage for a given object.
100 KeyFunc func(runtime.Object) (string, error)
101
102 // GetAttrsFunc is used to get object labels, fields
103 GetAttrsFunc func(runtime.Object) (label labels.Set, field fields.Set, err error)
104
105 // IndexerFuncs is used for optimizing amount of watchers that
106 // needs to process an incoming event.
107 IndexerFuncs storage.IndexerFuncs
108
109 // Indexers is used to accelerate the list operation, falls back to regular list
110 // operation if no indexer found.
111 Indexers *cache.Indexers
112
113 // NewFunc is a function that creates new empty object storing a object of type Type.
114 NewFunc func() runtime.Object
115
116 // NewList is a function that creates new empty object storing a list of
117 // objects of type Type.
118 NewListFunc func() runtime.Object
119
120 Codec runtime.Codec
121
122 Clock clock.WithTicker
123 }
124
125 type watchersMap map[int]*cacheWatcher
126
127 func (wm watchersMap) addWatcher(w *cacheWatcher, number int) {
128 wm[number] = w
129 }
130
131 func (wm watchersMap) deleteWatcher(number int) {
132 delete(wm, number)
133 }
134
135 func (wm watchersMap) terminateAll(done func(*cacheWatcher)) {
136 for key, watcher := range wm {
137 delete(wm, key)
138 done(watcher)
139 }
140 }
141
142 type indexedWatchers struct {
143 allWatchers map[namespacedName]watchersMap
144 valueWatchers map[string]watchersMap
145 }
146
147 func (i *indexedWatchers) addWatcher(w *cacheWatcher, number int, scope namespacedName, value string, supported bool) {
148 if supported {
149 if _, ok := i.valueWatchers[value]; !ok {
150 i.valueWatchers[value] = watchersMap{}
151 }
152 i.valueWatchers[value].addWatcher(w, number)
153 } else {
154 scopedWatchers, ok := i.allWatchers[scope]
155 if !ok {
156 scopedWatchers = watchersMap{}
157 i.allWatchers[scope] = scopedWatchers
158 }
159 scopedWatchers.addWatcher(w, number)
160 }
161 }
162
163 func (i *indexedWatchers) deleteWatcher(number int, scope namespacedName, value string, supported bool) {
164 if supported {
165 i.valueWatchers[value].deleteWatcher(number)
166 if len(i.valueWatchers[value]) == 0 {
167 delete(i.valueWatchers, value)
168 }
169 } else {
170 i.allWatchers[scope].deleteWatcher(number)
171 if len(i.allWatchers[scope]) == 0 {
172 delete(i.allWatchers, scope)
173 }
174 }
175 }
176
177 func (i *indexedWatchers) terminateAll(groupResource schema.GroupResource, done func(*cacheWatcher)) {
178 // note that we don't have to call setDrainInputBufferLocked method on the watchers
179 // because we take advantage of the default value - stop immediately
180 // also watchers that have had already its draining strategy set
181 // are no longer available (they were removed from the allWatchers and the valueWatchers maps)
182 if len(i.allWatchers) > 0 || len(i.valueWatchers) > 0 {
183 klog.Warningf("Terminating all watchers from cacher %v", groupResource)
184 }
185 for _, watchers := range i.allWatchers {
186 watchers.terminateAll(done)
187 }
188 for _, watchers := range i.valueWatchers {
189 watchers.terminateAll(done)
190 }
191 i.allWatchers = map[namespacedName]watchersMap{}
192 i.valueWatchers = map[string]watchersMap{}
193 }
194
195 // As we don't need a high precision here, we keep all watchers timeout within a
196 // second in a bucket, and pop up them once at the timeout. To be more specific,
197 // if you set fire time at X, you can get the bookmark within (X-1,X+1) period.
198 type watcherBookmarkTimeBuckets struct {
199 // the key of watcherBuckets is the number of seconds since createTime
200 watchersBuckets map[int64][]*cacheWatcher
201 createTime time.Time
202 startBucketID int64
203 clock clock.Clock
204 bookmarkFrequency time.Duration
205 }
206
207 func newTimeBucketWatchers(clock clock.Clock, bookmarkFrequency time.Duration) *watcherBookmarkTimeBuckets {
208 return &watcherBookmarkTimeBuckets{
209 watchersBuckets: make(map[int64][]*cacheWatcher),
210 createTime: clock.Now(),
211 startBucketID: 0,
212 clock: clock,
213 bookmarkFrequency: bookmarkFrequency,
214 }
215 }
216
217 // adds a watcher to the bucket, if the deadline is before the start, it will be
218 // added to the first one.
219 func (t *watcherBookmarkTimeBuckets) addWatcherThreadUnsafe(w *cacheWatcher) bool {
220 // note that the returned time can be before t.createTime,
221 // especially in cases when the nextBookmarkTime method
222 // give us the zero value of type Time
223 // so buckedID can hold a negative value
224 nextTime, ok := w.nextBookmarkTime(t.clock.Now(), t.bookmarkFrequency)
225 if !ok {
226 return false
227 }
228 bucketID := int64(nextTime.Sub(t.createTime) / time.Second)
229 if bucketID < t.startBucketID {
230 bucketID = t.startBucketID
231 }
232 watchers := t.watchersBuckets[bucketID]
233 t.watchersBuckets[bucketID] = append(watchers, w)
234 return true
235 }
236
237 func (t *watcherBookmarkTimeBuckets) popExpiredWatchersThreadUnsafe() [][]*cacheWatcher {
238 currentBucketID := int64(t.clock.Since(t.createTime) / time.Second)
239 // There should be one or two elements in almost all cases
240 expiredWatchers := make([][]*cacheWatcher, 0, 2)
241 for ; t.startBucketID <= currentBucketID; t.startBucketID++ {
242 if watchers, ok := t.watchersBuckets[t.startBucketID]; ok {
243 delete(t.watchersBuckets, t.startBucketID)
244 expiredWatchers = append(expiredWatchers, watchers)
245 }
246 }
247 return expiredWatchers
248 }
249
250 type filterWithAttrsFunc func(key string, l labels.Set, f fields.Set) bool
251
252 type indexedTriggerFunc struct {
253 indexName string
254 indexerFunc storage.IndexerFunc
255 }
256
257 // Cacher is responsible for serving WATCH and LIST requests for a given
258 // resource from its internal cache and updating its cache in the background
259 // based on the underlying storage contents.
260 // Cacher implements storage.Interface (although most of the calls are just
261 // delegated to the underlying storage).
262 type Cacher struct {
263 // HighWaterMarks for performance debugging.
264 // Important: Since HighWaterMark is using sync/atomic, it has to be at the top of the struct due to a bug on 32-bit platforms
265 // See: https://golang.org/pkg/sync/atomic/ for more information
266 incomingHWM storage.HighWaterMark
267 // Incoming events that should be dispatched to watchers.
268 incoming chan watchCacheEvent
269
270 resourcePrefix string
271
272 sync.RWMutex
273
274 // Before accessing the cacher's cache, wait for the ready to be ok.
275 // This is necessary to prevent users from accessing structures that are
276 // uninitialized or are being repopulated right now.
277 // ready needs to be set to false when the cacher is paused or stopped.
278 // ready needs to be set to true when the cacher is ready to use after
279 // initialization.
280 ready *ready
281
282 // Underlying storage.Interface.
283 storage storage.Interface
284
285 // Expected type of objects in the underlying cache.
286 objectType reflect.Type
287 // Used for logging, to disambiguate *unstructured.Unstructured (CRDs)
288 groupResource schema.GroupResource
289
290 // "sliding window" of recent changes of objects and the current state.
291 watchCache *watchCache
292 reflector *cache.Reflector
293
294 // Versioner is used to handle resource versions.
295 versioner storage.Versioner
296
297 // newFunc is a function that creates new empty object storing a object of type Type.
298 newFunc func() runtime.Object
299
300 // newListFunc is a function that creates new empty list for storing objects of type Type.
301 newListFunc func() runtime.Object
302
303 // indexedTrigger is used for optimizing amount of watchers that needs to process
304 // an incoming event.
305 indexedTrigger *indexedTriggerFunc
306 // watchers is mapping from the value of trigger function that a
307 // watcher is interested into the watchers
308 watcherIdx int
309 watchers indexedWatchers
310
311 // Defines a time budget that can be spend on waiting for not-ready watchers
312 // while dispatching event before shutting them down.
313 dispatchTimeoutBudget timeBudget
314
315 // Handling graceful termination.
316 stopLock sync.RWMutex
317 stopped bool
318 stopCh chan struct{}
319 stopWg sync.WaitGroup
320
321 clock clock.Clock
322 // timer is used to avoid unnecessary allocations in underlying watchers.
323 timer *time.Timer
324
325 // dispatching determines whether there is currently dispatching of
326 // any event in flight.
327 dispatching bool
328 // watchersBuffer is a list of watchers potentially interested in currently
329 // dispatched event.
330 watchersBuffer []*cacheWatcher
331 // blockedWatchers is a list of watchers whose buffer is currently full.
332 blockedWatchers []*cacheWatcher
333 // watchersToStop is a list of watchers that were supposed to be stopped
334 // during current dispatching, but stopping was deferred to the end of
335 // dispatching that event to avoid race with closing channels in watchers.
336 watchersToStop []*cacheWatcher
337 // Maintain a timeout queue to send the bookmark event before the watcher times out.
338 // Note that this field when accessed MUST be protected by the Cacher.lock.
339 bookmarkWatchers *watcherBookmarkTimeBuckets
340 // expiredBookmarkWatchers is a list of watchers that were expired and need to be schedule for a next bookmark event
341 expiredBookmarkWatchers []*cacheWatcher
342 }
343
344 // NewCacherFromConfig creates a new Cacher responsible for servicing WATCH and LIST requests from
345 // its internal cache and updating its cache in the background based on the
346 // given configuration.
347 func NewCacherFromConfig(config Config) (*Cacher, error) {
348 stopCh := make(chan struct{})
349 obj := config.NewFunc()
350 // Give this error when it is constructed rather than when you get the
351 // first watch item, because it's much easier to track down that way.
352 if err := runtime.CheckCodec(config.Codec, obj); err != nil {
353 return nil, fmt.Errorf("storage codec doesn't seem to match given type: %v", err)
354 }
355
356 var indexedTrigger *indexedTriggerFunc
357 if config.IndexerFuncs != nil {
358 // For now, we don't support multiple trigger functions defined
359 // for a given resource.
360 if len(config.IndexerFuncs) > 1 {
361 return nil, fmt.Errorf("cacher %s doesn't support more than one IndexerFunc: ", reflect.TypeOf(obj).String())
362 }
363 for key, value := range config.IndexerFuncs {
364 if value != nil {
365 indexedTrigger = &indexedTriggerFunc{
366 indexName: key,
367 indexerFunc: value,
368 }
369 }
370 }
371 }
372
373 if config.Clock == nil {
374 config.Clock = clock.RealClock{}
375 }
376 objType := reflect.TypeOf(obj)
377 cacher := &Cacher{
378 resourcePrefix: config.ResourcePrefix,
379 ready: newReady(config.Clock),
380 storage: config.Storage,
381 objectType: objType,
382 groupResource: config.GroupResource,
383 versioner: config.Versioner,
384 newFunc: config.NewFunc,
385 newListFunc: config.NewListFunc,
386 indexedTrigger: indexedTrigger,
387 watcherIdx: 0,
388 watchers: indexedWatchers{
389 allWatchers: make(map[namespacedName]watchersMap),
390 valueWatchers: make(map[string]watchersMap),
391 },
392 // TODO: Figure out the correct value for the buffer size.
393 incoming: make(chan watchCacheEvent, 100),
394 dispatchTimeoutBudget: newTimeBudget(),
395 // We need to (potentially) stop both:
396 // - wait.Until go-routine
397 // - reflector.ListAndWatch
398 // and there are no guarantees on the order that they will stop.
399 // So we will be simply closing the channel, and synchronizing on the WaitGroup.
400 stopCh: stopCh,
401 clock: config.Clock,
402 timer: time.NewTimer(time.Duration(0)),
403 bookmarkWatchers: newTimeBucketWatchers(config.Clock, defaultBookmarkFrequency),
404 }
405
406 // Ensure that timer is stopped.
407 if !cacher.timer.Stop() {
408 // Consume triggered (but not yet received) timer event
409 // so that future reuse does not get a spurious timeout.
410 <-cacher.timer.C
411 }
412 var contextMetadata metadata.MD
413 if utilfeature.DefaultFeatureGate.Enabled(features.SeparateCacheWatchRPC) {
414 // Add grpc context metadata to watch and progress notify requests done by cacher to:
415 // * Prevent starvation of watch opened by cacher, by moving it to separate Watch RPC than watch request that bypass cacher.
416 // * Ensure that progress notification requests are executed on the same Watch RPC as their watch, which is required for it to work.
417 contextMetadata = metadata.New(map[string]string{"source": "cache"})
418 }
419
420 eventFreshDuration := config.EventsHistoryWindow
421 if eventFreshDuration < DefaultEventFreshDuration {
422 return nil, fmt.Errorf("config.EventsHistoryWindow (%v) must not be below %v", eventFreshDuration, DefaultEventFreshDuration)
423 }
424
425 progressRequester := progress.NewConditionalProgressRequester(config.Storage.RequestWatchProgress, config.Clock, contextMetadata)
426 watchCache := newWatchCache(
427 config.KeyFunc, cacher.processEvent, config.GetAttrsFunc, config.Versioner, config.Indexers,
428 config.Clock, eventFreshDuration, config.GroupResource, progressRequester)
429 listerWatcher := NewListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc, contextMetadata)
430 reflectorName := "storage/cacher.go:" + config.ResourcePrefix
431
432 reflector := cache.NewNamedReflector(reflectorName, listerWatcher, obj, watchCache, 0)
433 // Configure reflector's pager to for an appropriate pagination chunk size for fetching data from
434 // storage. The pager falls back to full list if paginated list calls fail due to an "Expired" error.
435 reflector.WatchListPageSize = storageWatchListPageSize
436 // When etcd loses leader for 3 cycles, it returns error "no leader".
437 // We don't want to terminate all watchers as recreating all watchers puts high load on api-server.
438 // In most of the cases, leader is reelected within few cycles.
439 reflector.MaxInternalErrorRetryDuration = time.Second * 30
440 // since the watch-list is provided by the watch cache instruct
441 // the reflector to issue a regular LIST against the store
442 reflector.UseWatchList = ptr.To(false)
443
444 cacher.watchCache = watchCache
445 cacher.reflector = reflector
446
447 go cacher.dispatchEvents()
448 go progressRequester.Run(stopCh)
449
450 cacher.stopWg.Add(1)
451 go func() {
452 defer cacher.stopWg.Done()
453 defer cacher.terminateAllWatchers()
454 wait.Until(
455 func() {
456 if !cacher.isStopped() {
457 cacher.startCaching(stopCh)
458 }
459 }, time.Second, stopCh,
460 )
461 }()
462
463 return cacher, nil
464 }
465
466 func (c *Cacher) startCaching(stopChannel <-chan struct{}) {
467 c.watchCache.SetOnReplace(func() {
468 c.ready.setReady()
469 klog.V(1).Infof("cacher (%v): initialized", c.groupResource.String())
470 metrics.WatchCacheInitializations.WithLabelValues(c.groupResource.String()).Inc()
471 })
472 var err error
473 defer func() {
474 c.ready.setError(err)
475 }()
476
477 c.terminateAllWatchers()
478 err = c.reflector.ListAndWatch(stopChannel)
479 if err != nil {
480 klog.Errorf("cacher (%v): unexpected ListAndWatch error: %v; reinitializing...", c.groupResource.String(), err)
481 }
482 }
483
484 type namespacedName struct {
485 namespace string
486 name string
487 }
488
489 func (c *Cacher) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
490 pred := opts.Predicate
491 requestedWatchRV, err := c.versioner.ParseResourceVersion(opts.ResourceVersion)
492 if err != nil {
493 return nil, err
494 }
495
496 var readyGeneration int
497 if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
498 var err error
499 var downtime time.Duration
500 readyGeneration, downtime, err = c.ready.checkAndReadGeneration()
501 if err != nil {
502 return nil, errors.NewTooManyRequests(err.Error(), calculateRetryAfterForUnreadyCache(downtime))
503 }
504 } else {
505 readyGeneration, err = c.ready.waitAndReadGeneration(ctx)
506 if err != nil {
507 return nil, errors.NewServiceUnavailable(err.Error())
508 }
509 }
510
511 // determine the namespace and name scope of the watch, first from the request, secondarily from the field selector
512 scope := namespacedName{}
513 if requestNamespace, ok := request.NamespaceFrom(ctx); ok && len(requestNamespace) > 0 {
514 scope.namespace = requestNamespace
515 } else if selectorNamespace, ok := pred.Field.RequiresExactMatch("metadata.namespace"); ok {
516 scope.namespace = selectorNamespace
517 }
518 if requestInfo, ok := request.RequestInfoFrom(ctx); ok && requestInfo != nil && len(requestInfo.Name) > 0 {
519 scope.name = requestInfo.Name
520 } else if selectorName, ok := pred.Field.RequiresExactMatch("metadata.name"); ok {
521 scope.name = selectorName
522 }
523
524 // for request like '/api/v1/watch/namespaces/*', set scope.namespace to empty.
525 // namespaces don't populate metadata.namespace in ObjFields.
526 if c.groupResource == coreNamespaceResource && len(scope.namespace) > 0 && scope.namespace == scope.name {
527 scope.namespace = ""
528 }
529
530 triggerValue, triggerSupported := "", false
531 if c.indexedTrigger != nil {
532 for _, field := range pred.IndexFields {
533 if field == c.indexedTrigger.indexName {
534 if value, ok := pred.Field.RequiresExactMatch(field); ok {
535 triggerValue, triggerSupported = value, true
536 break
537 }
538 }
539 }
540 }
541
542 // It boils down to a tradeoff between:
543 // - having it as small as possible to reduce memory usage
544 // - having it large enough to ensure that watchers that need to process
545 // a bunch of changes have enough buffer to avoid from blocking other
546 // watchers on our watcher having a processing hiccup
547 chanSize := c.watchCache.suggestedWatchChannelSize(c.indexedTrigger != nil, triggerSupported)
548
549 // client-go is going to fall back to a standard LIST on any error
550 // returned for watch-list requests
551 if isListWatchRequest(opts) && !etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress) {
552 return newErrWatcher(fmt.Errorf("a watch stream was requested by the client but the required storage feature %s is disabled", storage.RequestWatchProgress)), nil
553 }
554
555 // Determine the ResourceVersion to which the watch cache must be synchronized
556 requiredResourceVersion, err := c.getWatchCacheResourceVersion(ctx, requestedWatchRV, opts)
557 if err != nil {
558 return newErrWatcher(err), nil
559 }
560
561 // Determine a function that computes the bookmarkAfterResourceVersion
562 bookmarkAfterResourceVersionFn, err := c.getBookmarkAfterResourceVersionLockedFunc(requestedWatchRV, requiredResourceVersion, opts)
563 if err != nil {
564 return newErrWatcher(err), nil
565 }
566
567 // Determine watch timeout('0' means deadline is not set, ignore checking)
568 deadline, _ := ctx.Deadline()
569
570 identifier := fmt.Sprintf("key: %q, labels: %q, fields: %q", key, pred.Label, pred.Field)
571
572 // Create a watcher here to reduce memory allocations under lock,
573 // given that memory allocation may trigger GC and block the thread.
574 // Also note that emptyFunc is a placeholder, until we will be able
575 // to compute watcher.forget function (which has to happen under lock).
576 watcher := newCacheWatcher(
577 chanSize,
578 filterWithAttrsAndPrefixFunction(key, pred),
579 emptyFunc,
580 c.versioner,
581 deadline,
582 pred.AllowWatchBookmarks,
583 c.groupResource,
584 identifier,
585 )
586
587 // note that c.waitUntilWatchCacheFreshAndForceAllEvents must be called without
588 // the c.watchCache.RLock held otherwise we are at risk of a deadlock
589 // mainly because c.watchCache.processEvent method won't be able to make progress
590 //
591 // moreover even though the c.waitUntilWatchCacheFreshAndForceAllEvents acquires a lock
592 // it is safe to release the lock after the method finishes because we don't require
593 // any atomicity between the call to the method and further calls that actually get the events.
594 err = c.waitUntilWatchCacheFreshAndForceAllEvents(ctx, requiredResourceVersion, opts)
595 if err != nil {
596 return newErrWatcher(err), nil
597 }
598
599 // We explicitly use thread unsafe version and do locking ourself to ensure that
600 // no new events will be processed in the meantime. The watchCache will be unlocked
601 // on return from this function.
602 // Note that we cannot do it under Cacher lock, to avoid a deadlock, since the
603 // underlying watchCache is calling processEvent under its lock.
604 c.watchCache.RLock()
605 defer c.watchCache.RUnlock()
606
607 var cacheInterval *watchCacheInterval
608 cacheInterval, err = c.watchCache.getAllEventsSinceLocked(requiredResourceVersion, key, opts)
609 if err != nil {
610 // To match the uncached watch implementation, once we have passed authn/authz/admission,
611 // and successfully parsed a resource version, other errors must fail with a watch event of type ERROR,
612 // rather than a directly returned error.
613 return newErrWatcher(err), nil
614 }
615
616 c.setInitialEventsEndBookmarkIfRequested(cacheInterval, opts, c.watchCache.resourceVersion)
617
618 addedWatcher := false
619 func() {
620 c.Lock()
621 defer c.Unlock()
622
623 if generation, _, err := c.ready.checkAndReadGeneration(); generation != readyGeneration || err != nil {
624 // We went unready or are already on a different generation.
625 // Avoid registering and starting the watch as it will have to be
626 // terminated immediately anyway.
627 return
628 }
629
630 // Update watcher.forget function once we can compute it.
631 watcher.forget = forgetWatcher(c, watcher, c.watcherIdx, scope, triggerValue, triggerSupported)
632 // Update the bookMarkAfterResourceVersion
633 watcher.setBookmarkAfterResourceVersion(bookmarkAfterResourceVersionFn())
634 c.watchers.addWatcher(watcher, c.watcherIdx, scope, triggerValue, triggerSupported)
635 addedWatcher = true
636
637 // Add it to the queue only when the client support watch bookmarks.
638 if watcher.allowWatchBookmarks {
639 c.bookmarkWatchers.addWatcherThreadUnsafe(watcher)
640 }
641 c.watcherIdx++
642 }()
643
644 if !addedWatcher {
645 // Watcher isn't really started at this point, so it's safe to just drop it.
646 //
647 // We're simulating the immediate watch termination, which boils down to simply
648 // closing the watcher.
649 return newImmediateCloseWatcher(), nil
650 }
651
652 go watcher.processInterval(ctx, cacheInterval, requiredResourceVersion)
653 return watcher, nil
654 }
655
656 func (c *Cacher) Get(ctx context.Context, key string, opts storage.GetOptions, objPtr runtime.Object) error {
657 getRV, err := c.versioner.ParseResourceVersion(opts.ResourceVersion)
658 if err != nil {
659 return err
660 }
661
662 objVal, err := conversion.EnforcePtr(objPtr)
663 if err != nil {
664 return err
665 }
666
667 obj, exists, readResourceVersion, err := c.watchCache.WaitUntilFreshAndGet(ctx, getRV, key)
668 if err != nil {
669 return err
670 }
671
672 if exists {
673 elem, ok := obj.(*storeElement)
674 if !ok {
675 return fmt.Errorf("non *storeElement returned from storage: %v", obj)
676 }
677 objVal.Set(reflect.ValueOf(elem.Object).Elem())
678 } else {
679 objVal.Set(reflect.Zero(objVal.Type()))
680 if !opts.IgnoreNotFound {
681 return storage.NewKeyNotFoundError(key, int64(readResourceVersion))
682 }
683 }
684 return nil
685 }
686
687 // computeListLimit determines whether the cacher should
688 // apply a limit to an incoming LIST request and returns its value.
689 //
690 // note that this function doesn't check RVM nor the Continuation token.
691 // these parameters are validated by the shouldDelegateList function.
692 //
693 // as of today, the limit is ignored for requests that set RV == 0
694 func computeListLimit(opts storage.ListOptions) int64 {
695 if opts.Predicate.Limit <= 0 || opts.ResourceVersion == "0" {
696 return 0
697 }
698 return opts.Predicate.Limit
699 }
700
701 func (c *Cacher) listItems(ctx context.Context, listRV uint64, key string, opts storage.ListOptions) (listResp, string, error) {
702 if !opts.Recursive {
703 obj, exists, readResourceVersion, err := c.watchCache.WaitUntilFreshAndGet(ctx, listRV, key)
704 if err != nil {
705 return listResp{}, "", err
706 }
707 if exists {
708 return listResp{Items: []interface{}{obj}, ResourceVersion: readResourceVersion}, "", nil
709 }
710 return listResp{ResourceVersion: readResourceVersion}, "", nil
711 }
712 return c.watchCache.WaitUntilFreshAndList(ctx, listRV, key, opts)
713 }
714
715 type listResp struct {
716 Items []interface{}
717 ResourceVersion uint64
718 }
719
720 // GetList implements storage.Interface
721 func (c *Cacher) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
722 // For recursive lists, we need to make sure the key ended with "/" so that we only
723 // get children "directories". e.g. if we have key "/a", "/a/b", "/ab", getting keys
724 // with prefix "/a" will return all three, while with prefix "/a/" will return only
725 // "/a/b" which is the correct answer.
726 preparedKey := key
727 if opts.Recursive && !strings.HasSuffix(key, "/") {
728 preparedKey += "/"
729 }
730 listRV, err := c.versioner.ParseResourceVersion(opts.ResourceVersion)
731 if err != nil {
732 return err
733 }
734
735 ctx, span := tracing.Start(ctx, "cacher.GetList",
736 attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)),
737 attribute.Stringer("type", c.groupResource))
738 defer span.End(500 * time.Millisecond)
739
740 if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
741 if downtime, err := c.ready.check(); err != nil {
742 // If Cacher is not initialized, reject List requests
743 // as described in https://kep.k8s.io/4568
744 return errors.NewTooManyRequests(err.Error(), calculateRetryAfterForUnreadyCache(downtime))
745 }
746 } else {
747 if err := c.ready.wait(ctx); err != nil {
748 return errors.NewServiceUnavailable(err.Error())
749 }
750 }
751 span.AddEvent("Ready")
752
753 // List elements with at least 'listRV' from cache.
754 listPtr, err := meta.GetItemsPtr(listObj)
755 if err != nil {
756 return err
757 }
758 listVal, err := conversion.EnforcePtr(listPtr)
759 if err != nil {
760 return err
761 }
762 if listVal.Kind() != reflect.Slice {
763 return fmt.Errorf("need a pointer to slice, got %v", listVal.Kind())
764 }
765
766 resp, indexUsed, err := c.listItems(ctx, listRV, preparedKey, opts)
767 if err != nil {
768 return err
769 }
770 span.AddEvent("Listed items from cache", attribute.Int("count", len(resp.Items)))
771 // store pointer of eligible objects,
772 // Why not directly put object in the items of listObj?
773 // the elements in ListObject are Struct type, making slice will bring excessive memory consumption.
774 // so we try to delay this action as much as possible
775 var selectedObjects []runtime.Object
776 var lastSelectedObjectKey string
777 var hasMoreListItems bool
778 limit := computeListLimit(opts)
779 for i, obj := range resp.Items {
780 elem, ok := obj.(*storeElement)
781 if !ok {
782 return fmt.Errorf("non *storeElement returned from storage: %v", obj)
783 }
784 if opts.Predicate.MatchesObjectAttributes(elem.Labels, elem.Fields) {
785 selectedObjects = append(selectedObjects, elem.Object)
786 lastSelectedObjectKey = elem.Key
787 }
788 if limit > 0 && int64(len(selectedObjects)) >= limit {
789 hasMoreListItems = i < len(resp.Items)-1
790 break
791 }
792 }
793 if len(selectedObjects) == 0 {
794 // Ensure that we never return a nil Items pointer in the result for consistency.
795 listVal.Set(reflect.MakeSlice(listVal.Type(), 0, 0))
796 } else {
797 // Resize the slice appropriately, since we already know that size of result set
798 listVal.Set(reflect.MakeSlice(listVal.Type(), len(selectedObjects), len(selectedObjects)))
799 span.AddEvent("Resized result")
800 for i, o := range selectedObjects {
801 listVal.Index(i).Set(reflect.ValueOf(o).Elem())
802 }
803 }
804 span.AddEvent("Filtered items", attribute.Int("count", listVal.Len()))
805 if c.versioner != nil {
806 continueValue, remainingItemCount, err := storage.PrepareContinueToken(lastSelectedObjectKey, key, int64(resp.ResourceVersion), int64(len(resp.Items)), hasMoreListItems, opts)
807 if err != nil {
808 return err
809 }
810
811 if err = c.versioner.UpdateList(listObj, resp.ResourceVersion, continueValue, remainingItemCount); err != nil {
812 return err
813 }
814 }
815 metrics.RecordListCacheMetrics(c.resourcePrefix, indexUsed, len(resp.Items), listVal.Len())
816 return nil
817 }
818
819 // baseObjectThreadUnsafe omits locking for cachingObject.
820 func baseObjectThreadUnsafe(object runtime.Object) runtime.Object {
821 if co, ok := object.(*cachingObject); ok {
822 return co.object
823 }
824 return object
825 }
826
827 func (c *Cacher) triggerValuesThreadUnsafe(event *watchCacheEvent) ([]string, bool) {
828 if c.indexedTrigger == nil {
829 return nil, false
830 }
831
832 result := make([]string, 0, 2)
833 result = append(result, c.indexedTrigger.indexerFunc(baseObjectThreadUnsafe(event.Object)))
834 if event.PrevObject == nil {
835 return result, true
836 }
837 prevTriggerValue := c.indexedTrigger.indexerFunc(baseObjectThreadUnsafe(event.PrevObject))
838 if result[0] != prevTriggerValue {
839 result = append(result, prevTriggerValue)
840 }
841 return result, true
842 }
843
844 func (c *Cacher) processEvent(event *watchCacheEvent) {
845 if curLen := int64(len(c.incoming)); c.incomingHWM.Update(curLen) {
846 // Monitor if this gets backed up, and how much.
847 klog.V(1).Infof("cacher (%v): %v objects queued in incoming channel.", c.groupResource.String(), curLen)
848 }
849 c.incoming <- *event
850 }
851
852 func (c *Cacher) dispatchEvents() {
853 // Jitter to help level out any aggregate load.
854 bookmarkTimer := c.clock.NewTimer(wait.Jitter(time.Second, 0.25))
855 defer bookmarkTimer.Stop()
856
857 // The internal informer populates the RV as soon as it conducts
858 // The first successful sync with the underlying store.
859 // The cache must wait until this first sync is completed to be deemed ready.
860 // Since we cannot send a bookmark when the lastProcessedResourceVersion is 0,
861 // we poll aggressively for the first list RV before entering the dispatch loop.
862 lastProcessedResourceVersion := uint64(0)
863 if err := wait.PollUntilContextCancel(wait.ContextForChannel(c.stopCh), 10*time.Millisecond, true, func(_ context.Context) (bool, error) {
864 if rv := c.watchCache.getListResourceVersion(); rv != 0 {
865 lastProcessedResourceVersion = rv
866 return true, nil
867 }
868 return false, nil
869 }); err != nil {
870 // given the function above never returns error,
871 // the non-empty error means that the stopCh was closed
872 return
873 }
874 for {
875 select {
876 case event, ok := <-c.incoming:
877 if !ok {
878 return
879 }
880 // Don't dispatch bookmarks coming from the storage layer.
881 // They can be very frequent (even to the level of subseconds)
882 // to allow efficient watch resumption on kube-apiserver restarts,
883 // and propagating them down may overload the whole system.
884 //
885 // TODO: If at some point we decide the performance and scalability
886 // footprint is acceptable, this is the place to hook them in.
887 // However, we then need to check if this was called as a result
888 // of a bookmark event or regular Add/Update/Delete operation by
889 // checking if resourceVersion here has changed.
890 if event.Type != watch.Bookmark {
891 c.dispatchEvent(&event)
892 }
893 lastProcessedResourceVersion = event.ResourceVersion
894 metrics.EventsCounter.WithLabelValues(c.groupResource.String()).Inc()
895 case <-bookmarkTimer.C():
896 bookmarkTimer.Reset(wait.Jitter(time.Second, 0.25))
897 bookmarkEvent := &watchCacheEvent{
898 Type: watch.Bookmark,
899 Object: c.newFunc(),
900 ResourceVersion: lastProcessedResourceVersion,
901 }
902 if err := c.versioner.UpdateObject(bookmarkEvent.Object, bookmarkEvent.ResourceVersion); err != nil {
903 klog.Errorf("failure to set resourceVersion to %d on bookmark event %+v", bookmarkEvent.ResourceVersion, bookmarkEvent.Object)
904 continue
905 }
906 c.dispatchEvent(bookmarkEvent)
907 case <-c.stopCh:
908 return
909 }
910 }
911 }
912
913 func setCachingObjects(event *watchCacheEvent, versioner storage.Versioner) {
914 switch event.Type {
915 case watch.Added, watch.Modified:
916 if object, err := newCachingObject(event.Object); err == nil {
917 event.Object = object
918 } else {
919 klog.Errorf("couldn't create cachingObject from: %#v", event.Object)
920 }
921 // Don't wrap PrevObject for update event (for create events it is nil).
922 // We only encode those to deliver DELETE watch events, so if
923 // event.Object is not nil it can be used only for watchers for which
924 // selector was satisfied for its previous version and is no longer
925 // satisfied for the current version.
926 // This is rare enough that it doesn't justify making deep-copy of the
927 // object (done by newCachingObject) every time.
928 case watch.Deleted:
929 // Don't wrap Object for delete events - these are not to deliver any
930 // events. Only wrap PrevObject.
931 if object, err := newCachingObject(event.PrevObject); err == nil {
932 // Update resource version of the object.
933 // event.PrevObject is used to deliver DELETE watch events and
934 // for them, we set resourceVersion to <current> instead of
935 // the resourceVersion of the last modification of the object.
936 updateResourceVersion(object, versioner, event.ResourceVersion)
937 event.PrevObject = object
938 } else {
939 klog.Errorf("couldn't create cachingObject from: %#v", event.Object)
940 }
941 }
942 }
943
944 func (c *Cacher) dispatchEvent(event *watchCacheEvent) {
945 c.startDispatching(event)
946 defer c.finishDispatching()
947 // Watchers stopped after startDispatching will be delayed to finishDispatching,
948
949 // Since add() can block, we explicitly add when cacher is unlocked.
950 // Dispatching event in nonblocking way first, which make faster watchers
951 // not be blocked by slower ones.
952 //
953 // Note: if we ever decide to cache the serialization of bookmark events,
954 // we will also need to modify the watchEncoder encoder
955 if event.Type == watch.Bookmark {
956 for _, watcher := range c.watchersBuffer {
957 watcher.nonblockingAdd(event)
958 }
959 } else {
960 // Set up caching of object serializations only for dispatching this event.
961 //
962 // Storing serializations in memory would result in increased memory usage,
963 // but it would help for caching encodings for watches started from old
964 // versions. However, we still don't have a convincing data that the gain
965 // from it justifies increased memory usage, so for now we drop the cached
966 // serializations after dispatching this event.
967 //
968 // Given that CachingObject is just wrapping the object and not perfoming
969 // deep-copying (until some field is explicitly being modified), we create
970 // it unconditionally to ensure safety and reduce deep-copying.
971 //
972 // Make a shallow copy to allow overwriting Object and PrevObject.
973 wcEvent := *event
974 setCachingObjects(&wcEvent, c.versioner)
975 event = &wcEvent
976
977 c.blockedWatchers = c.blockedWatchers[:0]
978 for _, watcher := range c.watchersBuffer {
979 if !watcher.nonblockingAdd(event) {
980 c.blockedWatchers = append(c.blockedWatchers, watcher)
981 }
982 }
983
984 if len(c.blockedWatchers) > 0 {
985 // dispatchEvent is called very often, so arrange
986 // to reuse timers instead of constantly allocating.
987 startTime := time.Now()
988 timeout := c.dispatchTimeoutBudget.takeAvailable()
989 c.timer.Reset(timeout)
990
991 // Send event to all blocked watchers. As long as timer is running,
992 // `add` will wait for the watcher to unblock. After timeout,
993 // `add` will not wait, but immediately close a still blocked watcher.
994 // Hence, every watcher gets the chance to unblock itself while timer
995 // is running, not only the first ones in the list.
996 timer := c.timer
997 for _, watcher := range c.blockedWatchers {
998 if !watcher.add(event, timer) {
999 // fired, clean the timer by set it to nil.
1000 timer = nil
1001 }
1002 }
1003
1004 // Stop the timer if it is not fired
1005 if timer != nil && !timer.Stop() {
1006 // Consume triggered (but not yet received) timer event
1007 // so that future reuse does not get a spurious timeout.
1008 <-timer.C
1009 }
1010
1011 c.dispatchTimeoutBudget.returnUnused(timeout - time.Since(startTime))
1012 }
1013 }
1014 }
1015
1016 func (c *Cacher) startDispatchingBookmarkEventsLocked() {
1017 // Pop already expired watchers. However, explicitly ignore stopped ones,
1018 // as we don't delete watcher from bookmarkWatchers when it is stopped.
1019 for _, watchers := range c.bookmarkWatchers.popExpiredWatchersThreadUnsafe() {
1020 for _, watcher := range watchers {
1021 // c.Lock() is held here.
1022 // watcher.stopThreadUnsafe() is protected by c.Lock()
1023 if watcher.stopped {
1024 continue
1025 }
1026 c.watchersBuffer = append(c.watchersBuffer, watcher)
1027 c.expiredBookmarkWatchers = append(c.expiredBookmarkWatchers, watcher)
1028 }
1029 }
1030 }
1031
1032 // startDispatching chooses watchers potentially interested in a given event
1033 // a marks dispatching as true.
1034 func (c *Cacher) startDispatching(event *watchCacheEvent) {
1035 // It is safe to call triggerValuesThreadUnsafe here, because at this
1036 // point only this thread can access this event (we create a separate
1037 // watchCacheEvent for every dispatch).
1038 triggerValues, supported := c.triggerValuesThreadUnsafe(event)
1039
1040 c.Lock()
1041 defer c.Unlock()
1042
1043 c.dispatching = true
1044 // We are reusing the slice to avoid memory reallocations in every
1045 // dispatchEvent() call. That may prevent Go GC from freeing items
1046 // from previous phases that are sitting behind the current length
1047 // of the slice, but there is only a limited number of those and the
1048 // gain from avoiding memory allocations is much bigger.
1049 c.watchersBuffer = c.watchersBuffer[:0]
1050
1051 if event.Type == watch.Bookmark {
1052 c.startDispatchingBookmarkEventsLocked()
1053 // return here to reduce following code indentation and diff
1054 return
1055 }
1056
1057 // iterate over watchers for each applicable namespace/name tuple
1058 namespace := event.ObjFields["metadata.namespace"]
1059 name := event.ObjFields["metadata.name"]
1060 if len(namespace) > 0 {
1061 if len(name) > 0 {
1062 // namespaced watchers scoped by name
1063 for _, watcher := range c.watchers.allWatchers[namespacedName{namespace: namespace, name: name}] {
1064 c.watchersBuffer = append(c.watchersBuffer, watcher)
1065 }
1066 }
1067 // namespaced watchers not scoped by name
1068 for _, watcher := range c.watchers.allWatchers[namespacedName{namespace: namespace}] {
1069 c.watchersBuffer = append(c.watchersBuffer, watcher)
1070 }
1071 }
1072 if len(name) > 0 {
1073 // cluster-wide watchers scoped by name
1074 for _, watcher := range c.watchers.allWatchers[namespacedName{name: name}] {
1075 c.watchersBuffer = append(c.watchersBuffer, watcher)
1076 }
1077 }
1078 // cluster-wide watchers unscoped by name
1079 for _, watcher := range c.watchers.allWatchers[namespacedName{}] {
1080 c.watchersBuffer = append(c.watchersBuffer, watcher)
1081 }
1082
1083 if supported {
1084 // Iterate over watchers interested in the given values of the trigger.
1085 for _, triggerValue := range triggerValues {
1086 for _, watcher := range c.watchers.valueWatchers[triggerValue] {
1087 c.watchersBuffer = append(c.watchersBuffer, watcher)
1088 }
1089 }
1090 } else {
1091 // supported equal to false generally means that trigger function
1092 // is not defined (or not aware of any indexes). In this case,
1093 // watchers filters should generally also don't generate any
1094 // trigger values, but can cause problems in case of some
1095 // misconfiguration. Thus we paranoidly leave this branch.
1096
1097 // Iterate over watchers interested in exact values for all values.
1098 for _, watchers := range c.watchers.valueWatchers {
1099 for _, watcher := range watchers {
1100 c.watchersBuffer = append(c.watchersBuffer, watcher)
1101 }
1102 }
1103 }
1104 }
1105
1106 // finishDispatching stops all the watchers that were supposed to be
1107 // stopped in the meantime, but it was deferred to avoid closing input
1108 // channels of watchers, as add() may still have writing to it.
1109 // It also marks dispatching as false.
1110 func (c *Cacher) finishDispatching() {
1111 c.Lock()
1112 defer c.Unlock()
1113 c.dispatching = false
1114 for _, watcher := range c.watchersToStop {
1115 watcher.stopLocked()
1116 }
1117 c.watchersToStop = c.watchersToStop[:0]
1118
1119 for _, watcher := range c.expiredBookmarkWatchers {
1120 if watcher.stopped {
1121 continue
1122 }
1123 // requeue the watcher for the next bookmark if needed.
1124 c.bookmarkWatchers.addWatcherThreadUnsafe(watcher)
1125 }
1126 c.expiredBookmarkWatchers = c.expiredBookmarkWatchers[:0]
1127 }
1128
1129 func (c *Cacher) terminateAllWatchers() {
1130 c.Lock()
1131 defer c.Unlock()
1132 c.watchers.terminateAll(c.groupResource, c.stopWatcherLocked)
1133 }
1134
1135 func (c *Cacher) stopWatcherLocked(watcher *cacheWatcher) {
1136 if c.dispatching {
1137 c.watchersToStop = append(c.watchersToStop, watcher)
1138 } else {
1139 watcher.stopLocked()
1140 }
1141 }
1142
1143 func (c *Cacher) isStopped() bool {
1144 c.stopLock.RLock()
1145 defer c.stopLock.RUnlock()
1146 return c.stopped
1147 }
1148
1149 // Stop implements the graceful termination.
1150 func (c *Cacher) Stop() {
1151 c.stopLock.Lock()
1152 if c.stopped {
1153 // avoid stopping twice (note: cachers are shared with subresources)
1154 c.stopLock.Unlock()
1155 return
1156 }
1157 c.stopped = true
1158 c.ready.stop()
1159 c.stopLock.Unlock()
1160 close(c.stopCh)
1161 c.stopWg.Wait()
1162 }
1163
1164 func forgetWatcher(c *Cacher, w *cacheWatcher, index int, scope namespacedName, triggerValue string, triggerSupported bool) func(bool) {
1165 return func(drainWatcher bool) {
1166 c.Lock()
1167 defer c.Unlock()
1168
1169 w.setDrainInputBufferLocked(drainWatcher)
1170
1171 // It's possible that the watcher is already not in the structure (e.g. in case of
1172 // simultaneous Stop() and terminateAllWatchers(), but it is safe to call stopLocked()
1173 // on a watcher multiple times.
1174 c.watchers.deleteWatcher(index, scope, triggerValue, triggerSupported)
1175 c.stopWatcherLocked(w)
1176 }
1177 }
1178
1179 func filterWithAttrsAndPrefixFunction(key string, p storage.SelectionPredicate) filterWithAttrsFunc {
1180 filterFunc := func(objKey string, label labels.Set, field fields.Set) bool {
1181 if !hasPathPrefix(objKey, key) {
1182 return false
1183 }
1184 return p.MatchesObjectAttributes(label, field)
1185 }
1186 return filterFunc
1187 }
1188
1189 // LastSyncResourceVersion returns resource version to which the underlying cache is synced.
1190 func (c *Cacher) LastSyncResourceVersion() (uint64, error) {
1191 if err := c.ready.wait(context.Background()); err != nil {
1192 return 0, errors.NewServiceUnavailable(err.Error())
1193 }
1194
1195 resourceVersion := c.reflector.LastSyncResourceVersion()
1196 return c.versioner.ParseResourceVersion(resourceVersion)
1197 }
1198
1199 // getBookmarkAfterResourceVersionLockedFunc returns a function that
1200 // spits a ResourceVersion after which the bookmark event will be delivered.
1201 //
1202 // The returned function must be called under the watchCache lock.
1203 func (c *Cacher) getBookmarkAfterResourceVersionLockedFunc(parsedResourceVersion, requiredResourceVersion uint64, opts storage.ListOptions) (func() uint64, error) {
1204 if !isListWatchRequest(opts) {
1205 return func() uint64 { return 0 }, nil
1206 }
1207
1208 switch {
1209 case len(opts.ResourceVersion) == 0:
1210 return func() uint64 { return requiredResourceVersion }, nil
1211 case parsedResourceVersion == 0:
1212 // here we assume that watchCache locked is already held
1213 return func() uint64 { return c.watchCache.resourceVersion }, nil
1214 default:
1215 return func() uint64 { return parsedResourceVersion }, nil
1216 }
1217 }
1218
1219 func isListWatchRequest(opts storage.ListOptions) bool {
1220 return opts.SendInitialEvents != nil && *opts.SendInitialEvents && opts.Predicate.AllowWatchBookmarks
1221 }
1222
1223 // getWatchCacheResourceVersion returns a ResourceVersion to which the watch cache must be synchronized to
1224 //
1225 // Depending on the input parameters, the semantics of the returned ResourceVersion are:
1226 // - must be at Exact RV (when parsedWatchResourceVersion > 0)
1227 // - can be at Any RV (when parsedWatchResourceVersion = 0)
1228 // - must be at Most Recent RV (return an RV from etcd)
1229 //
1230 // note that the above semantic is enforced by the API validation (defined elsewhere):
1231 //
1232 // if SendInitiaEvents != nil => ResourceVersionMatch = NotOlderThan
1233 // if ResourceVersionmatch != nil => ResourceVersionMatch = NotOlderThan & SendInitialEvents != nil
1234 func (c *Cacher) getWatchCacheResourceVersion(ctx context.Context, parsedWatchResourceVersion uint64, opts storage.ListOptions) (uint64, error) {
1235 if len(opts.ResourceVersion) != 0 {
1236 return parsedWatchResourceVersion, nil
1237 }
1238 // legacy case
1239 if !utilfeature.DefaultFeatureGate.Enabled(features.WatchFromStorageWithoutResourceVersion) && opts.SendInitialEvents == nil && opts.ResourceVersion == "" {
1240 return 0, nil
1241 }
1242 rv, err := c.storage.GetCurrentResourceVersion(ctx)
1243 return rv, err
1244 }
1245
1246 // waitUntilWatchCacheFreshAndForceAllEvents waits until cache is at least
1247 // as fresh as given requestedWatchRV if sendInitialEvents was requested.
1248 // otherwise, we allow for establishing the connection because the clients
1249 // can wait for events without unnecessary blocking.
1250 func (c *Cacher) waitUntilWatchCacheFreshAndForceAllEvents(ctx context.Context, requestedWatchRV uint64, opts storage.ListOptions) error {
1251 if opts.SendInitialEvents != nil && *opts.SendInitialEvents {
1252 // Here be dragons:
1253 // Since the etcd feature checker needs to check all members
1254 // to determine whether a given feature is supported,
1255 // we may receive a positive response even if the feature is not supported.
1256 //
1257 // In this very rare scenario, the worst case will be that this
1258 // request will wait for 3 seconds before it fails.
1259 if etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress) && c.watchCache.notFresh(requestedWatchRV) {
1260 c.watchCache.waitingUntilFresh.Add()
1261 defer c.watchCache.waitingUntilFresh.Remove()
1262 }
1263 err := c.watchCache.waitUntilFreshAndBlock(ctx, requestedWatchRV)
1264 defer c.watchCache.RUnlock()
1265 return err
1266 }
1267 return nil
1268 }
1269
1270 // Wait blocks until the cacher is Ready or Stopped, it returns an error if Stopped.
1271 func (c *Cacher) Wait(ctx context.Context) error {
1272 return c.ready.wait(ctx)
1273 }
1274
1275 // setInitialEventsEndBookmarkIfRequested sets initialEventsEndBookmark field in watchCacheInterval for watchlist request
1276 func (c *Cacher) setInitialEventsEndBookmarkIfRequested(cacheInterval *watchCacheInterval, opts storage.ListOptions, currentResourceVersion uint64) {
1277 if opts.SendInitialEvents != nil && *opts.SendInitialEvents && opts.Predicate.AllowWatchBookmarks {
1278 // We don't need to set the InitialEventsAnnotation for this bookmark event,
1279 // because this will be automatically set during event conversion in cacheWatcher.convertToWatchEvent method
1280 initialEventsEndBookmark := &watchCacheEvent{
1281 Type: watch.Bookmark,
1282 Object: c.newFunc(),
1283 ResourceVersion: currentResourceVersion,
1284 }
1285
1286 if err := c.versioner.UpdateObject(initialEventsEndBookmark.Object, initialEventsEndBookmark.ResourceVersion); err != nil {
1287 klog.Errorf("failure to set resourceVersion to %d on initialEventsEndBookmark event %+v for watchlist request and wait for bookmark trigger to send", initialEventsEndBookmark.ResourceVersion, initialEventsEndBookmark.Object)
1288 initialEventsEndBookmark = nil
1289 }
1290
1291 cacheInterval.initialEventsEndBookmark = initialEventsEndBookmark
1292 }
1293 }
1294
1295 func (c *Cacher) Ready() bool {
1296 _, err := c.ready.check()
1297 return err == nil
1298 }
1299
1300 // errWatcher implements watch.Interface to return a single error
1301 type errWatcher struct {
1302 result chan watch.Event
1303 }
1304
1305 func newErrWatcher(err error) *errWatcher {
1306 // Create an error event
1307 errEvent := watch.Event{Type: watch.Error}
1308 switch err := err.(type) {
1309 case runtime.Object:
1310 errEvent.Object = err
1311 case *errors.StatusError:
1312 errEvent.Object = &err.ErrStatus
1313 default:
1314 errEvent.Object = &metav1.Status{
1315 Status: metav1.StatusFailure,
1316 Message: err.Error(),
1317 Reason: metav1.StatusReasonInternalError,
1318 Code: http.StatusInternalServerError,
1319 }
1320 }
1321
1322 // Create a watcher with room for a single event, populate it, and close the channel
1323 watcher := &errWatcher{result: make(chan watch.Event, 1)}
1324 watcher.result <- errEvent
1325 close(watcher.result)
1326
1327 return watcher
1328 }
1329
1330 func (c *Cacher) ShouldDelegateExactRV(resourceVersion string, recursive bool) (delegator.Result, error) {
1331 // Not Recursive is not supported unitl exact RV is implemented for WaitUntilFreshAndGet.
1332 if !recursive || c.watchCache.snapshots == nil {
1333 return delegator.Result{ShouldDelegate: true}, nil
1334 }
1335 listRV, err := c.versioner.ParseResourceVersion(resourceVersion)
1336 if err != nil {
1337 return delegator.Result{}, err
1338 }
1339 return c.shouldDelegateExactRV(listRV)
1340 }
1341
1342 func (c *Cacher) ShouldDelegateContinue(continueToken string, recursive bool) (delegator.Result, error) {
1343 // Not Recursive is not supported unitl exact RV is implemented for WaitUntilFreshAndGet.
1344 if !recursive || c.watchCache.snapshots == nil {
1345 return delegator.Result{ShouldDelegate: true}, nil
1346 }
1347 _, continueRV, err := storage.DecodeContinue(continueToken, c.resourcePrefix)
1348 if err != nil {
1349 return delegator.Result{}, err
1350 }
1351 if continueRV > 0 {
1352 return c.shouldDelegateExactRV(uint64(continueRV))
1353 } else {
1354 // Continue with negative RV is a consistent read.
1355 return c.ShouldDelegateConsistentRead()
1356 }
1357 }
1358
1359 func (c *Cacher) shouldDelegateExactRV(rv uint64) (delegator.Result, error) {
1360 // Exact requests on future revision require support for consistent read, but are not a consistent read by themselves.
1361 if c.watchCache.notFresh(rv) {
1362 return delegator.Result{
1363 ShouldDelegate: !delegator.ConsistentReadSupported(),
1364 }, nil
1365 }
1366 _, canServe := c.watchCache.snapshots.GetLessOrEqual(rv)
1367 return delegator.Result{
1368 ShouldDelegate: !canServe,
1369 }, nil
1370 }
1371
1372 func (c *Cacher) ShouldDelegateConsistentRead() (delegator.Result, error) {
1373 return delegator.Result{
1374 ConsistentRead: true,
1375 ShouldDelegate: !delegator.ConsistentReadSupported(),
1376 }, nil
1377 }
1378
1379 // Implements watch.Interface.
1380 func (c *errWatcher) ResultChan() <-chan watch.Event {
1381 return c.result
1382 }
1383
1384 // Implements watch.Interface.
1385 func (c *errWatcher) Stop() {
1386 // no-op
1387 }
1388
1389 // immediateCloseWatcher implements watch.Interface that is immediately closed
1390 type immediateCloseWatcher struct {
1391 result chan watch.Event
1392 }
1393
1394 func newImmediateCloseWatcher() *immediateCloseWatcher {
1395 watcher := &immediateCloseWatcher{result: make(chan watch.Event)}
1396 close(watcher.result)
1397 return watcher
1398 }
1399
1400 // Implements watch.Interface.
1401 func (c *immediateCloseWatcher) ResultChan() <-chan watch.Event {
1402 return c.result
1403 }
1404
1405 // Implements watch.Interface.
1406 func (c *immediateCloseWatcher) Stop() {
1407 // no-op
1408 }