Debian Codegraph @

pkg/storage/cacher/cache_watcher.go

raw
1 /*
2 Copyright 2023 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package cacher
18
19 import (
20 "context"
21 "fmt"
22 "sync"
23 "time"
24
25 "k8s.io/apimachinery/pkg/runtime"
26 "k8s.io/apimachinery/pkg/runtime/schema"
27 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
28 "k8s.io/apimachinery/pkg/watch"
29 "k8s.io/apiserver/pkg/storage"
30 "k8s.io/apiserver/pkg/storage/cacher/metrics"
31 utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
32
33 "k8s.io/klog/v2"
34 )
35
36 // possible states of the cache watcher
37 const (
38 // cacheWatcherWaitingForBookmark indicates the cacher
39 // is waiting for a bookmark event with a specific RV set
40 cacheWatcherWaitingForBookmark = iota
41
42 // cacheWatcherBookmarkReceived indicates that the cacher
43 // has received a bookmark event with required RV
44 cacheWatcherBookmarkReceived
45
46 // cacheWatcherBookmarkSent indicates that the cacher
47 // has already sent a bookmark event to a client
48 cacheWatcherBookmarkSent
49 )
50
51 // cacheWatcher implements watch.Interface
52 // this is not thread-safe
53 type cacheWatcher struct {
54 input chan *watchCacheEvent
55 result chan watch.Event
56 done chan struct{}
57 filter filterWithAttrsFunc
58 stopped bool
59 forget func(bool)
60 versioner storage.Versioner
61 // The watcher will be closed by server after the deadline,
62 // save it here to send bookmark events before that.
63 deadline time.Time
64 allowWatchBookmarks bool
65 groupResource schema.GroupResource
66
67 // human readable identifier that helps assigning cacheWatcher
68 // instance with request
69 identifier string
70
71 // drainInputBuffer indicates whether we should delay closing this watcher
72 // and send all event in the input buffer.
73 drainInputBuffer bool
74
75 // bookmarkAfterResourceVersion holds an RV that indicates
76 // when we should start delivering bookmark events.
77 // If this field holds the value of 0 that means
78 // we don't have any special preferences toward delivering bookmark events.
79 // Note that this field is used in conjunction with the state field.
80 // It should not be changed once the watcher has been started.
81 bookmarkAfterResourceVersion uint64
82
83 // stateMutex protects state
84 stateMutex sync.Mutex
85
86 // state holds a numeric value indicating the current state of the watcher
87 state int
88 }
89
90 func newCacheWatcher(
91 chanSize int,
92 filter filterWithAttrsFunc,
93 forget func(bool),
94 versioner storage.Versioner,
95 deadline time.Time,
96 allowWatchBookmarks bool,
97 groupResource schema.GroupResource,
98 identifier string,
99 ) *cacheWatcher {
100 return &cacheWatcher{
101 input: make(chan *watchCacheEvent, chanSize),
102 result: make(chan watch.Event, chanSize),
103 done: make(chan struct{}),
104 filter: filter,
105 stopped: false,
106 forget: forget,
107 versioner: versioner,
108 deadline: deadline,
109 allowWatchBookmarks: allowWatchBookmarks,
110 groupResource: groupResource,
111 identifier: identifier,
112 }
113 }
114
115 // Implements watch.Interface.
116 func (c *cacheWatcher) ResultChan() <-chan watch.Event {
117 return c.result
118 }
119
120 // Implements watch.Interface.
121 func (c *cacheWatcher) Stop() {
122 c.forget(false)
123 }
124
125 // we rely on the fact that stopLocked is actually protected by Cacher.Lock()
126 func (c *cacheWatcher) stopLocked() {
127 if !c.stopped {
128 c.stopped = true
129 // stop without draining the input channel was requested.
130 if !c.drainInputBuffer {
131 close(c.done)
132 }
133 close(c.input)
134 }
135
136 // Even if the watcher was already stopped, if it previously was
137 // using draining mode and it's not using it now we need to
138 // close the done channel now. Otherwise we could leak the
139 // processing goroutine if it will be trying to put more objects
140 // into result channel, the channel will be full and there will
141 // already be noone on the processing the events on the receiving end.
142 if !c.drainInputBuffer && !c.isDoneChannelClosedLocked() {
143 close(c.done)
144 }
145 }
146
147 func (c *cacheWatcher) nonblockingAdd(event *watchCacheEvent) bool {
148 // if the bookmarkAfterResourceVersion hasn't been seen
149 // we will try to deliver a bookmark event every second.
150 // the following check will discard a bookmark event
151 // if it is < than the bookmarkAfterResourceVersion
152 // so that we don't pollute the input channel
153 if event.Type == watch.Bookmark && event.ResourceVersion < c.bookmarkAfterResourceVersion {
154 return false
155 }
156 select {
157 case c.input <- event:
158 c.markBookmarkAfterRvAsReceived(event)
159 return true
160 default:
161 return false
162 }
163 }
164
165 // Nil timer means that add will not block (if it can't send event immediately, it will break the watcher)
166 //
167 // Note that bookmark events are never added via the add method only via the nonblockingAdd.
168 // Changing this behaviour will require moving the markBookmarkAfterRvAsReceived method
169 func (c *cacheWatcher) add(event *watchCacheEvent, timer *time.Timer) bool {
170 // Try to send the event immediately, without blocking.
171 if c.nonblockingAdd(event) {
172 return true
173 }
174
175 closeFunc := func() {
176 // This means that we couldn't send event to that watcher.
177 // Since we don't want to block on it infinitely,
178 // we simply terminate it.
179 metrics.TerminatedWatchersCounter.WithLabelValues(c.groupResource.String()).Inc()
180 // This means that we couldn't send event to that watcher.
181 // Since we don't want to block on it infinitely, we simply terminate it.
182
183 // we are graceful = false, when:
184 //
185 // (a) The bookmarkAfterResourceVersionReceived hasn't been received,
186 // we can safely terminate the watcher. Because the client is waiting
187 // for this specific bookmark, and we even haven't received one.
188 // (b) We have seen the bookmarkAfterResourceVersion, and it was sent already to the client.
189 // We can simply terminate the watcher.
190
191 // we are graceful = true, when:
192 //
193 // (a) We have seen a bookmark, but it hasn't been sent to the client yet.
194 // That means we should drain the input buffer which contains
195 // the bookmarkAfterResourceVersion we want. We do that to make progress
196 // as clients can re-establish a new watch with the given RV and receive
197 // further notifications.
198 graceful := func() bool {
199 c.stateMutex.Lock()
200 defer c.stateMutex.Unlock()
201 return c.state == cacheWatcherBookmarkReceived
202 }()
203 klog.V(1).Infof("Forcing %v watcher close due to unresponsiveness: %v. len(c.input) = %v, len(c.result) = %v, graceful = %v", c.groupResource.String(), c.identifier, len(c.input), len(c.result), graceful)
204 c.forget(graceful)
205 }
206
207 if timer == nil {
208 closeFunc()
209 return false
210 }
211
212 // OK, block sending, but only until timer fires.
213 select {
214 case c.input <- event:
215 return true
216 case <-timer.C:
217 closeFunc()
218 return false
219 }
220 }
221
222 func (c *cacheWatcher) nextBookmarkTime(now time.Time, bookmarkFrequency time.Duration) (time.Time, bool) {
223 // We try to send bookmarks:
224 //
225 // (a) right before the watcher timeout - for now we simply set it 2s before
226 // the deadline
227 //
228 // (b) roughly every minute
229 //
230 // (c) immediately when the bookmarkAfterResourceVersion wasn't confirmed
231 // in this scenario the client have already seen (or is in the process of sending)
232 // all initial data and is interested in seeing
233 // a specific RV value (aka. the bookmarkAfterResourceVersion)
234 // since we don't know when the cacher will see the RV we increase frequency
235 //
236 // (b) gives us periodicity if the watch breaks due to unexpected
237 // conditions, (a) ensures that on timeout the watcher is as close to
238 // now as possible - this covers 99% of cases.
239
240 if !c.wasBookmarkAfterRvReceived() {
241 return time.Time{}, true // schedule immediately
242 }
243
244 heartbeatTime := now.Add(bookmarkFrequency)
245 if c.deadline.IsZero() {
246 // Timeout is set by our client libraries (e.g. reflector) as well as defaulted by
247 // apiserver if properly configured. So this shoudln't happen in practice.
248 return heartbeatTime, true
249 }
250 if pretimeoutTime := c.deadline.Add(-2 * time.Second); pretimeoutTime.Before(heartbeatTime) {
251 heartbeatTime = pretimeoutTime
252 }
253
254 if heartbeatTime.Before(now) {
255 return time.Time{}, false
256 }
257 return heartbeatTime, true
258 }
259
260 // wasBookmarkAfterRvReceived same as wasBookmarkAfterRvReceivedLocked just acquires a lock
261 func (c *cacheWatcher) wasBookmarkAfterRvReceived() bool {
262 c.stateMutex.Lock()
263 defer c.stateMutex.Unlock()
264 return c.wasBookmarkAfterRvReceivedLocked()
265 }
266
267 // wasBookmarkAfterRvReceivedLocked checks if the given cacheWatcher
268 // have seen a bookmark event >= bookmarkAfterResourceVersion
269 func (c *cacheWatcher) wasBookmarkAfterRvReceivedLocked() bool {
270 return c.state != cacheWatcherWaitingForBookmark
271 }
272
273 // markBookmarkAfterRvAsReceived indicates that the given cacheWatcher
274 // have seen a bookmark event >= bookmarkAfterResourceVersion
275 func (c *cacheWatcher) markBookmarkAfterRvAsReceived(event *watchCacheEvent) {
276 if event.Type == watch.Bookmark {
277 c.stateMutex.Lock()
278 defer c.stateMutex.Unlock()
279 if c.wasBookmarkAfterRvReceivedLocked() {
280 return
281 }
282 // bookmark events are scheduled by startDispatchingBookmarkEvents method
283 // since we received a bookmark event that means we have
284 // converged towards the expected RV and it is okay to update the state so that
285 // this cacher can be scheduler for a regular bookmark events
286 c.state = cacheWatcherBookmarkReceived
287 }
288 }
289
290 // wasBookmarkAfterRvSentLocked checks if a bookmark event
291 // with an RV >= the bookmarkAfterResourceVersion has been sent by this watcher
292 func (c *cacheWatcher) wasBookmarkAfterRvSentLocked() bool {
293 return c.state == cacheWatcherBookmarkSent
294 }
295
296 // wasBookmarkAfterRvSent same as wasBookmarkAfterRvSentLocked just acquires a lock
297 func (c *cacheWatcher) wasBookmarkAfterRvSent() bool {
298 c.stateMutex.Lock()
299 defer c.stateMutex.Unlock()
300 return c.wasBookmarkAfterRvSentLocked()
301 }
302
303 // markBookmarkAfterRvSent indicates that the given cacheWatcher
304 // have sent a bookmark event with an RV >= the bookmarkAfterResourceVersion
305 //
306 // this function relies on the fact that the nonblockingAdd method
307 // won't admit a bookmark event with an RV < the bookmarkAfterResourceVersion
308 // so the first received bookmark event is considered to match the bookmarkAfterResourceVersion
309 func (c *cacheWatcher) markBookmarkAfterRvSent(event *watchCacheEvent) {
310 // note that bookmark events are not so common so will acquire a lock every ~60 second or so
311 if event.Type == watch.Bookmark {
312 c.stateMutex.Lock()
313 defer c.stateMutex.Unlock()
314 if !c.wasBookmarkAfterRvSentLocked() {
315 c.state = cacheWatcherBookmarkSent
316 }
317 }
318 }
319
320 // setBookmarkAfterResourceVersion sets the bookmarkAfterResourceVersion and the state associated with it
321 func (c *cacheWatcher) setBookmarkAfterResourceVersion(bookmarkAfterResourceVersion uint64) {
322 state := cacheWatcherWaitingForBookmark
323 if bookmarkAfterResourceVersion == 0 {
324 state = cacheWatcherBookmarkSent // if no specific RV was requested we assume no-op
325 }
326 c.state = state
327 c.bookmarkAfterResourceVersion = bookmarkAfterResourceVersion
328 }
329
330 // setDrainInputBufferLocked if set to true indicates that we should delay closing this watcher
331 // until we send all events residing in the input buffer.
332 func (c *cacheWatcher) setDrainInputBufferLocked(drain bool) {
333 c.drainInputBuffer = drain
334 }
335
336 // isDoneChannelClosed checks if c.done channel is closed
337 func (c *cacheWatcher) isDoneChannelClosedLocked() bool {
338 select {
339 case <-c.done:
340 return true
341 default:
342 }
343 return false
344 }
345
346 func getMutableObject(object runtime.Object) runtime.Object {
347 if _, ok := object.(*cachingObject); ok {
348 // It is safe to return without deep-copy, because the underlying
349 // object will lazily perform deep-copy on the first try to change
350 // any of its fields.
351 return object
352 }
353 return object.DeepCopyObject()
354 }
355
356 func updateResourceVersion(object runtime.Object, versioner storage.Versioner, resourceVersion uint64) {
357 if err := versioner.UpdateObject(object, resourceVersion); err != nil {
358 utilruntime.HandleError(fmt.Errorf("failure to version api object (%d) %#v: %v", resourceVersion, object, err))
359 }
360 }
361
362 func (c *cacheWatcher) convertToWatchEvent(event *watchCacheEvent) *watch.Event {
363 if event.Type == watch.Bookmark {
364 e := &watch.Event{Type: watch.Bookmark, Object: event.Object.DeepCopyObject()}
365 if !c.wasBookmarkAfterRvSent() {
366 if err := storage.AnnotateInitialEventsEndBookmark(e.Object); err != nil {
367 utilruntime.HandleError(fmt.Errorf("error while accessing object's metadata gr: %v, identifier: %v, obj: %#v, err: %v", c.groupResource, c.identifier, e.Object, err))
368 return nil
369 }
370 }
371 return e
372 }
373
374 curObjPasses := event.Type != watch.Deleted && c.filter(event.Key, event.ObjLabels, event.ObjFields)
375 oldObjPasses := false
376 if event.PrevObject != nil {
377 oldObjPasses = c.filter(event.Key, event.PrevObjLabels, event.PrevObjFields)
378 }
379 if !curObjPasses && !oldObjPasses {
380 // Watcher is not interested in that object.
381 return nil
382 }
383
384 switch {
385 case curObjPasses && !oldObjPasses:
386 return &watch.Event{Type: watch.Added, Object: getMutableObject(event.Object)}
387 case curObjPasses && oldObjPasses:
388 return &watch.Event{Type: watch.Modified, Object: getMutableObject(event.Object)}
389 case !curObjPasses && oldObjPasses:
390 // return a delete event with the previous object content, but with the event's resource version
391 oldObj := getMutableObject(event.PrevObject)
392 // We know that if oldObj is cachingObject (which can only be set via
393 // setCachingObjects), its resourceVersion is already set correctly and
394 // we don't need to update it. However, since cachingObject efficiently
395 // handles noop updates, we avoid this microoptimization here.
396 updateResourceVersion(oldObj, c.versioner, event.ResourceVersion)
397 return &watch.Event{Type: watch.Deleted, Object: oldObj}
398 }
399
400 return nil
401 }
402
403 // NOTE: sendWatchCacheEvent is assumed to not modify <event> !!!
404 func (c *cacheWatcher) sendWatchCacheEvent(event *watchCacheEvent) {
405 watchEvent := c.convertToWatchEvent(event)
406 if watchEvent == nil {
407 // Watcher is not interested in that object.
408 return
409 }
410
411 // We need to ensure that if we put event X to the c.result, all
412 // previous events were already put into it before, no matter whether
413 // c.done is close or not.
414 // Thus we cannot simply select from c.done and c.result and this
415 // would give us non-determinism.
416 // At the same time, we don't want to block infinitely on putting
417 // to c.result, when c.done is already closed.
418 //
419 // This ensures that with c.done already close, we at most once go
420 // into the next select after this. With that, no matter which
421 // statement we choose there, we will deliver only consecutive
422 // events.
423 select {
424 case <-c.done:
425 return
426 default:
427 }
428
429 select {
430 case c.result <- *watchEvent:
431 c.markBookmarkAfterRvSent(event)
432 case <-c.done:
433 }
434 }
435
436 func (c *cacheWatcher) processInterval(ctx context.Context, cacheInterval *watchCacheInterval, resourceVersion uint64) {
437 defer utilruntime.HandleCrash()
438 defer close(c.result)
439 defer c.Stop()
440
441 // Check how long we are processing initEvents.
442 // As long as these are not processed, we are not processing
443 // any incoming events, so if it takes long, we may actually
444 // block all watchers for some time.
445 // TODO: From the logs it seems that there happens processing
446 // times even up to 1s which is very long. However, this doesn't
447 // depend that much on the number of initEvents. E.g. from the
448 // 2000-node Kubemark run we have logs like this, e.g.:
449 // ... processing 13862 initEvents took 66.808689ms
450 // ... processing 14040 initEvents took 993.532539ms
451 // We should understand what is blocking us in those cases (e.g.
452 // is it lack of CPU, network, or sth else) and potentially
453 // consider increase size of result buffer in those cases.
454 const initProcessThreshold = 500 * time.Millisecond
455 startTime := time.Now()
456
457 // cacheInterval may be created from a version being more fresh than requested
458 // (e.g. for NotOlderThan semantic). In such a case, we need to prevent watch event
459 // with lower resourceVersion from being delivered to ensure watch contract.
460 if cacheInterval.resourceVersion > resourceVersion {
461 resourceVersion = cacheInterval.resourceVersion
462 }
463
464 initEventCount := 0
465 for {
466 event, err := cacheInterval.Next()
467 if err != nil {
468 // An error indicates that the cache interval
469 // has been invalidated and can no longer serve
470 // events.
471 //
472 // Initially we considered sending an "out-of-history"
473 // Error event in this case, but because historically
474 // such events weren't sent out of the watchCache, we
475 // decided not to. This is still ok, because on watch
476 // closure, the watcher will try to re-instantiate the
477 // watch and then will get an explicit "out-of-history"
478 // window. There is potential for optimization, but for
479 // now, in order to be on the safe side and not break
480 // custom clients, the cost of it is something that we
481 // are fully accepting.
482 klog.Warningf("couldn't retrieve watch event to serve: %#v", err)
483 return
484 }
485 if event == nil {
486 break
487 }
488 c.sendWatchCacheEvent(event)
489
490 // With some events already sent, update resourceVersion so that
491 // events that were buffered and not yet processed won't be delivered
492 // to this watcher second time causing going back in time.
493 //
494 // There is one case where events are not necessary ordered by
495 // resourceVersion, being a case of watching from resourceVersion=0,
496 // which at the beginning returns the state of each objects.
497 // For the purpose of it, we need to max it with the resource version
498 // that we have so far.
499 if event.ResourceVersion > resourceVersion {
500 resourceVersion = event.ResourceVersion
501 }
502 initEventCount++
503 }
504
505 if initEventCount > 0 {
506 metrics.InitCounter.WithLabelValues(c.groupResource.String()).Add(float64(initEventCount))
507 }
508 processingTime := time.Since(startTime)
509 if processingTime > initProcessThreshold {
510 klog.V(2).Infof("processing %d initEvents of %s (%s) took %v", initEventCount, c.groupResource, c.identifier, processingTime)
511 }
512
513 // send bookmark after sending all events in cacheInterval for watchlist request
514 if cacheInterval.initialEventsEndBookmark != nil {
515 c.sendWatchCacheEvent(cacheInterval.initialEventsEndBookmark)
516 }
517 c.process(ctx, resourceVersion)
518 }
519
520 func (c *cacheWatcher) process(ctx context.Context, resourceVersion uint64) {
521 // At this point we already start processing incoming watch events.
522 // However, the init event can still be processed because their serialization
523 // and sending to the client happens asynchrnously.
524 // TODO: As describe in the KEP, we would like to estimate that by delaying
525 // the initialization signal proportionally to the number of events to
526 // process, but we're leaving this to the tuning phase.
527 utilflowcontrol.WatchInitialized(ctx)
528
529 for {
530 select {
531 case event, ok := <-c.input:
532 if !ok {
533 return
534 }
535 // only send events newer than resourceVersion
536 // or a bookmark event with an RV equal to resourceVersion
537 // if we haven't sent one to the client
538 if event.ResourceVersion > resourceVersion || (event.Type == watch.Bookmark && event.ResourceVersion == resourceVersion && !c.wasBookmarkAfterRvSent()) {
539 c.sendWatchCacheEvent(event)
540 }
541 case <-ctx.Done():
542 return
543 }
544 }
545 }