| 1 |
/* |
| 2 |
Copyright 2015 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package cacher |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"net/http" |
| 23 |
"reflect" |
| 24 |
"strings" |
| 25 |
"sync" |
| 26 |
"time" |
| 27 |
|
| 28 |
"go.opentelemetry.io/otel/attribute" |
| 29 |
"google.golang.org/grpc/metadata" |
| 30 |
|
| 31 |
"k8s.io/apimachinery/pkg/api/errors" |
| 32 |
"k8s.io/apimachinery/pkg/api/meta" |
| 33 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 34 |
"k8s.io/apimachinery/pkg/conversion" |
| 35 |
"k8s.io/apimachinery/pkg/fields" |
| 36 |
"k8s.io/apimachinery/pkg/labels" |
| 37 |
"k8s.io/apimachinery/pkg/runtime" |
| 38 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 39 |
"k8s.io/apimachinery/pkg/util/wait" |
| 40 |
"k8s.io/apimachinery/pkg/watch" |
| 41 |
"k8s.io/apiserver/pkg/audit" |
| 42 |
"k8s.io/apiserver/pkg/endpoints/request" |
| 43 |
"k8s.io/apiserver/pkg/features" |
| 44 |
"k8s.io/apiserver/pkg/storage" |
| 45 |
"k8s.io/apiserver/pkg/storage/cacher/delegator" |
| 46 |
"k8s.io/apiserver/pkg/storage/cacher/metrics" |
| 47 |
"k8s.io/apiserver/pkg/storage/cacher/progress" |
| 48 |
etcdfeature "k8s.io/apiserver/pkg/storage/feature" |
| 49 |
utilfeature "k8s.io/apiserver/pkg/util/feature" |
| 50 |
"k8s.io/client-go/tools/cache" |
| 51 |
"k8s.io/component-base/tracing" |
| 52 |
"k8s.io/klog/v2" |
| 53 |
"k8s.io/utils/clock" |
| 54 |
"k8s.io/utils/ptr" |
| 55 |
) |
| 56 |
|
| 57 |
var ( |
| 58 |
emptyFunc = func(bool) {} |
| 59 |
coreNamespaceResource = schema.GroupResource{Group: "", Resource: "namespaces"} |
| 60 |
) |
| 61 |
|
| 62 |
const ( |
| 63 |
// storageWatchListPageSize is the cacher's request chunk size of |
| 64 |
// initial and resync watch lists to storage. |
| 65 |
storageWatchListPageSize = int64(10000) |
| 66 |
|
| 67 |
// DefaultEventFreshDuration is the default time duration of events |
| 68 |
// we want to keep. |
| 69 |
// We set it to defaultBookmarkFrequency plus epsilon to maximize |
| 70 |
// chances that last bookmark was sent within kept history, at the |
| 71 |
// same time, minimizing the needed memory usage. |
| 72 |
DefaultEventFreshDuration = defaultBookmarkFrequency + 15*time.Second |
| 73 |
|
| 74 |
// defaultBookmarkFrequency defines how frequently watch bookmarks should be send |
| 75 |
// in addition to sending a bookmark right before watch deadline. |
| 76 |
defaultBookmarkFrequency = time.Minute |
| 77 |
) |
| 78 |
|
| 79 |
// Config contains the configuration for a given Cache. |
| 80 |
type Config struct { |
| 81 |
// An underlying storage.Interface. |
| 82 |
Storage storage.Interface |
| 83 |
|
| 84 |
// An underlying storage.Versioner. |
| 85 |
Versioner storage.Versioner |
| 86 |
|
| 87 |
// The GroupResource the cacher is caching. Used for disambiguating *unstructured.Unstructured (CRDs) in logging |
| 88 |
// and metrics. |
| 89 |
GroupResource schema.GroupResource |
| 90 |
|
| 91 |
// EventsHistoryWindow specifies minimum history duration that storage is keeping. |
| 92 |
// If lower than DefaultEventFreshDuration, the cache creation will fail. |
| 93 |
EventsHistoryWindow time.Duration |
| 94 |
|
| 95 |
// The Cache will be caching objects of a given Type and assumes that they |
| 96 |
// are all stored under ResourcePrefix directory in the underlying database. |
| 97 |
ResourcePrefix string |
| 98 |
|
| 99 |
// KeyFunc is used to get a key in the underlying storage for a given object. |
| 100 |
KeyFunc func(runtime.Object) (string, error) |
| 101 |
|
| 102 |
// GetAttrsFunc is used to get object labels, fields |
| 103 |
GetAttrsFunc func(runtime.Object) (label labels.Set, field fields.Set, err error) |
| 104 |
|
| 105 |
// IndexerFuncs is used for optimizing amount of watchers that |
| 106 |
// needs to process an incoming event. |
| 107 |
IndexerFuncs storage.IndexerFuncs |
| 108 |
|
| 109 |
// Indexers is used to accelerate the list operation, falls back to regular list |
| 110 |
// operation if no indexer found. |
| 111 |
Indexers *cache.Indexers |
| 112 |
|
| 113 |
// NewFunc is a function that creates new empty object storing a object of type Type. |
| 114 |
NewFunc func() runtime.Object |
| 115 |
|
| 116 |
// NewList is a function that creates new empty object storing a list of |
| 117 |
// objects of type Type. |
| 118 |
NewListFunc func() runtime.Object |
| 119 |
|
| 120 |
Codec runtime.Codec |
| 121 |
|
| 122 |
Clock clock.WithTicker |
| 123 |
} |
| 124 |
|
| 125 |
type watchersMap map[int]*cacheWatcher |
| 126 |
|
| 127 |
func (wm watchersMap) addWatcher(w *cacheWatcher, number int) { |
| 128 |
wm[number] = w |
| 129 |
} |
| 130 |
|
| 131 |
func (wm watchersMap) deleteWatcher(number int) { |
| 132 |
delete(wm, number) |
| 133 |
} |
| 134 |
|
| 135 |
func (wm watchersMap) terminateAll(done func(*cacheWatcher)) { |
| 136 |
for key, watcher := range wm { |
| 137 |
delete(wm, key) |
| 138 |
done(watcher) |
| 139 |
} |
| 140 |
} |
| 141 |
|
| 142 |
type indexedWatchers struct { |
| 143 |
allWatchers map[namespacedName]watchersMap |
| 144 |
valueWatchers map[string]watchersMap |
| 145 |
} |
| 146 |
|
| 147 |
func (i *indexedWatchers) addWatcher(w *cacheWatcher, number int, scope namespacedName, value string, supported bool) { |
| 148 |
if supported { |
| 149 |
if _, ok := i.valueWatchers[value]; !ok { |
| 150 |
i.valueWatchers[value] = watchersMap{} |
| 151 |
} |
| 152 |
i.valueWatchers[value].addWatcher(w, number) |
| 153 |
} else { |
| 154 |
scopedWatchers, ok := i.allWatchers[scope] |
| 155 |
if !ok { |
| 156 |
scopedWatchers = watchersMap{} |
| 157 |
i.allWatchers[scope] = scopedWatchers |
| 158 |
} |
| 159 |
scopedWatchers.addWatcher(w, number) |
| 160 |
} |
| 161 |
} |
| 162 |
|
| 163 |
func (i *indexedWatchers) deleteWatcher(number int, scope namespacedName, value string, supported bool) { |
| 164 |
if supported { |
| 165 |
i.valueWatchers[value].deleteWatcher(number) |
| 166 |
if len(i.valueWatchers[value]) == 0 { |
| 167 |
delete(i.valueWatchers, value) |
| 168 |
} |
| 169 |
} else { |
| 170 |
i.allWatchers[scope].deleteWatcher(number) |
| 171 |
if len(i.allWatchers[scope]) == 0 { |
| 172 |
delete(i.allWatchers, scope) |
| 173 |
} |
| 174 |
} |
| 175 |
} |
| 176 |
|
| 177 |
func (i *indexedWatchers) terminateAll(groupResource schema.GroupResource, done func(*cacheWatcher)) { |
| 178 |
// note that we don't have to call setDrainInputBufferLocked method on the watchers |
| 179 |
// because we take advantage of the default value - stop immediately |
| 180 |
// also watchers that have had already its draining strategy set |
| 181 |
// are no longer available (they were removed from the allWatchers and the valueWatchers maps) |
| 182 |
if len(i.allWatchers) > 0 || len(i.valueWatchers) > 0 { |
| 183 |
klog.Warningf("Terminating all watchers from cacher %v", groupResource) |
| 184 |
} |
| 185 |
for _, watchers := range i.allWatchers { |
| 186 |
watchers.terminateAll(done) |
| 187 |
} |
| 188 |
for _, watchers := range i.valueWatchers { |
| 189 |
watchers.terminateAll(done) |
| 190 |
} |
| 191 |
i.allWatchers = map[namespacedName]watchersMap{} |
| 192 |
i.valueWatchers = map[string]watchersMap{} |
| 193 |
} |
| 194 |
|
| 195 |
// As we don't need a high precision here, we keep all watchers timeout within a |
| 196 |
// second in a bucket, and pop up them once at the timeout. To be more specific, |
| 197 |
// if you set fire time at X, you can get the bookmark within (X-1,X+1) period. |
| 198 |
type watcherBookmarkTimeBuckets struct { |
| 199 |
// the key of watcherBuckets is the number of seconds since createTime |
| 200 |
watchersBuckets map[int64][]*cacheWatcher |
| 201 |
createTime time.Time |
| 202 |
startBucketID int64 |
| 203 |
clock clock.Clock |
| 204 |
bookmarkFrequency time.Duration |
| 205 |
} |
| 206 |
|
| 207 |
func newTimeBucketWatchers(clock clock.Clock, bookmarkFrequency time.Duration) *watcherBookmarkTimeBuckets { |
| 208 |
return &watcherBookmarkTimeBuckets{ |
| 209 |
watchersBuckets: make(map[int64][]*cacheWatcher), |
| 210 |
createTime: clock.Now(), |
| 211 |
startBucketID: 0, |
| 212 |
clock: clock, |
| 213 |
bookmarkFrequency: bookmarkFrequency, |
| 214 |
} |
| 215 |
} |
| 216 |
|
| 217 |
// adds a watcher to the bucket, if the deadline is before the start, it will be |
| 218 |
// added to the first one. |
| 219 |
func (t *watcherBookmarkTimeBuckets) addWatcherThreadUnsafe(w *cacheWatcher) bool { |
| 220 |
// note that the returned time can be before t.createTime, |
| 221 |
// especially in cases when the nextBookmarkTime method |
| 222 |
// give us the zero value of type Time |
| 223 |
// so buckedID can hold a negative value |
| 224 |
nextTime, ok := w.nextBookmarkTime(t.clock.Now(), t.bookmarkFrequency) |
| 225 |
if !ok { |
| 226 |
return false |
| 227 |
} |
| 228 |
bucketID := int64(nextTime.Sub(t.createTime) / time.Second) |
| 229 |
if bucketID < t.startBucketID { |
| 230 |
bucketID = t.startBucketID |
| 231 |
} |
| 232 |
watchers := t.watchersBuckets[bucketID] |
| 233 |
t.watchersBuckets[bucketID] = append(watchers, w) |
| 234 |
return true |
| 235 |
} |
| 236 |
|
| 237 |
func (t *watcherBookmarkTimeBuckets) popExpiredWatchersThreadUnsafe() [][]*cacheWatcher { |
| 238 |
currentBucketID := int64(t.clock.Since(t.createTime) / time.Second) |
| 239 |
// There should be one or two elements in almost all cases |
| 240 |
expiredWatchers := make([][]*cacheWatcher, 0, 2) |
| 241 |
for ; t.startBucketID <= currentBucketID; t.startBucketID++ { |
| 242 |
if watchers, ok := t.watchersBuckets[t.startBucketID]; ok { |
| 243 |
delete(t.watchersBuckets, t.startBucketID) |
| 244 |
expiredWatchers = append(expiredWatchers, watchers) |
| 245 |
} |
| 246 |
} |
| 247 |
return expiredWatchers |
| 248 |
} |
| 249 |
|
| 250 |
type filterWithAttrsFunc func(key string, l labels.Set, f fields.Set) bool |
| 251 |
|
| 252 |
type indexedTriggerFunc struct { |
| 253 |
indexName string |
| 254 |
indexerFunc storage.IndexerFunc |
| 255 |
} |
| 256 |
|
| 257 |
// Cacher is responsible for serving WATCH and LIST requests for a given |
| 258 |
// resource from its internal cache and updating its cache in the background |
| 259 |
// based on the underlying storage contents. |
| 260 |
// Cacher implements storage.Interface (although most of the calls are just |
| 261 |
// delegated to the underlying storage). |
| 262 |
type Cacher struct { |
| 263 |
// HighWaterMarks for performance debugging. |
| 264 |
// Important: Since HighWaterMark is using sync/atomic, it has to be at the top of the struct due to a bug on 32-bit platforms |
| 265 |
// See: https://golang.org/pkg/sync/atomic/ for more information |
| 266 |
incomingHWM storage.HighWaterMark |
| 267 |
// Incoming events that should be dispatched to watchers. |
| 268 |
incoming chan watchCacheEvent |
| 269 |
|
| 270 |
resourcePrefix string |
| 271 |
|
| 272 |
sync.RWMutex |
| 273 |
|
| 274 |
// Before accessing the cacher's cache, wait for the ready to be ok. |
| 275 |
// This is necessary to prevent users from accessing structures that are |
| 276 |
// uninitialized or are being repopulated right now. |
| 277 |
// ready needs to be set to false when the cacher is paused or stopped. |
| 278 |
// ready needs to be set to true when the cacher is ready to use after |
| 279 |
// initialization. |
| 280 |
ready *ready |
| 281 |
|
| 282 |
// Underlying storage.Interface. |
| 283 |
storage storage.Interface |
| 284 |
|
| 285 |
// Expected type of objects in the underlying cache. |
| 286 |
objectType reflect.Type |
| 287 |
// Used for logging, to disambiguate *unstructured.Unstructured (CRDs) |
| 288 |
groupResource schema.GroupResource |
| 289 |
|
| 290 |
// "sliding window" of recent changes of objects and the current state. |
| 291 |
watchCache *watchCache |
| 292 |
reflector *cache.Reflector |
| 293 |
|
| 294 |
// Versioner is used to handle resource versions. |
| 295 |
versioner storage.Versioner |
| 296 |
|
| 297 |
// newFunc is a function that creates new empty object storing a object of type Type. |
| 298 |
newFunc func() runtime.Object |
| 299 |
|
| 300 |
// newListFunc is a function that creates new empty list for storing objects of type Type. |
| 301 |
newListFunc func() runtime.Object |
| 302 |
|
| 303 |
// indexedTrigger is used for optimizing amount of watchers that needs to process |
| 304 |
// an incoming event. |
| 305 |
indexedTrigger *indexedTriggerFunc |
| 306 |
// watchers is mapping from the value of trigger function that a |
| 307 |
// watcher is interested into the watchers |
| 308 |
watcherIdx int |
| 309 |
watchers indexedWatchers |
| 310 |
|
| 311 |
// Defines a time budget that can be spend on waiting for not-ready watchers |
| 312 |
// while dispatching event before shutting them down. |
| 313 |
dispatchTimeoutBudget timeBudget |
| 314 |
|
| 315 |
// Handling graceful termination. |
| 316 |
stopLock sync.RWMutex |
| 317 |
stopped bool |
| 318 |
stopCh chan struct{} |
| 319 |
stopWg sync.WaitGroup |
| 320 |
|
| 321 |
clock clock.Clock |
| 322 |
// timer is used to avoid unnecessary allocations in underlying watchers. |
| 323 |
timer *time.Timer |
| 324 |
|
| 325 |
// dispatching determines whether there is currently dispatching of |
| 326 |
// any event in flight. |
| 327 |
dispatching bool |
| 328 |
// watchersBuffer is a list of watchers potentially interested in currently |
| 329 |
// dispatched event. |
| 330 |
watchersBuffer []*cacheWatcher |
| 331 |
// blockedWatchers is a list of watchers whose buffer is currently full. |
| 332 |
blockedWatchers []*cacheWatcher |
| 333 |
// watchersToStop is a list of watchers that were supposed to be stopped |
| 334 |
// during current dispatching, but stopping was deferred to the end of |
| 335 |
// dispatching that event to avoid race with closing channels in watchers. |
| 336 |
watchersToStop []*cacheWatcher |
| 337 |
// Maintain a timeout queue to send the bookmark event before the watcher times out. |
| 338 |
// Note that this field when accessed MUST be protected by the Cacher.lock. |
| 339 |
bookmarkWatchers *watcherBookmarkTimeBuckets |
| 340 |
// expiredBookmarkWatchers is a list of watchers that were expired and need to be schedule for a next bookmark event |
| 341 |
expiredBookmarkWatchers []*cacheWatcher |
| 342 |
} |
| 343 |
|
| 344 |
// NewCacherFromConfig creates a new Cacher responsible for servicing WATCH and LIST requests from |
| 345 |
// its internal cache and updating its cache in the background based on the |
| 346 |
// given configuration. |
| 347 |
func NewCacherFromConfig(config Config) (*Cacher, error) { |
| 348 |
stopCh := make(chan struct{}) |
| 349 |
obj := config.NewFunc() |
| 350 |
// Give this error when it is constructed rather than when you get the |
| 351 |
// first watch item, because it's much easier to track down that way. |
| 352 |
if err := runtime.CheckCodec(config.Codec, obj); err != nil { |
| 353 |
return nil, fmt.Errorf("storage codec doesn't seem to match given type: %v", err) |
| 354 |
} |
| 355 |
|
| 356 |
var indexedTrigger *indexedTriggerFunc |
| 357 |
if config.IndexerFuncs != nil { |
| 358 |
// For now, we don't support multiple trigger functions defined |
| 359 |
// for a given resource. |
| 360 |
if len(config.IndexerFuncs) > 1 { |
| 361 |
return nil, fmt.Errorf("cacher %s doesn't support more than one IndexerFunc: ", reflect.TypeOf(obj).String()) |
| 362 |
} |
| 363 |
for key, value := range config.IndexerFuncs { |
| 364 |
if value != nil { |
| 365 |
indexedTrigger = &indexedTriggerFunc{ |
| 366 |
indexName: key, |
| 367 |
indexerFunc: value, |
| 368 |
} |
| 369 |
} |
| 370 |
} |
| 371 |
} |
| 372 |
|
| 373 |
if config.Clock == nil { |
| 374 |
config.Clock = clock.RealClock{} |
| 375 |
} |
| 376 |
objType := reflect.TypeOf(obj) |
| 377 |
cacher := &Cacher{ |
| 378 |
resourcePrefix: config.ResourcePrefix, |
| 379 |
ready: newReady(config.Clock), |
| 380 |
storage: config.Storage, |
| 381 |
objectType: objType, |
| 382 |
groupResource: config.GroupResource, |
| 383 |
versioner: config.Versioner, |
| 384 |
newFunc: config.NewFunc, |
| 385 |
newListFunc: config.NewListFunc, |
| 386 |
indexedTrigger: indexedTrigger, |
| 387 |
watcherIdx: 0, |
| 388 |
watchers: indexedWatchers{ |
| 389 |
allWatchers: make(map[namespacedName]watchersMap), |
| 390 |
valueWatchers: make(map[string]watchersMap), |
| 391 |
}, |
| 392 |
// TODO: Figure out the correct value for the buffer size. |
| 393 |
incoming: make(chan watchCacheEvent, 100), |
| 394 |
dispatchTimeoutBudget: newTimeBudget(), |
| 395 |
// We need to (potentially) stop both: |
| 396 |
// - wait.Until go-routine |
| 397 |
// - reflector.ListAndWatch |
| 398 |
// and there are no guarantees on the order that they will stop. |
| 399 |
// So we will be simply closing the channel, and synchronizing on the WaitGroup. |
| 400 |
stopCh: stopCh, |
| 401 |
clock: config.Clock, |
| 402 |
timer: time.NewTimer(time.Duration(0)), |
| 403 |
bookmarkWatchers: newTimeBucketWatchers(config.Clock, defaultBookmarkFrequency), |
| 404 |
} |
| 405 |
|
| 406 |
// Ensure that timer is stopped. |
| 407 |
if !cacher.timer.Stop() { |
| 408 |
// Consume triggered (but not yet received) timer event |
| 409 |
// so that future reuse does not get a spurious timeout. |
| 410 |
<-cacher.timer.C |
| 411 |
} |
| 412 |
var contextMetadata metadata.MD |
| 413 |
if utilfeature.DefaultFeatureGate.Enabled(features.SeparateCacheWatchRPC) { |
| 414 |
// Add grpc context metadata to watch and progress notify requests done by cacher to: |
| 415 |
// * Prevent starvation of watch opened by cacher, by moving it to separate Watch RPC than watch request that bypass cacher. |
| 416 |
// * Ensure that progress notification requests are executed on the same Watch RPC as their watch, which is required for it to work. |
| 417 |
contextMetadata = metadata.New(map[string]string{"source": "cache"}) |
| 418 |
} |
| 419 |
|
| 420 |
eventFreshDuration := config.EventsHistoryWindow |
| 421 |
if eventFreshDuration < DefaultEventFreshDuration { |
| 422 |
return nil, fmt.Errorf("config.EventsHistoryWindow (%v) must not be below %v", eventFreshDuration, DefaultEventFreshDuration) |
| 423 |
} |
| 424 |
|
| 425 |
progressRequester := progress.NewConditionalProgressRequester(config.Storage.RequestWatchProgress, config.Clock, contextMetadata) |
| 426 |
watchCache := newWatchCache( |
| 427 |
config.KeyFunc, cacher.processEvent, config.GetAttrsFunc, config.Versioner, config.Indexers, |
| 428 |
config.Clock, eventFreshDuration, config.GroupResource, progressRequester) |
| 429 |
listerWatcher := NewListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc, contextMetadata) |
| 430 |
reflectorName := "storage/cacher.go:" + config.ResourcePrefix |
| 431 |
|
| 432 |
reflector := cache.NewNamedReflector(reflectorName, listerWatcher, obj, watchCache, 0) |
| 433 |
// Configure reflector's pager to for an appropriate pagination chunk size for fetching data from |
| 434 |
// storage. The pager falls back to full list if paginated list calls fail due to an "Expired" error. |
| 435 |
reflector.WatchListPageSize = storageWatchListPageSize |
| 436 |
// When etcd loses leader for 3 cycles, it returns error "no leader". |
| 437 |
// We don't want to terminate all watchers as recreating all watchers puts high load on api-server. |
| 438 |
// In most of the cases, leader is reelected within few cycles. |
| 439 |
reflector.MaxInternalErrorRetryDuration = time.Second * 30 |
| 440 |
// since the watch-list is provided by the watch cache instruct |
| 441 |
// the reflector to issue a regular LIST against the store |
| 442 |
reflector.UseWatchList = ptr.To(false) |
| 443 |
|
| 444 |
cacher.watchCache = watchCache |
| 445 |
cacher.reflector = reflector |
| 446 |
|
| 447 |
go cacher.dispatchEvents() |
| 448 |
go progressRequester.Run(stopCh) |
| 449 |
|
| 450 |
cacher.stopWg.Add(1) |
| 451 |
go func() { |
| 452 |
defer cacher.stopWg.Done() |
| 453 |
defer cacher.terminateAllWatchers() |
| 454 |
wait.Until( |
| 455 |
func() { |
| 456 |
if !cacher.isStopped() { |
| 457 |
cacher.startCaching(stopCh) |
| 458 |
} |
| 459 |
}, time.Second, stopCh, |
| 460 |
) |
| 461 |
}() |
| 462 |
|
| 463 |
return cacher, nil |
| 464 |
} |
| 465 |
|
| 466 |
func (c *Cacher) startCaching(stopChannel <-chan struct{}) { |
| 467 |
c.watchCache.SetOnReplace(func() { |
| 468 |
c.ready.setReady() |
| 469 |
klog.V(1).Infof("cacher (%v): initialized", c.groupResource.String()) |
| 470 |
metrics.WatchCacheInitializations.WithLabelValues(c.groupResource.String()).Inc() |
| 471 |
}) |
| 472 |
var err error |
| 473 |
defer func() { |
| 474 |
c.ready.setError(err) |
| 475 |
}() |
| 476 |
|
| 477 |
c.terminateAllWatchers() |
| 478 |
err = c.reflector.ListAndWatch(stopChannel) |
| 479 |
if err != nil { |
| 480 |
klog.Errorf("cacher (%v): unexpected ListAndWatch error: %v; reinitializing...", c.groupResource.String(), err) |
| 481 |
} |
| 482 |
} |
| 483 |
|
| 484 |
type namespacedName struct { |
| 485 |
namespace string |
| 486 |
name string |
| 487 |
} |
| 488 |
|
| 489 |
func (c *Cacher) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) { |
| 490 |
pred := opts.Predicate |
| 491 |
requestedWatchRV, err := c.versioner.ParseResourceVersion(opts.ResourceVersion) |
| 492 |
if err != nil { |
| 493 |
return nil, err |
| 494 |
} |
| 495 |
|
| 496 |
var readyGeneration int |
| 497 |
if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { |
| 498 |
var err error |
| 499 |
var downtime time.Duration |
| 500 |
readyGeneration, downtime, err = c.ready.checkAndReadGeneration() |
| 501 |
if err != nil { |
| 502 |
return nil, errors.NewTooManyRequests(err.Error(), calculateRetryAfterForUnreadyCache(downtime)) |
| 503 |
} |
| 504 |
} else { |
| 505 |
readyGeneration, err = c.ready.waitAndReadGeneration(ctx) |
| 506 |
if err != nil { |
| 507 |
return nil, errors.NewServiceUnavailable(err.Error()) |
| 508 |
} |
| 509 |
} |
| 510 |
|
| 511 |
// determine the namespace and name scope of the watch, first from the request, secondarily from the field selector |
| 512 |
scope := namespacedName{} |
| 513 |
if requestNamespace, ok := request.NamespaceFrom(ctx); ok && len(requestNamespace) > 0 { |
| 514 |
scope.namespace = requestNamespace |
| 515 |
} else if selectorNamespace, ok := pred.Field.RequiresExactMatch("metadata.namespace"); ok { |
| 516 |
scope.namespace = selectorNamespace |
| 517 |
} |
| 518 |
if requestInfo, ok := request.RequestInfoFrom(ctx); ok && requestInfo != nil && len(requestInfo.Name) > 0 { |
| 519 |
scope.name = requestInfo.Name |
| 520 |
} else if selectorName, ok := pred.Field.RequiresExactMatch("metadata.name"); ok { |
| 521 |
scope.name = selectorName |
| 522 |
} |
| 523 |
|
| 524 |
// for request like '/api/v1/watch/namespaces/*', set scope.namespace to empty. |
| 525 |
// namespaces don't populate metadata.namespace in ObjFields. |
| 526 |
if c.groupResource == coreNamespaceResource && len(scope.namespace) > 0 && scope.namespace == scope.name { |
| 527 |
scope.namespace = "" |
| 528 |
} |
| 529 |
|
| 530 |
triggerValue, triggerSupported := "", false |
| 531 |
if c.indexedTrigger != nil { |
| 532 |
for _, field := range pred.IndexFields { |
| 533 |
if field == c.indexedTrigger.indexName { |
| 534 |
if value, ok := pred.Field.RequiresExactMatch(field); ok { |
| 535 |
triggerValue, triggerSupported = value, true |
| 536 |
break |
| 537 |
} |
| 538 |
} |
| 539 |
} |
| 540 |
} |
| 541 |
|
| 542 |
// It boils down to a tradeoff between: |
| 543 |
// - having it as small as possible to reduce memory usage |
| 544 |
// - having it large enough to ensure that watchers that need to process |
| 545 |
// a bunch of changes have enough buffer to avoid from blocking other |
| 546 |
// watchers on our watcher having a processing hiccup |
| 547 |
chanSize := c.watchCache.suggestedWatchChannelSize(c.indexedTrigger != nil, triggerSupported) |
| 548 |
|
| 549 |
// client-go is going to fall back to a standard LIST on any error |
| 550 |
// returned for watch-list requests |
| 551 |
if isListWatchRequest(opts) && !etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress) { |
| 552 |
return newErrWatcher(fmt.Errorf("a watch stream was requested by the client but the required storage feature %s is disabled", storage.RequestWatchProgress)), nil |
| 553 |
} |
| 554 |
|
| 555 |
// Determine the ResourceVersion to which the watch cache must be synchronized |
| 556 |
requiredResourceVersion, err := c.getWatchCacheResourceVersion(ctx, requestedWatchRV, opts) |
| 557 |
if err != nil { |
| 558 |
return newErrWatcher(err), nil |
| 559 |
} |
| 560 |
|
| 561 |
// Determine a function that computes the bookmarkAfterResourceVersion |
| 562 |
bookmarkAfterResourceVersionFn, err := c.getBookmarkAfterResourceVersionLockedFunc(requestedWatchRV, requiredResourceVersion, opts) |
| 563 |
if err != nil { |
| 564 |
return newErrWatcher(err), nil |
| 565 |
} |
| 566 |
|
| 567 |
// Determine watch timeout('0' means deadline is not set, ignore checking) |
| 568 |
deadline, _ := ctx.Deadline() |
| 569 |
|
| 570 |
identifier := fmt.Sprintf("key: %q, labels: %q, fields: %q", key, pred.Label, pred.Field) |
| 571 |
|
| 572 |
// Create a watcher here to reduce memory allocations under lock, |
| 573 |
// given that memory allocation may trigger GC and block the thread. |
| 574 |
// Also note that emptyFunc is a placeholder, until we will be able |
| 575 |
// to compute watcher.forget function (which has to happen under lock). |
| 576 |
watcher := newCacheWatcher( |
| 577 |
chanSize, |
| 578 |
filterWithAttrsAndPrefixFunction(key, pred), |
| 579 |
emptyFunc, |
| 580 |
c.versioner, |
| 581 |
deadline, |
| 582 |
pred.AllowWatchBookmarks, |
| 583 |
c.groupResource, |
| 584 |
identifier, |
| 585 |
) |
| 586 |
|
| 587 |
// note that c.waitUntilWatchCacheFreshAndForceAllEvents must be called without |
| 588 |
// the c.watchCache.RLock held otherwise we are at risk of a deadlock |
| 589 |
// mainly because c.watchCache.processEvent method won't be able to make progress |
| 590 |
// |
| 591 |
// moreover even though the c.waitUntilWatchCacheFreshAndForceAllEvents acquires a lock |
| 592 |
// it is safe to release the lock after the method finishes because we don't require |
| 593 |
// any atomicity between the call to the method and further calls that actually get the events. |
| 594 |
err = c.waitUntilWatchCacheFreshAndForceAllEvents(ctx, requiredResourceVersion, opts) |
| 595 |
if err != nil { |
| 596 |
return newErrWatcher(err), nil |
| 597 |
} |
| 598 |
|
| 599 |
// We explicitly use thread unsafe version and do locking ourself to ensure that |
| 600 |
// no new events will be processed in the meantime. The watchCache will be unlocked |
| 601 |
// on return from this function. |
| 602 |
// Note that we cannot do it under Cacher lock, to avoid a deadlock, since the |
| 603 |
// underlying watchCache is calling processEvent under its lock. |
| 604 |
c.watchCache.RLock() |
| 605 |
defer c.watchCache.RUnlock() |
| 606 |
|
| 607 |
var cacheInterval *watchCacheInterval |
| 608 |
cacheInterval, err = c.watchCache.getAllEventsSinceLocked(requiredResourceVersion, key, opts) |
| 609 |
if err != nil { |
| 610 |
// To match the uncached watch implementation, once we have passed authn/authz/admission, |
| 611 |
// and successfully parsed a resource version, other errors must fail with a watch event of type ERROR, |
| 612 |
// rather than a directly returned error. |
| 613 |
return newErrWatcher(err), nil |
| 614 |
} |
| 615 |
|
| 616 |
c.setInitialEventsEndBookmarkIfRequested(cacheInterval, opts, c.watchCache.resourceVersion) |
| 617 |
|
| 618 |
addedWatcher := false |
| 619 |
func() { |
| 620 |
c.Lock() |
| 621 |
defer c.Unlock() |
| 622 |
|
| 623 |
if generation, _, err := c.ready.checkAndReadGeneration(); generation != readyGeneration || err != nil { |
| 624 |
// We went unready or are already on a different generation. |
| 625 |
// Avoid registering and starting the watch as it will have to be |
| 626 |
// terminated immediately anyway. |
| 627 |
return |
| 628 |
} |
| 629 |
|
| 630 |
// Update watcher.forget function once we can compute it. |
| 631 |
watcher.forget = forgetWatcher(c, watcher, c.watcherIdx, scope, triggerValue, triggerSupported) |
| 632 |
// Update the bookMarkAfterResourceVersion |
| 633 |
watcher.setBookmarkAfterResourceVersion(bookmarkAfterResourceVersionFn()) |
| 634 |
c.watchers.addWatcher(watcher, c.watcherIdx, scope, triggerValue, triggerSupported) |
| 635 |
addedWatcher = true |
| 636 |
|
| 637 |
// Add it to the queue only when the client support watch bookmarks. |
| 638 |
if watcher.allowWatchBookmarks { |
| 639 |
c.bookmarkWatchers.addWatcherThreadUnsafe(watcher) |
| 640 |
} |
| 641 |
c.watcherIdx++ |
| 642 |
}() |
| 643 |
|
| 644 |
if !addedWatcher { |
| 645 |
// Watcher isn't really started at this point, so it's safe to just drop it. |
| 646 |
// |
| 647 |
// We're simulating the immediate watch termination, which boils down to simply |
| 648 |
// closing the watcher. |
| 649 |
return newImmediateCloseWatcher(), nil |
| 650 |
} |
| 651 |
|
| 652 |
go watcher.processInterval(ctx, cacheInterval, requiredResourceVersion) |
| 653 |
return watcher, nil |
| 654 |
} |
| 655 |
|
| 656 |
func (c *Cacher) Get(ctx context.Context, key string, opts storage.GetOptions, objPtr runtime.Object) error { |
| 657 |
getRV, err := c.versioner.ParseResourceVersion(opts.ResourceVersion) |
| 658 |
if err != nil { |
| 659 |
return err |
| 660 |
} |
| 661 |
|
| 662 |
objVal, err := conversion.EnforcePtr(objPtr) |
| 663 |
if err != nil { |
| 664 |
return err |
| 665 |
} |
| 666 |
|
| 667 |
obj, exists, readResourceVersion, err := c.watchCache.WaitUntilFreshAndGet(ctx, getRV, key) |
| 668 |
if err != nil { |
| 669 |
return err |
| 670 |
} |
| 671 |
|
| 672 |
if exists { |
| 673 |
elem, ok := obj.(*storeElement) |
| 674 |
if !ok { |
| 675 |
return fmt.Errorf("non *storeElement returned from storage: %v", obj) |
| 676 |
} |
| 677 |
objVal.Set(reflect.ValueOf(elem.Object).Elem()) |
| 678 |
} else { |
| 679 |
objVal.Set(reflect.Zero(objVal.Type())) |
| 680 |
if !opts.IgnoreNotFound { |
| 681 |
return storage.NewKeyNotFoundError(key, int64(readResourceVersion)) |
| 682 |
} |
| 683 |
} |
| 684 |
return nil |
| 685 |
} |
| 686 |
|
| 687 |
// computeListLimit determines whether the cacher should |
| 688 |
// apply a limit to an incoming LIST request and returns its value. |
| 689 |
// |
| 690 |
// note that this function doesn't check RVM nor the Continuation token. |
| 691 |
// these parameters are validated by the shouldDelegateList function. |
| 692 |
// |
| 693 |
// as of today, the limit is ignored for requests that set RV == 0 |
| 694 |
func computeListLimit(opts storage.ListOptions) int64 { |
| 695 |
if opts.Predicate.Limit <= 0 || opts.ResourceVersion == "0" { |
| 696 |
return 0 |
| 697 |
} |
| 698 |
return opts.Predicate.Limit |
| 699 |
} |
| 700 |
|
| 701 |
func (c *Cacher) listItems(ctx context.Context, listRV uint64, key string, opts storage.ListOptions) (listResp, string, error) { |
| 702 |
if !opts.Recursive { |
| 703 |
obj, exists, readResourceVersion, err := c.watchCache.WaitUntilFreshAndGet(ctx, listRV, key) |
| 704 |
if err != nil { |
| 705 |
return listResp{}, "", err |
| 706 |
} |
| 707 |
if exists { |
| 708 |
return listResp{Items: []interface{}{obj}, ResourceVersion: readResourceVersion}, "", nil |
| 709 |
} |
| 710 |
return listResp{ResourceVersion: readResourceVersion}, "", nil |
| 711 |
} |
| 712 |
return c.watchCache.WaitUntilFreshAndList(ctx, listRV, key, opts) |
| 713 |
} |
| 714 |
|
| 715 |
type listResp struct { |
| 716 |
Items []interface{} |
| 717 |
ResourceVersion uint64 |
| 718 |
} |
| 719 |
|
| 720 |
// GetList implements storage.Interface |
| 721 |
func (c *Cacher) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { |
| 722 |
// For recursive lists, we need to make sure the key ended with "/" so that we only |
| 723 |
// get children "directories". e.g. if we have key "/a", "/a/b", "/ab", getting keys |
| 724 |
// with prefix "/a" will return all three, while with prefix "/a/" will return only |
| 725 |
// "/a/b" which is the correct answer. |
| 726 |
preparedKey := key |
| 727 |
if opts.Recursive && !strings.HasSuffix(key, "/") { |
| 728 |
preparedKey += "/" |
| 729 |
} |
| 730 |
listRV, err := c.versioner.ParseResourceVersion(opts.ResourceVersion) |
| 731 |
if err != nil { |
| 732 |
return err |
| 733 |
} |
| 734 |
|
| 735 |
ctx, span := tracing.Start(ctx, "cacher.GetList", |
| 736 |
attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)), |
| 737 |
attribute.Stringer("type", c.groupResource)) |
| 738 |
defer span.End(500 * time.Millisecond) |
| 739 |
|
| 740 |
if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { |
| 741 |
if downtime, err := c.ready.check(); err != nil { |
| 742 |
// If Cacher is not initialized, reject List requests |
| 743 |
// as described in https://kep.k8s.io/4568 |
| 744 |
return errors.NewTooManyRequests(err.Error(), calculateRetryAfterForUnreadyCache(downtime)) |
| 745 |
} |
| 746 |
} else { |
| 747 |
if err := c.ready.wait(ctx); err != nil { |
| 748 |
return errors.NewServiceUnavailable(err.Error()) |
| 749 |
} |
| 750 |
} |
| 751 |
span.AddEvent("Ready") |
| 752 |
|
| 753 |
// List elements with at least 'listRV' from cache. |
| 754 |
listPtr, err := meta.GetItemsPtr(listObj) |
| 755 |
if err != nil { |
| 756 |
return err |
| 757 |
} |
| 758 |
listVal, err := conversion.EnforcePtr(listPtr) |
| 759 |
if err != nil { |
| 760 |
return err |
| 761 |
} |
| 762 |
if listVal.Kind() != reflect.Slice { |
| 763 |
return fmt.Errorf("need a pointer to slice, got %v", listVal.Kind()) |
| 764 |
} |
| 765 |
|
| 766 |
resp, indexUsed, err := c.listItems(ctx, listRV, preparedKey, opts) |
| 767 |
if err != nil { |
| 768 |
return err |
| 769 |
} |
| 770 |
span.AddEvent("Listed items from cache", attribute.Int("count", len(resp.Items))) |
| 771 |
// store pointer of eligible objects, |
| 772 |
// Why not directly put object in the items of listObj? |
| 773 |
// the elements in ListObject are Struct type, making slice will bring excessive memory consumption. |
| 774 |
// so we try to delay this action as much as possible |
| 775 |
var selectedObjects []runtime.Object |
| 776 |
var lastSelectedObjectKey string |
| 777 |
var hasMoreListItems bool |
| 778 |
limit := computeListLimit(opts) |
| 779 |
for i, obj := range resp.Items { |
| 780 |
elem, ok := obj.(*storeElement) |
| 781 |
if !ok { |
| 782 |
return fmt.Errorf("non *storeElement returned from storage: %v", obj) |
| 783 |
} |
| 784 |
if opts.Predicate.MatchesObjectAttributes(elem.Labels, elem.Fields) { |
| 785 |
selectedObjects = append(selectedObjects, elem.Object) |
| 786 |
lastSelectedObjectKey = elem.Key |
| 787 |
} |
| 788 |
if limit > 0 && int64(len(selectedObjects)) >= limit { |
| 789 |
hasMoreListItems = i < len(resp.Items)-1 |
| 790 |
break |
| 791 |
} |
| 792 |
} |
| 793 |
if len(selectedObjects) == 0 { |
| 794 |
// Ensure that we never return a nil Items pointer in the result for consistency. |
| 795 |
listVal.Set(reflect.MakeSlice(listVal.Type(), 0, 0)) |
| 796 |
} else { |
| 797 |
// Resize the slice appropriately, since we already know that size of result set |
| 798 |
listVal.Set(reflect.MakeSlice(listVal.Type(), len(selectedObjects), len(selectedObjects))) |
| 799 |
span.AddEvent("Resized result") |
| 800 |
for i, o := range selectedObjects { |
| 801 |
listVal.Index(i).Set(reflect.ValueOf(o).Elem()) |
| 802 |
} |
| 803 |
} |
| 804 |
span.AddEvent("Filtered items", attribute.Int("count", listVal.Len())) |
| 805 |
if c.versioner != nil { |
| 806 |
continueValue, remainingItemCount, err := storage.PrepareContinueToken(lastSelectedObjectKey, key, int64(resp.ResourceVersion), int64(len(resp.Items)), hasMoreListItems, opts) |
| 807 |
if err != nil { |
| 808 |
return err |
| 809 |
} |
| 810 |
|
| 811 |
if err = c.versioner.UpdateList(listObj, resp.ResourceVersion, continueValue, remainingItemCount); err != nil { |
| 812 |
return err |
| 813 |
} |
| 814 |
} |
| 815 |
metrics.RecordListCacheMetrics(c.resourcePrefix, indexUsed, len(resp.Items), listVal.Len()) |
| 816 |
return nil |
| 817 |
} |
| 818 |
|
| 819 |
// baseObjectThreadUnsafe omits locking for cachingObject. |
| 820 |
func baseObjectThreadUnsafe(object runtime.Object) runtime.Object { |
| 821 |
if co, ok := object.(*cachingObject); ok { |
| 822 |
return co.object |
| 823 |
} |
| 824 |
return object |
| 825 |
} |
| 826 |
|
| 827 |
func (c *Cacher) triggerValuesThreadUnsafe(event *watchCacheEvent) ([]string, bool) { |
| 828 |
if c.indexedTrigger == nil { |
| 829 |
return nil, false |
| 830 |
} |
| 831 |
|
| 832 |
result := make([]string, 0, 2) |
| 833 |
result = append(result, c.indexedTrigger.indexerFunc(baseObjectThreadUnsafe(event.Object))) |
| 834 |
if event.PrevObject == nil { |
| 835 |
return result, true |
| 836 |
} |
| 837 |
prevTriggerValue := c.indexedTrigger.indexerFunc(baseObjectThreadUnsafe(event.PrevObject)) |
| 838 |
if result[0] != prevTriggerValue { |
| 839 |
result = append(result, prevTriggerValue) |
| 840 |
} |
| 841 |
return result, true |
| 842 |
} |
| 843 |
|
| 844 |
func (c *Cacher) processEvent(event *watchCacheEvent) { |
| 845 |
if curLen := int64(len(c.incoming)); c.incomingHWM.Update(curLen) { |
| 846 |
// Monitor if this gets backed up, and how much. |
| 847 |
klog.V(1).Infof("cacher (%v): %v objects queued in incoming channel.", c.groupResource.String(), curLen) |
| 848 |
} |
| 849 |
c.incoming <- *event |
| 850 |
} |
| 851 |
|
| 852 |
func (c *Cacher) dispatchEvents() { |
| 853 |
// Jitter to help level out any aggregate load. |
| 854 |
bookmarkTimer := c.clock.NewTimer(wait.Jitter(time.Second, 0.25)) |
| 855 |
defer bookmarkTimer.Stop() |
| 856 |
|
| 857 |
// The internal informer populates the RV as soon as it conducts |
| 858 |
// The first successful sync with the underlying store. |
| 859 |
// The cache must wait until this first sync is completed to be deemed ready. |
| 860 |
// Since we cannot send a bookmark when the lastProcessedResourceVersion is 0, |
| 861 |
// we poll aggressively for the first list RV before entering the dispatch loop. |
| 862 |
lastProcessedResourceVersion := uint64(0) |
| 863 |
if err := wait.PollUntilContextCancel(wait.ContextForChannel(c.stopCh), 10*time.Millisecond, true, func(_ context.Context) (bool, error) { |
| 864 |
if rv := c.watchCache.getListResourceVersion(); rv != 0 { |
| 865 |
lastProcessedResourceVersion = rv |
| 866 |
return true, nil |
| 867 |
} |
| 868 |
return false, nil |
| 869 |
}); err != nil { |
| 870 |
// given the function above never returns error, |
| 871 |
// the non-empty error means that the stopCh was closed |
| 872 |
return |
| 873 |
} |
| 874 |
for { |
| 875 |
select { |
| 876 |
case event, ok := <-c.incoming: |
| 877 |
if !ok { |
| 878 |
return |
| 879 |
} |
| 880 |
// Don't dispatch bookmarks coming from the storage layer. |
| 881 |
// They can be very frequent (even to the level of subseconds) |
| 882 |
// to allow efficient watch resumption on kube-apiserver restarts, |
| 883 |
// and propagating them down may overload the whole system. |
| 884 |
// |
| 885 |
// TODO: If at some point we decide the performance and scalability |
| 886 |
// footprint is acceptable, this is the place to hook them in. |
| 887 |
// However, we then need to check if this was called as a result |
| 888 |
// of a bookmark event or regular Add/Update/Delete operation by |
| 889 |
// checking if resourceVersion here has changed. |
| 890 |
if event.Type != watch.Bookmark { |
| 891 |
c.dispatchEvent(&event) |
| 892 |
} |
| 893 |
lastProcessedResourceVersion = event.ResourceVersion |
| 894 |
metrics.EventsCounter.WithLabelValues(c.groupResource.String()).Inc() |
| 895 |
case <-bookmarkTimer.C(): |
| 896 |
bookmarkTimer.Reset(wait.Jitter(time.Second, 0.25)) |
| 897 |
bookmarkEvent := &watchCacheEvent{ |
| 898 |
Type: watch.Bookmark, |
| 899 |
Object: c.newFunc(), |
| 900 |
ResourceVersion: lastProcessedResourceVersion, |
| 901 |
} |
| 902 |
if err := c.versioner.UpdateObject(bookmarkEvent.Object, bookmarkEvent.ResourceVersion); err != nil { |
| 903 |
klog.Errorf("failure to set resourceVersion to %d on bookmark event %+v", bookmarkEvent.ResourceVersion, bookmarkEvent.Object) |
| 904 |
continue |
| 905 |
} |
| 906 |
c.dispatchEvent(bookmarkEvent) |
| 907 |
case <-c.stopCh: |
| 908 |
return |
| 909 |
} |
| 910 |
} |
| 911 |
} |
| 912 |
|
| 913 |
func setCachingObjects(event *watchCacheEvent, versioner storage.Versioner) { |
| 914 |
switch event.Type { |
| 915 |
case watch.Added, watch.Modified: |
| 916 |
if object, err := newCachingObject(event.Object); err == nil { |
| 917 |
event.Object = object |
| 918 |
} else { |
| 919 |
klog.Errorf("couldn't create cachingObject from: %#v", event.Object) |
| 920 |
} |
| 921 |
// Don't wrap PrevObject for update event (for create events it is nil). |
| 922 |
// We only encode those to deliver DELETE watch events, so if |
| 923 |
// event.Object is not nil it can be used only for watchers for which |
| 924 |
// selector was satisfied for its previous version and is no longer |
| 925 |
// satisfied for the current version. |
| 926 |
// This is rare enough that it doesn't justify making deep-copy of the |
| 927 |
// object (done by newCachingObject) every time. |
| 928 |
case watch.Deleted: |
| 929 |
// Don't wrap Object for delete events - these are not to deliver any |
| 930 |
// events. Only wrap PrevObject. |
| 931 |
if object, err := newCachingObject(event.PrevObject); err == nil { |
| 932 |
// Update resource version of the object. |
| 933 |
// event.PrevObject is used to deliver DELETE watch events and |
| 934 |
// for them, we set resourceVersion to <current> instead of |
| 935 |
// the resourceVersion of the last modification of the object. |
| 936 |
updateResourceVersion(object, versioner, event.ResourceVersion) |
| 937 |
event.PrevObject = object |
| 938 |
} else { |
| 939 |
klog.Errorf("couldn't create cachingObject from: %#v", event.Object) |
| 940 |
} |
| 941 |
} |
| 942 |
} |
| 943 |
|
| 944 |
func (c *Cacher) dispatchEvent(event *watchCacheEvent) { |
| 945 |
c.startDispatching(event) |
| 946 |
defer c.finishDispatching() |
| 947 |
// Watchers stopped after startDispatching will be delayed to finishDispatching, |
| 948 |
|
| 949 |
// Since add() can block, we explicitly add when cacher is unlocked. |
| 950 |
// Dispatching event in nonblocking way first, which make faster watchers |
| 951 |
// not be blocked by slower ones. |
| 952 |
// |
| 953 |
// Note: if we ever decide to cache the serialization of bookmark events, |
| 954 |
// we will also need to modify the watchEncoder encoder |
| 955 |
if event.Type == watch.Bookmark { |
| 956 |
for _, watcher := range c.watchersBuffer { |
| 957 |
watcher.nonblockingAdd(event) |
| 958 |
} |
| 959 |
} else { |
| 960 |
// Set up caching of object serializations only for dispatching this event. |
| 961 |
// |
| 962 |
// Storing serializations in memory would result in increased memory usage, |
| 963 |
// but it would help for caching encodings for watches started from old |
| 964 |
// versions. However, we still don't have a convincing data that the gain |
| 965 |
// from it justifies increased memory usage, so for now we drop the cached |
| 966 |
// serializations after dispatching this event. |
| 967 |
// |
| 968 |
// Given that CachingObject is just wrapping the object and not perfoming |
| 969 |
// deep-copying (until some field is explicitly being modified), we create |
| 970 |
// it unconditionally to ensure safety and reduce deep-copying. |
| 971 |
// |
| 972 |
// Make a shallow copy to allow overwriting Object and PrevObject. |
| 973 |
wcEvent := *event |
| 974 |
setCachingObjects(&wcEvent, c.versioner) |
| 975 |
event = &wcEvent |
| 976 |
|
| 977 |
c.blockedWatchers = c.blockedWatchers[:0] |
| 978 |
for _, watcher := range c.watchersBuffer { |
| 979 |
if !watcher.nonblockingAdd(event) { |
| 980 |
c.blockedWatchers = append(c.blockedWatchers, watcher) |
| 981 |
} |
| 982 |
} |
| 983 |
|
| 984 |
if len(c.blockedWatchers) > 0 { |
| 985 |
// dispatchEvent is called very often, so arrange |
| 986 |
// to reuse timers instead of constantly allocating. |
| 987 |
startTime := time.Now() |
| 988 |
timeout := c.dispatchTimeoutBudget.takeAvailable() |
| 989 |
c.timer.Reset(timeout) |
| 990 |
|
| 991 |
// Send event to all blocked watchers. As long as timer is running, |
| 992 |
// `add` will wait for the watcher to unblock. After timeout, |
| 993 |
// `add` will not wait, but immediately close a still blocked watcher. |
| 994 |
// Hence, every watcher gets the chance to unblock itself while timer |
| 995 |
// is running, not only the first ones in the list. |
| 996 |
timer := c.timer |
| 997 |
for _, watcher := range c.blockedWatchers { |
| 998 |
if !watcher.add(event, timer) { |
| 999 |
// fired, clean the timer by set it to nil. |
| 1000 |
timer = nil |
| 1001 |
} |
| 1002 |
} |
| 1003 |
|
| 1004 |
// Stop the timer if it is not fired |
| 1005 |
if timer != nil && !timer.Stop() { |
| 1006 |
// Consume triggered (but not yet received) timer event |
| 1007 |
// so that future reuse does not get a spurious timeout. |
| 1008 |
<-timer.C |
| 1009 |
} |
| 1010 |
|
| 1011 |
c.dispatchTimeoutBudget.returnUnused(timeout - time.Since(startTime)) |
| 1012 |
} |
| 1013 |
} |
| 1014 |
} |
| 1015 |
|
| 1016 |
func (c *Cacher) startDispatchingBookmarkEventsLocked() { |
| 1017 |
// Pop already expired watchers. However, explicitly ignore stopped ones, |
| 1018 |
// as we don't delete watcher from bookmarkWatchers when it is stopped. |
| 1019 |
for _, watchers := range c.bookmarkWatchers.popExpiredWatchersThreadUnsafe() { |
| 1020 |
for _, watcher := range watchers { |
| 1021 |
// c.Lock() is held here. |
| 1022 |
// watcher.stopThreadUnsafe() is protected by c.Lock() |
| 1023 |
if watcher.stopped { |
| 1024 |
continue |
| 1025 |
} |
| 1026 |
c.watchersBuffer = append(c.watchersBuffer, watcher) |
| 1027 |
c.expiredBookmarkWatchers = append(c.expiredBookmarkWatchers, watcher) |
| 1028 |
} |
| 1029 |
} |
| 1030 |
} |
| 1031 |
|
| 1032 |
// startDispatching chooses watchers potentially interested in a given event |
| 1033 |
// a marks dispatching as true. |
| 1034 |
func (c *Cacher) startDispatching(event *watchCacheEvent) { |
| 1035 |
// It is safe to call triggerValuesThreadUnsafe here, because at this |
| 1036 |
// point only this thread can access this event (we create a separate |
| 1037 |
// watchCacheEvent for every dispatch). |
| 1038 |
triggerValues, supported := c.triggerValuesThreadUnsafe(event) |
| 1039 |
|
| 1040 |
c.Lock() |
| 1041 |
defer c.Unlock() |
| 1042 |
|
| 1043 |
c.dispatching = true |
| 1044 |
// We are reusing the slice to avoid memory reallocations in every |
| 1045 |
// dispatchEvent() call. That may prevent Go GC from freeing items |
| 1046 |
// from previous phases that are sitting behind the current length |
| 1047 |
// of the slice, but there is only a limited number of those and the |
| 1048 |
// gain from avoiding memory allocations is much bigger. |
| 1049 |
c.watchersBuffer = c.watchersBuffer[:0] |
| 1050 |
|
| 1051 |
if event.Type == watch.Bookmark { |
| 1052 |
c.startDispatchingBookmarkEventsLocked() |
| 1053 |
// return here to reduce following code indentation and diff |
| 1054 |
return |
| 1055 |
} |
| 1056 |
|
| 1057 |
// iterate over watchers for each applicable namespace/name tuple |
| 1058 |
namespace := event.ObjFields["metadata.namespace"] |
| 1059 |
name := event.ObjFields["metadata.name"] |
| 1060 |
if len(namespace) > 0 { |
| 1061 |
if len(name) > 0 { |
| 1062 |
// namespaced watchers scoped by name |
| 1063 |
for _, watcher := range c.watchers.allWatchers[namespacedName{namespace: namespace, name: name}] { |
| 1064 |
c.watchersBuffer = append(c.watchersBuffer, watcher) |
| 1065 |
} |
| 1066 |
} |
| 1067 |
// namespaced watchers not scoped by name |
| 1068 |
for _, watcher := range c.watchers.allWatchers[namespacedName{namespace: namespace}] { |
| 1069 |
c.watchersBuffer = append(c.watchersBuffer, watcher) |
| 1070 |
} |
| 1071 |
} |
| 1072 |
if len(name) > 0 { |
| 1073 |
// cluster-wide watchers scoped by name |
| 1074 |
for _, watcher := range c.watchers.allWatchers[namespacedName{name: name}] { |
| 1075 |
c.watchersBuffer = append(c.watchersBuffer, watcher) |
| 1076 |
} |
| 1077 |
} |
| 1078 |
// cluster-wide watchers unscoped by name |
| 1079 |
for _, watcher := range c.watchers.allWatchers[namespacedName{}] { |
| 1080 |
c.watchersBuffer = append(c.watchersBuffer, watcher) |
| 1081 |
} |
| 1082 |
|
| 1083 |
if supported { |
| 1084 |
// Iterate over watchers interested in the given values of the trigger. |
| 1085 |
for _, triggerValue := range triggerValues { |
| 1086 |
for _, watcher := range c.watchers.valueWatchers[triggerValue] { |
| 1087 |
c.watchersBuffer = append(c.watchersBuffer, watcher) |
| 1088 |
} |
| 1089 |
} |
| 1090 |
} else { |
| 1091 |
// supported equal to false generally means that trigger function |
| 1092 |
// is not defined (or not aware of any indexes). In this case, |
| 1093 |
// watchers filters should generally also don't generate any |
| 1094 |
// trigger values, but can cause problems in case of some |
| 1095 |
// misconfiguration. Thus we paranoidly leave this branch. |
| 1096 |
|
| 1097 |
// Iterate over watchers interested in exact values for all values. |
| 1098 |
for _, watchers := range c.watchers.valueWatchers { |
| 1099 |
for _, watcher := range watchers { |
| 1100 |
c.watchersBuffer = append(c.watchersBuffer, watcher) |
| 1101 |
} |
| 1102 |
} |
| 1103 |
} |
| 1104 |
} |
| 1105 |
|
| 1106 |
// finishDispatching stops all the watchers that were supposed to be |
| 1107 |
// stopped in the meantime, but it was deferred to avoid closing input |
| 1108 |
// channels of watchers, as add() may still have writing to it. |
| 1109 |
// It also marks dispatching as false. |
| 1110 |
func (c *Cacher) finishDispatching() { |
| 1111 |
c.Lock() |
| 1112 |
defer c.Unlock() |
| 1113 |
c.dispatching = false |
| 1114 |
for _, watcher := range c.watchersToStop { |
| 1115 |
watcher.stopLocked() |
| 1116 |
} |
| 1117 |
c.watchersToStop = c.watchersToStop[:0] |
| 1118 |
|
| 1119 |
for _, watcher := range c.expiredBookmarkWatchers { |
| 1120 |
if watcher.stopped { |
| 1121 |
continue |
| 1122 |
} |
| 1123 |
// requeue the watcher for the next bookmark if needed. |
| 1124 |
c.bookmarkWatchers.addWatcherThreadUnsafe(watcher) |
| 1125 |
} |
| 1126 |
c.expiredBookmarkWatchers = c.expiredBookmarkWatchers[:0] |
| 1127 |
} |
| 1128 |
|
| 1129 |
func (c *Cacher) terminateAllWatchers() { |
| 1130 |
c.Lock() |
| 1131 |
defer c.Unlock() |
| 1132 |
c.watchers.terminateAll(c.groupResource, c.stopWatcherLocked) |
| 1133 |
} |
| 1134 |
|
| 1135 |
func (c *Cacher) stopWatcherLocked(watcher *cacheWatcher) { |
| 1136 |
if c.dispatching { |
| 1137 |
c.watchersToStop = append(c.watchersToStop, watcher) |
| 1138 |
} else { |
| 1139 |
watcher.stopLocked() |
| 1140 |
} |
| 1141 |
} |
| 1142 |
|
| 1143 |
func (c *Cacher) isStopped() bool { |
| 1144 |
c.stopLock.RLock() |
| 1145 |
defer c.stopLock.RUnlock() |
| 1146 |
return c.stopped |
| 1147 |
} |
| 1148 |
|
| 1149 |
// Stop implements the graceful termination. |
| 1150 |
func (c *Cacher) Stop() { |
| 1151 |
c.stopLock.Lock() |
| 1152 |
if c.stopped { |
| 1153 |
// avoid stopping twice (note: cachers are shared with subresources) |
| 1154 |
c.stopLock.Unlock() |
| 1155 |
return |
| 1156 |
} |
| 1157 |
c.stopped = true |
| 1158 |
c.ready.stop() |
| 1159 |
c.stopLock.Unlock() |
| 1160 |
close(c.stopCh) |
| 1161 |
c.stopWg.Wait() |
| 1162 |
} |
| 1163 |
|
| 1164 |
func forgetWatcher(c *Cacher, w *cacheWatcher, index int, scope namespacedName, triggerValue string, triggerSupported bool) func(bool) { |
| 1165 |
return func(drainWatcher bool) { |
| 1166 |
c.Lock() |
| 1167 |
defer c.Unlock() |
| 1168 |
|
| 1169 |
w.setDrainInputBufferLocked(drainWatcher) |
| 1170 |
|
| 1171 |
// It's possible that the watcher is already not in the structure (e.g. in case of |
| 1172 |
// simultaneous Stop() and terminateAllWatchers(), but it is safe to call stopLocked() |
| 1173 |
// on a watcher multiple times. |
| 1174 |
c.watchers.deleteWatcher(index, scope, triggerValue, triggerSupported) |
| 1175 |
c.stopWatcherLocked(w) |
| 1176 |
} |
| 1177 |
} |
| 1178 |
|
| 1179 |
func filterWithAttrsAndPrefixFunction(key string, p storage.SelectionPredicate) filterWithAttrsFunc { |
| 1180 |
filterFunc := func(objKey string, label labels.Set, field fields.Set) bool { |
| 1181 |
if !hasPathPrefix(objKey, key) { |
| 1182 |
return false |
| 1183 |
} |
| 1184 |
return p.MatchesObjectAttributes(label, field) |
| 1185 |
} |
| 1186 |
return filterFunc |
| 1187 |
} |
| 1188 |
|
| 1189 |
// LastSyncResourceVersion returns resource version to which the underlying cache is synced. |
| 1190 |
func (c *Cacher) LastSyncResourceVersion() (uint64, error) { |
| 1191 |
if err := c.ready.wait(context.Background()); err != nil { |
| 1192 |
return 0, errors.NewServiceUnavailable(err.Error()) |
| 1193 |
} |
| 1194 |
|
| 1195 |
resourceVersion := c.reflector.LastSyncResourceVersion() |
| 1196 |
return c.versioner.ParseResourceVersion(resourceVersion) |
| 1197 |
} |
| 1198 |
|
| 1199 |
// getBookmarkAfterResourceVersionLockedFunc returns a function that |
| 1200 |
// spits a ResourceVersion after which the bookmark event will be delivered. |
| 1201 |
// |
| 1202 |
// The returned function must be called under the watchCache lock. |
| 1203 |
func (c *Cacher) getBookmarkAfterResourceVersionLockedFunc(parsedResourceVersion, requiredResourceVersion uint64, opts storage.ListOptions) (func() uint64, error) { |
| 1204 |
if !isListWatchRequest(opts) { |
| 1205 |
return func() uint64 { return 0 }, nil |
| 1206 |
} |
| 1207 |
|
| 1208 |
switch { |
| 1209 |
case len(opts.ResourceVersion) == 0: |
| 1210 |
return func() uint64 { return requiredResourceVersion }, nil |
| 1211 |
case parsedResourceVersion == 0: |
| 1212 |
// here we assume that watchCache locked is already held |
| 1213 |
return func() uint64 { return c.watchCache.resourceVersion }, nil |
| 1214 |
default: |
| 1215 |
return func() uint64 { return parsedResourceVersion }, nil |
| 1216 |
} |
| 1217 |
} |
| 1218 |
|
| 1219 |
func isListWatchRequest(opts storage.ListOptions) bool { |
| 1220 |
return opts.SendInitialEvents != nil && *opts.SendInitialEvents && opts.Predicate.AllowWatchBookmarks |
| 1221 |
} |
| 1222 |
|
| 1223 |
// getWatchCacheResourceVersion returns a ResourceVersion to which the watch cache must be synchronized to |
| 1224 |
// |
| 1225 |
// Depending on the input parameters, the semantics of the returned ResourceVersion are: |
| 1226 |
// - must be at Exact RV (when parsedWatchResourceVersion > 0) |
| 1227 |
// - can be at Any RV (when parsedWatchResourceVersion = 0) |
| 1228 |
// - must be at Most Recent RV (return an RV from etcd) |
| 1229 |
// |
| 1230 |
// note that the above semantic is enforced by the API validation (defined elsewhere): |
| 1231 |
// |
| 1232 |
// if SendInitiaEvents != nil => ResourceVersionMatch = NotOlderThan |
| 1233 |
// if ResourceVersionmatch != nil => ResourceVersionMatch = NotOlderThan & SendInitialEvents != nil |
| 1234 |
func (c *Cacher) getWatchCacheResourceVersion(ctx context.Context, parsedWatchResourceVersion uint64, opts storage.ListOptions) (uint64, error) { |
| 1235 |
if len(opts.ResourceVersion) != 0 { |
| 1236 |
return parsedWatchResourceVersion, nil |
| 1237 |
} |
| 1238 |
// legacy case |
| 1239 |
if !utilfeature.DefaultFeatureGate.Enabled(features.WatchFromStorageWithoutResourceVersion) && opts.SendInitialEvents == nil && opts.ResourceVersion == "" { |
| 1240 |
return 0, nil |
| 1241 |
} |
| 1242 |
rv, err := c.storage.GetCurrentResourceVersion(ctx) |
| 1243 |
return rv, err |
| 1244 |
} |
| 1245 |
|
| 1246 |
// waitUntilWatchCacheFreshAndForceAllEvents waits until cache is at least |
| 1247 |
// as fresh as given requestedWatchRV if sendInitialEvents was requested. |
| 1248 |
// otherwise, we allow for establishing the connection because the clients |
| 1249 |
// can wait for events without unnecessary blocking. |
| 1250 |
func (c *Cacher) waitUntilWatchCacheFreshAndForceAllEvents(ctx context.Context, requestedWatchRV uint64, opts storage.ListOptions) error { |
| 1251 |
if opts.SendInitialEvents != nil && *opts.SendInitialEvents { |
| 1252 |
// Here be dragons: |
| 1253 |
// Since the etcd feature checker needs to check all members |
| 1254 |
// to determine whether a given feature is supported, |
| 1255 |
// we may receive a positive response even if the feature is not supported. |
| 1256 |
// |
| 1257 |
// In this very rare scenario, the worst case will be that this |
| 1258 |
// request will wait for 3 seconds before it fails. |
| 1259 |
if etcdfeature.DefaultFeatureSupportChecker.Supports(storage.RequestWatchProgress) && c.watchCache.notFresh(requestedWatchRV) { |
| 1260 |
c.watchCache.waitingUntilFresh.Add() |
| 1261 |
defer c.watchCache.waitingUntilFresh.Remove() |
| 1262 |
} |
| 1263 |
err := c.watchCache.waitUntilFreshAndBlock(ctx, requestedWatchRV) |
| 1264 |
defer c.watchCache.RUnlock() |
| 1265 |
return err |
| 1266 |
} |
| 1267 |
return nil |
| 1268 |
} |
| 1269 |
|
| 1270 |
// Wait blocks until the cacher is Ready or Stopped, it returns an error if Stopped. |
| 1271 |
func (c *Cacher) Wait(ctx context.Context) error { |
| 1272 |
return c.ready.wait(ctx) |
| 1273 |
} |
| 1274 |
|
| 1275 |
// setInitialEventsEndBookmarkIfRequested sets initialEventsEndBookmark field in watchCacheInterval for watchlist request |
| 1276 |
func (c *Cacher) setInitialEventsEndBookmarkIfRequested(cacheInterval *watchCacheInterval, opts storage.ListOptions, currentResourceVersion uint64) { |
| 1277 |
if opts.SendInitialEvents != nil && *opts.SendInitialEvents && opts.Predicate.AllowWatchBookmarks { |
| 1278 |
// We don't need to set the InitialEventsAnnotation for this bookmark event, |
| 1279 |
// because this will be automatically set during event conversion in cacheWatcher.convertToWatchEvent method |
| 1280 |
initialEventsEndBookmark := &watchCacheEvent{ |
| 1281 |
Type: watch.Bookmark, |
| 1282 |
Object: c.newFunc(), |
| 1283 |
ResourceVersion: currentResourceVersion, |
| 1284 |
} |
| 1285 |
|
| 1286 |
if err := c.versioner.UpdateObject(initialEventsEndBookmark.Object, initialEventsEndBookmark.ResourceVersion); err != nil { |
| 1287 |
klog.Errorf("failure to set resourceVersion to %d on initialEventsEndBookmark event %+v for watchlist request and wait for bookmark trigger to send", initialEventsEndBookmark.ResourceVersion, initialEventsEndBookmark.Object) |
| 1288 |
initialEventsEndBookmark = nil |
| 1289 |
} |
| 1290 |
|
| 1291 |
cacheInterval.initialEventsEndBookmark = initialEventsEndBookmark |
| 1292 |
} |
| 1293 |
} |
| 1294 |
|
| 1295 |
func (c *Cacher) Ready() bool { |
| 1296 |
_, err := c.ready.check() |
| 1297 |
return err == nil |
| 1298 |
} |
| 1299 |
|
| 1300 |
// errWatcher implements watch.Interface to return a single error |
| 1301 |
type errWatcher struct { |
| 1302 |
result chan watch.Event |
| 1303 |
} |
| 1304 |
|
| 1305 |
func newErrWatcher(err error) *errWatcher { |
| 1306 |
// Create an error event |
| 1307 |
errEvent := watch.Event{Type: watch.Error} |
| 1308 |
switch err := err.(type) { |
| 1309 |
case runtime.Object: |
| 1310 |
errEvent.Object = err |
| 1311 |
case *errors.StatusError: |
| 1312 |
errEvent.Object = &err.ErrStatus |
| 1313 |
default: |
| 1314 |
errEvent.Object = &metav1.Status{ |
| 1315 |
Status: metav1.StatusFailure, |
| 1316 |
Message: err.Error(), |
| 1317 |
Reason: metav1.StatusReasonInternalError, |
| 1318 |
Code: http.StatusInternalServerError, |
| 1319 |
} |
| 1320 |
} |
| 1321 |
|
| 1322 |
// Create a watcher with room for a single event, populate it, and close the channel |
| 1323 |
watcher := &errWatcher{result: make(chan watch.Event, 1)} |
| 1324 |
watcher.result <- errEvent |
| 1325 |
close(watcher.result) |
| 1326 |
|
| 1327 |
return watcher |
| 1328 |
} |
| 1329 |
|
| 1330 |
func (c *Cacher) ShouldDelegateExactRV(resourceVersion string, recursive bool) (delegator.Result, error) { |
| 1331 |
// Not Recursive is not supported unitl exact RV is implemented for WaitUntilFreshAndGet. |
| 1332 |
if !recursive || c.watchCache.snapshots == nil { |
| 1333 |
return delegator.Result{ShouldDelegate: true}, nil |
| 1334 |
} |
| 1335 |
listRV, err := c.versioner.ParseResourceVersion(resourceVersion) |
| 1336 |
if err != nil { |
| 1337 |
return delegator.Result{}, err |
| 1338 |
} |
| 1339 |
return c.shouldDelegateExactRV(listRV) |
| 1340 |
} |
| 1341 |
|
| 1342 |
func (c *Cacher) ShouldDelegateContinue(continueToken string, recursive bool) (delegator.Result, error) { |
| 1343 |
// Not Recursive is not supported unitl exact RV is implemented for WaitUntilFreshAndGet. |
| 1344 |
if !recursive || c.watchCache.snapshots == nil { |
| 1345 |
return delegator.Result{ShouldDelegate: true}, nil |
| 1346 |
} |
| 1347 |
_, continueRV, err := storage.DecodeContinue(continueToken, c.resourcePrefix) |
| 1348 |
if err != nil { |
| 1349 |
return delegator.Result{}, err |
| 1350 |
} |
| 1351 |
if continueRV > 0 { |
| 1352 |
return c.shouldDelegateExactRV(uint64(continueRV)) |
| 1353 |
} else { |
| 1354 |
// Continue with negative RV is a consistent read. |
| 1355 |
return c.ShouldDelegateConsistentRead() |
| 1356 |
} |
| 1357 |
} |
| 1358 |
|
| 1359 |
func (c *Cacher) shouldDelegateExactRV(rv uint64) (delegator.Result, error) { |
| 1360 |
// Exact requests on future revision require support for consistent read, but are not a consistent read by themselves. |
| 1361 |
if c.watchCache.notFresh(rv) { |
| 1362 |
return delegator.Result{ |
| 1363 |
ShouldDelegate: !delegator.ConsistentReadSupported(), |
| 1364 |
}, nil |
| 1365 |
} |
| 1366 |
_, canServe := c.watchCache.snapshots.GetLessOrEqual(rv) |
| 1367 |
return delegator.Result{ |
| 1368 |
ShouldDelegate: !canServe, |
| 1369 |
}, nil |
| 1370 |
} |
| 1371 |
|
| 1372 |
func (c *Cacher) ShouldDelegateConsistentRead() (delegator.Result, error) { |
| 1373 |
return delegator.Result{ |
| 1374 |
ConsistentRead: true, |
| 1375 |
ShouldDelegate: !delegator.ConsistentReadSupported(), |
| 1376 |
}, nil |
| 1377 |
} |
| 1378 |
|
| 1379 |
// Implements watch.Interface. |
| 1380 |
func (c *errWatcher) ResultChan() <-chan watch.Event { |
| 1381 |
return c.result |
| 1382 |
} |
| 1383 |
|
| 1384 |
// Implements watch.Interface. |
| 1385 |
func (c *errWatcher) Stop() { |
| 1386 |
// no-op |
| 1387 |
} |
| 1388 |
|
| 1389 |
// immediateCloseWatcher implements watch.Interface that is immediately closed |
| 1390 |
type immediateCloseWatcher struct { |
| 1391 |
result chan watch.Event |
| 1392 |
} |
| 1393 |
|
| 1394 |
func newImmediateCloseWatcher() *immediateCloseWatcher { |
| 1395 |
watcher := &immediateCloseWatcher{result: make(chan watch.Event)} |
| 1396 |
close(watcher.result) |
| 1397 |
return watcher |
| 1398 |
} |
| 1399 |
|
| 1400 |
// Implements watch.Interface. |
| 1401 |
func (c *immediateCloseWatcher) ResultChan() <-chan watch.Event { |
| 1402 |
return c.result |
| 1403 |
} |
| 1404 |
|
| 1405 |
// Implements watch.Interface. |
| 1406 |
func (c *immediateCloseWatcher) Stop() { |
| 1407 |
// no-op |
| 1408 |
} |