Debian Codegraph @

pkg/storage/cacher/delegator.go

raw
1 /*
2 Copyright 2015 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package cacher
18
19 import (
20 "context"
21 "fmt"
22 "hash"
23 "hash/fnv"
24 "os"
25 "strconv"
26 "sync"
27 "time"
28
29 "go.opentelemetry.io/otel/attribute"
30 "k8s.io/apimachinery/pkg/api/errors"
31 "k8s.io/apimachinery/pkg/api/meta"
32 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33 "k8s.io/apimachinery/pkg/runtime"
34 "k8s.io/apimachinery/pkg/util/wait"
35 "k8s.io/apimachinery/pkg/watch"
36 "k8s.io/apiserver/pkg/audit"
37 "k8s.io/apiserver/pkg/features"
38 "k8s.io/apiserver/pkg/storage"
39 "k8s.io/apiserver/pkg/storage/cacher/delegator"
40 "k8s.io/apiserver/pkg/storage/cacher/metrics"
41 utilfeature "k8s.io/apiserver/pkg/util/feature"
42 "k8s.io/component-base/tracing"
43 "k8s.io/klog/v2"
44 )
45
46 var (
47 // ConsistencyCheckPeriod is the period of checking consistency between etcd and cache.
48 // 5 minutes were proposed to match the default compaction period. It's magnitute higher than
49 // List latency SLO (30 seconds) and timeout (1 minute).
50 ConsistencyCheckPeriod = 5 * time.Minute
51 // ConsistencyCheckerEnabled enables the consistency checking mechanism for cache.
52 // Based on KUBE_WATCHCACHE_CONSISTENCY_CHECKER environment variable.
53 ConsistencyCheckerEnabled = false
54 )
55
56 func init() {
57 ConsistencyCheckerEnabled, _ = strconv.ParseBool(os.Getenv("KUBE_WATCHCACHE_CONSISTENCY_CHECKER"))
58 }
59
60 func NewCacheDelegator(cacher *Cacher, storage storage.Interface) *CacheDelegator {
61 d := &CacheDelegator{
62 cacher: cacher,
63 storage: storage,
64 stopCh: make(chan struct{}),
65 }
66 if ConsistencyCheckerEnabled {
67 d.checker = newConsistencyChecker(cacher.resourcePrefix, cacher.newListFunc, cacher, storage)
68 d.wg.Add(1)
69 go func() {
70 defer d.wg.Done()
71 d.checker.startChecking(d.stopCh)
72 }()
73 }
74 return d
75 }
76
77 type CacheDelegator struct {
78 cacher *Cacher
79 storage storage.Interface
80 checker *consistencyChecker
81
82 wg sync.WaitGroup
83 stopOnce sync.Once
84 stopCh chan struct{}
85 }
86
87 var _ storage.Interface = (*CacheDelegator)(nil)
88
89 func (c *CacheDelegator) Versioner() storage.Versioner {
90 return c.storage.Versioner()
91 }
92
93 func (c *CacheDelegator) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error {
94 return c.storage.Create(ctx, key, obj, out, ttl)
95 }
96
97 func (c *CacheDelegator) GetCurrentResourceVersion(ctx context.Context) (uint64, error) {
98 return c.storage.GetCurrentResourceVersion(ctx)
99 }
100
101 func (c *CacheDelegator) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions, validateDeletion storage.ValidateObjectFunc, cachedExistingObject runtime.Object, opts storage.DeleteOptions) error {
102 // Ignore the suggestion and try to pass down the current version of the object
103 // read from cache.
104 if elem, exists, err := c.cacher.watchCache.GetByKey(key); err != nil {
105 klog.Errorf("GetByKey returned error: %v", err)
106 } else if exists {
107 // DeepCopy the object since we modify resource version when serializing the
108 // current object.
109 currObj := elem.(*storeElement).Object.DeepCopyObject()
110 return c.storage.Delete(ctx, key, out, preconditions, validateDeletion, currObj, opts)
111 }
112 // If we couldn't get the object, fallback to no-suggestion.
113 return c.storage.Delete(ctx, key, out, preconditions, validateDeletion, nil, opts)
114 }
115
116 func (c *CacheDelegator) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
117 // if the watch-list feature wasn't set and the resourceVersion is unset
118 // ensure that the rv from which the watch is being served, is the latest
119 // one. "latest" is ensured by serving the watch from
120 // the underlying storage.
121 //
122 // it should never happen due to our validation but let's just be super-safe here
123 // and disable sendingInitialEvents when the feature wasn't enabled
124 if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) && opts.SendInitialEvents != nil {
125 opts.SendInitialEvents = nil
126 }
127 // TODO: we should eventually get rid of this legacy case
128 if utilfeature.DefaultFeatureGate.Enabled(features.WatchFromStorageWithoutResourceVersion) && opts.SendInitialEvents == nil && opts.ResourceVersion == "" {
129 return c.storage.Watch(ctx, key, opts)
130 }
131 return c.cacher.Watch(ctx, key, opts)
132 }
133
134 func (c *CacheDelegator) Get(ctx context.Context, key string, opts storage.GetOptions, objPtr runtime.Object) error {
135 ctx, span := tracing.Start(ctx, "cacher.Get",
136 attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)),
137 attribute.String("key", key),
138 attribute.String("resource-version", opts.ResourceVersion))
139 defer span.End(500 * time.Millisecond)
140 if opts.ResourceVersion == "" {
141 // If resourceVersion is not specified, serve it from underlying
142 // storage (for backward compatibility).
143 span.AddEvent("About to Get from underlying storage")
144 return c.storage.Get(ctx, key, opts, objPtr)
145 }
146
147 if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
148 if !c.cacher.Ready() {
149 // If Cache is not initialized, delegator Get requests to storage
150 // as described in https://kep.k8s.io/4568
151 span.AddEvent("About to Get from underlying storage - cache not initialized")
152 return c.storage.Get(ctx, key, opts, objPtr)
153 }
154 }
155 // If resourceVersion is specified, serve it from cache.
156 // It's guaranteed that the returned value is at least that
157 // fresh as the given resourceVersion.
158 getRV, err := c.cacher.versioner.ParseResourceVersion(opts.ResourceVersion)
159 if err != nil {
160 return err
161 }
162 // Do not create a trace - it's not for free and there are tons
163 // of Get requests. We can add it if it will be really needed.
164 if !utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
165 if getRV == 0 && !c.cacher.Ready() {
166 // If Cacher is not yet initialized and we don't require any specific
167 // minimal resource version, simply forward the request to storage.
168 return c.storage.Get(ctx, key, opts, objPtr)
169 }
170 if err := c.cacher.ready.wait(ctx); err != nil {
171 return errors.NewServiceUnavailable(err.Error())
172 }
173 }
174 span.AddEvent("About to fetch object from cache")
175 return c.cacher.Get(ctx, key, opts, objPtr)
176 }
177
178 func (c *CacheDelegator) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
179 _, _, err := storage.ValidateListOptions(c.cacher.resourcePrefix, c.cacher.versioner, opts)
180 if err != nil {
181 return err
182 }
183 result, err := delegator.ShouldDelegateList(opts, c.cacher)
184 if err != nil {
185 return err
186 }
187 if result.ShouldDelegate {
188 return c.storage.GetList(ctx, key, opts, listObj)
189 }
190
191 listRV, err := c.cacher.versioner.ParseResourceVersion(opts.ResourceVersion)
192 if err != nil {
193 return err
194 }
195
196 if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) {
197 if !c.cacher.Ready() && shouldDelegateListOnNotReadyCache(opts) {
198 // If Cacher is not initialized, delegator List requests to storage
199 // as described in https://kep.k8s.io/4568
200 return c.storage.GetList(ctx, key, opts, listObj)
201 }
202 } else {
203 if listRV == 0 && !c.cacher.Ready() {
204 // If Cacher is not yet initialized and we don't require any specific
205 // minimal resource version, simply forward the request to storage.
206 return c.storage.GetList(ctx, key, opts, listObj)
207 }
208 }
209 fallbackOpts := opts
210 if result.ConsistentRead {
211 listRV, err = c.storage.GetCurrentResourceVersion(ctx)
212 if err != nil {
213 return err
214 }
215 // Setting resource version for consistent read in cache based on current ResourceVersion in etcd.
216 opts.ResourceVersion = strconv.FormatInt(int64(listRV), 10)
217 // If continue is not set, we need to set the resource version match to ResourceVersionMatchNotOlderThan to serve latest from cache
218 if opts.Predicate.Continue == "" {
219 opts.ResourceVersionMatch = metav1.ResourceVersionMatchNotOlderThan
220 }
221 }
222 err = c.cacher.GetList(ctx, key, opts, listObj)
223 success := "true"
224 fallback := "false"
225 if err != nil {
226 // ResourceExpired error occurs when attempting to list from cache with a specific resourceVersion
227 // that is no longer available in the cache. With ListFromCacheSnapshot feature (1.34+), we can
228 // serve exact resourceVersion requests from cache if available, falling back to storage only when
229 // the requested version is expired.
230 if errors.IsResourceExpired(err) && utilfeature.DefaultFeatureGate.Enabled(features.ListFromCacheSnapshot) {
231 return c.storage.GetList(ctx, key, fallbackOpts, listObj)
232 }
233 if result.ConsistentRead {
234 // IsTooLargeResourceVersion occurs when the requested RV is higher than cache's current RV
235 // and cache hasn't caught up within the timeout period. Fall back to etcd.
236 if storage.IsTooLargeResourceVersion(err) {
237 fallback = "true"
238 err = c.storage.GetList(ctx, key, fallbackOpts, listObj)
239 }
240 if err != nil {
241 success = "false"
242 }
243 metrics.ConsistentReadTotal.WithLabelValues(c.cacher.resourcePrefix, success, fallback).Add(1)
244 }
245 return err
246 }
247 if result.ConsistentRead {
248 metrics.ConsistentReadTotal.WithLabelValues(c.cacher.resourcePrefix, success, fallback).Add(1)
249 }
250 return nil
251 }
252
253 func shouldDelegateListOnNotReadyCache(opts storage.ListOptions) bool {
254 pred := opts.Predicate
255 noLabelSelector := pred.Label == nil || pred.Label.Empty()
256 noFieldSelector := pred.Field == nil || pred.Field.Empty()
257 hasLimit := pred.Limit > 0
258 return noLabelSelector && noFieldSelector && hasLimit
259 }
260
261 func (c *CacheDelegator) GuaranteedUpdate(ctx context.Context, key string, destination runtime.Object, ignoreNotFound bool, preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, cachedExistingObject runtime.Object) error {
262 // Ignore the suggestion and try to pass down the current version of the object
263 // read from cache.
264 if elem, exists, err := c.cacher.watchCache.GetByKey(key); err != nil {
265 klog.Errorf("GetByKey returned error: %v", err)
266 } else if exists {
267 // DeepCopy the object since we modify resource version when serializing the
268 // current object.
269 currObj := elem.(*storeElement).Object.DeepCopyObject()
270 return c.storage.GuaranteedUpdate(ctx, key, destination, ignoreNotFound, preconditions, tryUpdate, currObj)
271 }
272 // If we couldn't get the object, fallback to no-suggestion.
273 return c.storage.GuaranteedUpdate(ctx, key, destination, ignoreNotFound, preconditions, tryUpdate, nil)
274 }
275
276 func (c *CacheDelegator) Count(pathPrefix string) (int64, error) {
277 return c.storage.Count(pathPrefix)
278 }
279
280 func (c *CacheDelegator) ReadinessCheck() error {
281 if !c.cacher.Ready() {
282 return storage.ErrStorageNotReady
283 }
284 return nil
285 }
286
287 func (c *CacheDelegator) RequestWatchProgress(ctx context.Context) error {
288 return c.storage.RequestWatchProgress(ctx)
289 }
290
291 func (c *CacheDelegator) Stop() {
292 c.stopOnce.Do(func() {
293 close(c.stopCh)
294 })
295 c.wg.Wait()
296 }
297
298 func newConsistencyChecker(resourcePrefix string, newListFunc func() runtime.Object, cacher getListerReady, etcd getLister) *consistencyChecker {
299 return &consistencyChecker{
300 resourcePrefix: resourcePrefix,
301 newListFunc: newListFunc,
302 cacher: cacher,
303 etcd: etcd,
304 }
305 }
306
307 type consistencyChecker struct {
308 resourcePrefix string
309 newListFunc func() runtime.Object
310
311 cacher getListerReady
312 etcd getLister
313 }
314
315 type getListerReady interface {
316 getLister
317 Ready() bool
318 }
319
320 type getLister interface {
321 GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error
322 }
323
324 func (c consistencyChecker) startChecking(stopCh <-chan struct{}) {
325 err := wait.PollUntilContextCancel(wait.ContextForChannel(stopCh), ConsistencyCheckPeriod, false, func(ctx context.Context) (done bool, err error) {
326 c.check(ctx)
327 return false, nil
328 })
329 if err != nil {
330 klog.InfoS("Cache consistency check exiting", "resource", c.resourcePrefix, "err", err)
331 }
332 }
333
334 func (c *consistencyChecker) check(ctx context.Context) {
335 digests, err := c.calculateDigests(ctx)
336 if err != nil {
337 klog.ErrorS(err, "Cache consistency check error", "resource", c.resourcePrefix)
338 metrics.StorageConsistencyCheckTotal.WithLabelValues(c.resourcePrefix, "error").Inc()
339 return
340 }
341 if digests.CacheDigest == digests.EtcdDigest {
342 klog.V(3).InfoS("Cache consistency check passed", "resource", c.resourcePrefix, "resourceVersion", digests.ResourceVersion, "digest", digests.CacheDigest)
343 metrics.StorageConsistencyCheckTotal.WithLabelValues(c.resourcePrefix, "success").Inc()
344 } else {
345 klog.ErrorS(nil, "Cache consistency check failed", "resource", c.resourcePrefix, "resourceVersion", digests.ResourceVersion, "etcdDigest", digests.EtcdDigest, "cacheDigest", digests.CacheDigest)
346 metrics.StorageConsistencyCheckTotal.WithLabelValues(c.resourcePrefix, "failure").Inc()
347 // Panic on internal consistency checking enabled only by environment variable. R
348 panic(fmt.Sprintf("Cache consistency check failed, resource: %q, resourceVersion: %q, etcdDigest: %q, cacheDigest: %q", c.resourcePrefix, digests.ResourceVersion, digests.EtcdDigest, digests.CacheDigest))
349 }
350 }
351
352 func (c *consistencyChecker) calculateDigests(ctx context.Context) (*storageDigest, error) {
353 if !c.cacher.Ready() {
354 return nil, fmt.Errorf("cache is not ready")
355 }
356 cacheDigest, resourceVersion, err := c.calculateStoreDigest(ctx, c.cacher, storage.ListOptions{
357 Recursive: true,
358 ResourceVersion: "0",
359 Predicate: storage.Everything,
360 ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan,
361 })
362 if err != nil {
363 return nil, fmt.Errorf("failed calculating cache digest: %w", err)
364 }
365 etcdDigest, _, err := c.calculateStoreDigest(ctx, c.etcd, storage.ListOptions{
366 Recursive: true,
367 ResourceVersion: resourceVersion,
368 Predicate: storage.Everything,
369 ResourceVersionMatch: metav1.ResourceVersionMatchExact,
370 })
371 if err != nil {
372 return nil, fmt.Errorf("failed calculating etcd digest: %w", err)
373 }
374 return &storageDigest{
375 ResourceVersion: resourceVersion,
376 CacheDigest: cacheDigest,
377 EtcdDigest: etcdDigest,
378 }, nil
379 }
380
381 type storageDigest struct {
382 ResourceVersion string
383 CacheDigest string
384 EtcdDigest string
385 }
386
387 func (c *consistencyChecker) calculateStoreDigest(ctx context.Context, store getLister, opts storage.ListOptions) (digest, rv string, err error) {
388 // TODO: Implement pagination
389 resp := c.newListFunc()
390 err = store.GetList(ctx, c.resourcePrefix, opts, resp)
391 if err != nil {
392 return "", "", err
393 }
394 digest, err = listDigest(resp)
395 if err != nil {
396 return "", "", err
397 }
398 list, err := meta.ListAccessor(resp)
399 if err != nil {
400 return "", "", err
401 }
402 return digest, list.GetResourceVersion(), nil
403 }
404
405 func listDigest(list runtime.Object) (string, error) {
406 h := fnv.New64()
407 err := meta.EachListItem(list, func(obj runtime.Object) error {
408 objectMeta, err := meta.Accessor(obj)
409 if err != nil {
410 return err
411 }
412 err = addObjectToDigest(h, objectMeta)
413 if err != nil {
414 return err
415 }
416 return nil
417 })
418 if err != nil {
419 return "", err
420 }
421 return fmt.Sprintf("%x", h.Sum64()), nil
422 }
423
424 func addObjectToDigest(h hash.Hash64, objectMeta metav1.Object) error {
425 _, err := h.Write([]byte(objectMeta.GetNamespace()))
426 if err != nil {
427 return err
428 }
429 _, err = h.Write([]byte("/"))
430 if err != nil {
431 return err
432 }
433 _, err = h.Write([]byte(objectMeta.GetName()))
434 if err != nil {
435 return err
436 }
437 _, err = h.Write([]byte("/"))
438 if err != nil {
439 return err
440 }
441 _, err = h.Write([]byte(objectMeta.GetResourceVersion()))
442 if err != nil {
443 return err
444 }
445 return nil
446 }