| 1 |
/* |
| 2 |
Copyright 2016 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package etcd3 |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"errors" |
| 22 |
"fmt" |
| 23 |
"os" |
| 24 |
"strconv" |
| 25 |
"strings" |
| 26 |
"sync" |
| 27 |
"time" |
| 28 |
|
| 29 |
clientv3 "go.etcd.io/etcd/client/v3" |
| 30 |
grpccodes "google.golang.org/grpc/codes" |
| 31 |
grpcstatus "google.golang.org/grpc/status" |
| 32 |
|
| 33 |
apierrors "k8s.io/apimachinery/pkg/api/errors" |
| 34 |
"k8s.io/apimachinery/pkg/runtime" |
| 35 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 36 |
"k8s.io/apimachinery/pkg/util/wait" |
| 37 |
"k8s.io/apimachinery/pkg/watch" |
| 38 |
"k8s.io/apiserver/pkg/features" |
| 39 |
"k8s.io/apiserver/pkg/storage" |
| 40 |
"k8s.io/apiserver/pkg/storage/etcd3/metrics" |
| 41 |
"k8s.io/apiserver/pkg/storage/value" |
| 42 |
utilfeature "k8s.io/apiserver/pkg/util/feature" |
| 43 |
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" |
| 44 |
"k8s.io/klog/v2" |
| 45 |
) |
| 46 |
|
| 47 |
const ( |
| 48 |
// We have set a buffer in order to reduce times of context switches. |
| 49 |
incomingBufSize = 100 |
| 50 |
outgoingBufSize = 100 |
| 51 |
processEventConcurrency = 10 |
| 52 |
) |
| 53 |
|
| 54 |
// defaultWatcherMaxLimit is used to facilitate construction tests |
| 55 |
var defaultWatcherMaxLimit int64 = maxLimit |
| 56 |
|
| 57 |
// fatalOnDecodeError is used during testing to panic the server if watcher encounters a decoding error |
| 58 |
var fatalOnDecodeError = false |
| 59 |
|
| 60 |
func init() { |
| 61 |
// check to see if we are running in a test environment |
| 62 |
TestOnlySetFatalOnDecodeError(true) |
| 63 |
fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR")) |
| 64 |
} |
| 65 |
|
| 66 |
// TestOnlySetFatalOnDecodeError should only be used for cases where decode errors are expected and need to be tested. e.g. conversion webhooks. |
| 67 |
func TestOnlySetFatalOnDecodeError(b bool) { |
| 68 |
fatalOnDecodeError = b |
| 69 |
} |
| 70 |
|
| 71 |
type watcher struct { |
| 72 |
client *clientv3.Client |
| 73 |
codec runtime.Codec |
| 74 |
newFunc func() runtime.Object |
| 75 |
objectType string |
| 76 |
groupResource schema.GroupResource |
| 77 |
versioner storage.Versioner |
| 78 |
transformer value.Transformer |
| 79 |
getCurrentStorageRV func(context.Context) (uint64, error) |
| 80 |
} |
| 81 |
|
| 82 |
// watchChan implements watch.Interface. |
| 83 |
type watchChan struct { |
| 84 |
watcher *watcher |
| 85 |
key string |
| 86 |
initialRev int64 |
| 87 |
recursive bool |
| 88 |
progressNotify bool |
| 89 |
internalPred storage.SelectionPredicate |
| 90 |
ctx context.Context |
| 91 |
cancel context.CancelFunc |
| 92 |
incomingEventChan chan *event |
| 93 |
resultChan chan watch.Event |
| 94 |
errChan chan error |
| 95 |
} |
| 96 |
|
| 97 |
// Watch watches on a key and returns a watch.Interface that transfers relevant notifications. |
| 98 |
// If rev is zero, it will return the existing object(s) and then start watching from |
| 99 |
// the maximum revision+1 from returned objects. |
| 100 |
// If rev is non-zero, it will watch events happened after given revision. |
| 101 |
// If opts.Recursive is false, it watches on given key. |
| 102 |
// If opts.Recursive is true, it watches any children and directories under the key, excluding the root key itself. |
| 103 |
// pred must be non-nil. Only if opts.Predicate matches the change, it will be returned. |
| 104 |
func (w *watcher) Watch(ctx context.Context, key string, rev int64, opts storage.ListOptions) (watch.Interface, error) { |
| 105 |
if opts.Recursive && !strings.HasSuffix(key, "/") { |
| 106 |
key += "/" |
| 107 |
} |
| 108 |
if opts.ProgressNotify && w.newFunc == nil { |
| 109 |
return nil, apierrors.NewInternalError(errors.New("progressNotify for watch is unsupported by the etcd storage because no newFunc was provided")) |
| 110 |
} |
| 111 |
startWatchRV, err := w.getStartWatchResourceVersion(ctx, rev, opts) |
| 112 |
if err != nil { |
| 113 |
return nil, err |
| 114 |
} |
| 115 |
wc := w.createWatchChan(ctx, key, startWatchRV, opts.Recursive, opts.ProgressNotify, opts.Predicate) |
| 116 |
go wc.run(isInitialEventsEndBookmarkRequired(opts), areInitialEventsRequired(rev, opts)) |
| 117 |
|
| 118 |
// For etcd watch we don't have an easy way to answer whether the watch |
| 119 |
// has already caught up. So in the initial version (given that watchcache |
| 120 |
// is by default enabled for all resources but Events), we just deliver |
| 121 |
// the initialization signal immediately. Improving this will be explored |
| 122 |
// in the future. |
| 123 |
utilflowcontrol.WatchInitialized(ctx) |
| 124 |
|
| 125 |
return wc, nil |
| 126 |
} |
| 127 |
|
| 128 |
func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive, progressNotify bool, pred storage.SelectionPredicate) *watchChan { |
| 129 |
wc := &watchChan{ |
| 130 |
watcher: w, |
| 131 |
key: key, |
| 132 |
initialRev: rev, |
| 133 |
recursive: recursive, |
| 134 |
progressNotify: progressNotify, |
| 135 |
internalPred: pred, |
| 136 |
incomingEventChan: make(chan *event, incomingBufSize), |
| 137 |
resultChan: make(chan watch.Event, outgoingBufSize), |
| 138 |
errChan: make(chan error, 1), |
| 139 |
} |
| 140 |
if pred.Empty() { |
| 141 |
// The filter doesn't filter out any object. |
| 142 |
wc.internalPred = storage.Everything |
| 143 |
} |
| 144 |
wc.ctx, wc.cancel = context.WithCancel(ctx) |
| 145 |
return wc |
| 146 |
} |
| 147 |
|
| 148 |
// getStartWatchResourceVersion returns a ResourceVersion |
| 149 |
// the watch will be started from. |
| 150 |
// Depending on the input parameters the semantics of the returned ResourceVersion are: |
| 151 |
// - start at Exact (return resourceVersion) |
| 152 |
// - start at Most Recent (return an RV from etcd) |
| 153 |
func (w *watcher) getStartWatchResourceVersion(ctx context.Context, resourceVersion int64, opts storage.ListOptions) (int64, error) { |
| 154 |
if resourceVersion > 0 { |
| 155 |
return resourceVersion, nil |
| 156 |
} |
| 157 |
if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) { |
| 158 |
return 0, nil |
| 159 |
} |
| 160 |
if opts.SendInitialEvents == nil || *opts.SendInitialEvents { |
| 161 |
// note that when opts.SendInitialEvents=true |
| 162 |
// we will be issuing a consistent LIST request |
| 163 |
// against etcd followed by the special bookmark event |
| 164 |
return 0, nil |
| 165 |
} |
| 166 |
// at this point the clients is interested |
| 167 |
// only in getting a stream of events |
| 168 |
// starting at the MostRecent point in time (RV) |
| 169 |
currentStorageRV, err := w.getCurrentStorageRV(ctx) |
| 170 |
if err != nil { |
| 171 |
return 0, err |
| 172 |
} |
| 173 |
// currentStorageRV is taken from resp.Header.Revision (int64) |
| 174 |
// and cast to uint64, so it is safe to do reverse |
| 175 |
// at some point we should unify the interface but that |
| 176 |
// would require changing Versioner.UpdateList |
| 177 |
return int64(currentStorageRV), nil |
| 178 |
} |
| 179 |
|
| 180 |
// isInitialEventsEndBookmarkRequired since there is no way to directly set |
| 181 |
// opts.ProgressNotify from the API and the etcd3 impl doesn't support |
| 182 |
// notification for external clients we simply return initialEventsEndBookmarkRequired |
| 183 |
// to only send the bookmark event after the initial list call. |
| 184 |
// |
| 185 |
// see: https://github.com/kubernetes/kubernetes/issues/120348 |
| 186 |
func isInitialEventsEndBookmarkRequired(opts storage.ListOptions) bool { |
| 187 |
if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) { |
| 188 |
return false |
| 189 |
} |
| 190 |
return opts.SendInitialEvents != nil && *opts.SendInitialEvents && opts.Predicate.AllowWatchBookmarks |
| 191 |
} |
| 192 |
|
| 193 |
// areInitialEventsRequired returns true if all events from the etcd should be returned. |
| 194 |
func areInitialEventsRequired(resourceVersion int64, opts storage.ListOptions) bool { |
| 195 |
if opts.SendInitialEvents == nil && resourceVersion == 0 { |
| 196 |
return true // legacy case |
| 197 |
} |
| 198 |
if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) { |
| 199 |
return false |
| 200 |
} |
| 201 |
return opts.SendInitialEvents != nil && *opts.SendInitialEvents |
| 202 |
} |
| 203 |
|
| 204 |
type etcdError interface { |
| 205 |
Code() grpccodes.Code |
| 206 |
Error() string |
| 207 |
} |
| 208 |
|
| 209 |
type grpcError interface { |
| 210 |
GRPCStatus() *grpcstatus.Status |
| 211 |
} |
| 212 |
|
| 213 |
func isCancelError(err error) bool { |
| 214 |
if err == nil { |
| 215 |
return false |
| 216 |
} |
| 217 |
if err == context.Canceled { |
| 218 |
return true |
| 219 |
} |
| 220 |
if etcdErr, ok := err.(etcdError); ok && etcdErr.Code() == grpccodes.Canceled { |
| 221 |
return true |
| 222 |
} |
| 223 |
if grpcErr, ok := err.(grpcError); ok && grpcErr.GRPCStatus().Code() == grpccodes.Canceled { |
| 224 |
return true |
| 225 |
} |
| 226 |
return false |
| 227 |
} |
| 228 |
|
| 229 |
func (wc *watchChan) run(initialEventsEndBookmarkRequired, forceInitialEvents bool) { |
| 230 |
watchClosedCh := make(chan struct{}) |
| 231 |
go wc.startWatching(watchClosedCh, initialEventsEndBookmarkRequired, forceInitialEvents) |
| 232 |
|
| 233 |
var resultChanWG sync.WaitGroup |
| 234 |
wc.processEvents(&resultChanWG) |
| 235 |
|
| 236 |
select { |
| 237 |
case err := <-wc.errChan: |
| 238 |
if isCancelError(err) { |
| 239 |
break |
| 240 |
} |
| 241 |
errResult := transformErrorToEvent(err) |
| 242 |
if errResult != nil { |
| 243 |
// error result is guaranteed to be received by user before closing ResultChan. |
| 244 |
select { |
| 245 |
case wc.resultChan <- *errResult: |
| 246 |
case <-wc.ctx.Done(): // user has given up all results |
| 247 |
} |
| 248 |
} |
| 249 |
case <-watchClosedCh: |
| 250 |
case <-wc.ctx.Done(): // user cancel |
| 251 |
} |
| 252 |
|
| 253 |
// We use wc.ctx to reap all goroutines. Under whatever condition, we should stop them all. |
| 254 |
// It's fine to double cancel. |
| 255 |
wc.cancel() |
| 256 |
|
| 257 |
// we need to wait until resultChan wouldn't be used anymore |
| 258 |
resultChanWG.Wait() |
| 259 |
close(wc.resultChan) |
| 260 |
} |
| 261 |
|
| 262 |
func (wc *watchChan) Stop() { |
| 263 |
wc.cancel() |
| 264 |
} |
| 265 |
|
| 266 |
func (wc *watchChan) ResultChan() <-chan watch.Event { |
| 267 |
return wc.resultChan |
| 268 |
} |
| 269 |
|
| 270 |
func (wc *watchChan) RequestWatchProgress() error { |
| 271 |
return wc.watcher.client.RequestProgress(wc.ctx) |
| 272 |
} |
| 273 |
|
| 274 |
// sync tries to retrieve existing data and send them to process. |
| 275 |
// The revision to watch will be set to the revision in response. |
| 276 |
// All events sent will have isCreated=true |
| 277 |
func (wc *watchChan) sync() error { |
| 278 |
opts := []clientv3.OpOption{} |
| 279 |
if wc.recursive { |
| 280 |
opts = append(opts, clientv3.WithLimit(defaultWatcherMaxLimit)) |
| 281 |
rangeEnd := clientv3.GetPrefixRangeEnd(wc.key) |
| 282 |
opts = append(opts, clientv3.WithRange(rangeEnd)) |
| 283 |
} |
| 284 |
|
| 285 |
var err error |
| 286 |
var lastKey []byte |
| 287 |
var withRev int64 |
| 288 |
var getResp *clientv3.GetResponse |
| 289 |
|
| 290 |
metricsOp := "get" |
| 291 |
if wc.recursive { |
| 292 |
metricsOp = "list" |
| 293 |
} |
| 294 |
|
| 295 |
preparedKey := wc.key |
| 296 |
|
| 297 |
for { |
| 298 |
startTime := time.Now() |
| 299 |
getResp, err = wc.watcher.client.KV.Get(wc.ctx, preparedKey, opts...) |
| 300 |
metrics.RecordEtcdRequest(metricsOp, wc.watcher.groupResource.String(), err, startTime) |
| 301 |
if err != nil { |
| 302 |
return interpretListError(err, true, preparedKey, wc.key) |
| 303 |
} |
| 304 |
|
| 305 |
if len(getResp.Kvs) == 0 && getResp.More { |
| 306 |
return fmt.Errorf("no results were found, but etcd indicated there were more values remaining") |
| 307 |
} |
| 308 |
|
| 309 |
// send items from the response until no more results |
| 310 |
for i, kv := range getResp.Kvs { |
| 311 |
lastKey = kv.Key |
| 312 |
wc.sendEvent(parseKV(kv)) |
| 313 |
// free kv early. Long lists can take O(seconds) to decode. |
| 314 |
getResp.Kvs[i] = nil |
| 315 |
} |
| 316 |
|
| 317 |
if withRev == 0 { |
| 318 |
wc.initialRev = getResp.Header.Revision |
| 319 |
} |
| 320 |
|
| 321 |
// no more results remain |
| 322 |
if !getResp.More { |
| 323 |
return nil |
| 324 |
} |
| 325 |
|
| 326 |
preparedKey = string(lastKey) + "\x00" |
| 327 |
if withRev == 0 { |
| 328 |
withRev = getResp.Header.Revision |
| 329 |
opts = append(opts, clientv3.WithRev(withRev)) |
| 330 |
} |
| 331 |
} |
| 332 |
} |
| 333 |
|
| 334 |
func logWatchChannelErr(err error) { |
| 335 |
switch { |
| 336 |
case strings.Contains(err.Error(), "mvcc: required revision has been compacted"): |
| 337 |
// mvcc revision compaction which is regarded as warning, not error |
| 338 |
klog.Warningf("watch chan error: %v", err) |
| 339 |
case isCancelError(err): |
| 340 |
// expected when watches close, no need to log |
| 341 |
default: |
| 342 |
klog.Errorf("watch chan error: %v", err) |
| 343 |
} |
| 344 |
} |
| 345 |
|
| 346 |
// startWatching does: |
| 347 |
// - get current objects if initialRev=0; set initialRev to current rev |
| 348 |
// - watch on given key and send events to process. |
| 349 |
// |
| 350 |
// initialEventsEndBookmarkSent helps us keep track |
| 351 |
// of whether we have sent an annotated bookmark event. |
| 352 |
// |
| 353 |
// it's important to note that we don't |
| 354 |
// need to track the actual RV because |
| 355 |
// we only send the bookmark event |
| 356 |
// after the initial list call. |
| 357 |
// |
| 358 |
// when this variable is set to false, |
| 359 |
// it means we don't have any specific |
| 360 |
// preferences for delivering bookmark events. |
| 361 |
func (wc *watchChan) startWatching(watchClosedCh chan struct{}, initialEventsEndBookmarkRequired, forceInitialEvents bool) { |
| 362 |
if wc.initialRev > 0 && forceInitialEvents { |
| 363 |
currentStorageRV, err := wc.watcher.getCurrentStorageRV(wc.ctx) |
| 364 |
if err != nil { |
| 365 |
wc.sendError(err) |
| 366 |
return |
| 367 |
} |
| 368 |
if uint64(wc.initialRev) > currentStorageRV { |
| 369 |
wc.sendError(storage.NewTooLargeResourceVersionError(uint64(wc.initialRev), currentStorageRV, int(wait.Jitter(1*time.Second, 3).Seconds()))) |
| 370 |
return |
| 371 |
} |
| 372 |
} |
| 373 |
if forceInitialEvents { |
| 374 |
if err := wc.sync(); err != nil { |
| 375 |
klog.Errorf("failed to sync with latest state: %v", err) |
| 376 |
wc.sendError(err) |
| 377 |
return |
| 378 |
} |
| 379 |
} |
| 380 |
if initialEventsEndBookmarkRequired { |
| 381 |
wc.sendEvent(func() *event { |
| 382 |
e := progressNotifyEvent(wc.initialRev) |
| 383 |
e.isInitialEventsEndBookmark = true |
| 384 |
return e |
| 385 |
}()) |
| 386 |
} |
| 387 |
opts := []clientv3.OpOption{clientv3.WithRev(wc.initialRev + 1), clientv3.WithPrevKV()} |
| 388 |
if wc.recursive { |
| 389 |
opts = append(opts, clientv3.WithPrefix()) |
| 390 |
} |
| 391 |
if wc.progressNotify { |
| 392 |
opts = append(opts, clientv3.WithProgressNotify()) |
| 393 |
} |
| 394 |
wch := wc.watcher.client.Watch(wc.ctx, wc.key, opts...) |
| 395 |
for wres := range wch { |
| 396 |
if wres.Err() != nil { |
| 397 |
err := wres.Err() |
| 398 |
// If there is an error on server (e.g. compaction), the channel will return it before closed. |
| 399 |
logWatchChannelErr(err) |
| 400 |
wc.sendError(err) |
| 401 |
return |
| 402 |
} |
| 403 |
if wres.IsProgressNotify() { |
| 404 |
wc.sendEvent(progressNotifyEvent(wres.Header.GetRevision())) |
| 405 |
metrics.RecordEtcdBookmark(wc.watcher.groupResource.String()) |
| 406 |
continue |
| 407 |
} |
| 408 |
|
| 409 |
for _, e := range wres.Events { |
| 410 |
metrics.RecordEtcdEvent(wc.watcher.groupResource.String()) |
| 411 |
parsedEvent, err := parseEvent(e) |
| 412 |
if err != nil { |
| 413 |
logWatchChannelErr(err) |
| 414 |
wc.sendError(err) |
| 415 |
return |
| 416 |
} |
| 417 |
wc.sendEvent(parsedEvent) |
| 418 |
} |
| 419 |
} |
| 420 |
// When we come to this point, it's only possible that client side ends the watch. |
| 421 |
// e.g. cancel the context, close the client. |
| 422 |
// If this watch chan is broken and context isn't cancelled, other goroutines will still hang. |
| 423 |
// We should notify the main thread that this goroutine has exited. |
| 424 |
close(watchClosedCh) |
| 425 |
} |
| 426 |
|
| 427 |
// processEvents processes events from etcd watcher and sends results to resultChan. |
| 428 |
func (wc *watchChan) processEvents(wg *sync.WaitGroup) { |
| 429 |
if utilfeature.DefaultFeatureGate.Enabled(features.ConcurrentWatchObjectDecode) { |
| 430 |
wc.concurrentProcessEvents(wg) |
| 431 |
} else { |
| 432 |
wg.Add(1) |
| 433 |
go wc.serialProcessEvents(wg) |
| 434 |
} |
| 435 |
} |
| 436 |
func (wc *watchChan) serialProcessEvents(wg *sync.WaitGroup) { |
| 437 |
defer wg.Done() |
| 438 |
for { |
| 439 |
select { |
| 440 |
case e := <-wc.incomingEventChan: |
| 441 |
res, err := wc.transform(e) |
| 442 |
if err != nil { |
| 443 |
wc.sendError(err) |
| 444 |
return |
| 445 |
} |
| 446 |
|
| 447 |
if res == nil { |
| 448 |
continue |
| 449 |
} |
| 450 |
if len(wc.resultChan) == cap(wc.resultChan) { |
| 451 |
klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow dispatching events to watchers", "outgoingEvents", outgoingBufSize, "objectType", wc.watcher.objectType, "groupResource", wc.watcher.groupResource) |
| 452 |
} |
| 453 |
// If user couldn't receive results fast enough, we also block incoming events from watcher. |
| 454 |
// Because storing events in local will cause more memory usage. |
| 455 |
// The worst case would be closing the fast watcher. |
| 456 |
select { |
| 457 |
case wc.resultChan <- *res: |
| 458 |
case <-wc.ctx.Done(): |
| 459 |
return |
| 460 |
} |
| 461 |
case <-wc.ctx.Done(): |
| 462 |
return |
| 463 |
} |
| 464 |
} |
| 465 |
} |
| 466 |
|
| 467 |
func (wc *watchChan) concurrentProcessEvents(wg *sync.WaitGroup) { |
| 468 |
p := concurrentOrderedEventProcessing{ |
| 469 |
wc: wc, |
| 470 |
processingQueue: make(chan chan *processingResult, processEventConcurrency-1), |
| 471 |
|
| 472 |
objectType: wc.watcher.objectType, |
| 473 |
groupResource: wc.watcher.groupResource, |
| 474 |
} |
| 475 |
wg.Add(1) |
| 476 |
go func() { |
| 477 |
defer wg.Done() |
| 478 |
p.scheduleEventProcessing(wc.ctx, wg) |
| 479 |
}() |
| 480 |
wg.Add(1) |
| 481 |
go func() { |
| 482 |
defer wg.Done() |
| 483 |
p.collectEventProcessing(wc.ctx) |
| 484 |
}() |
| 485 |
} |
| 486 |
|
| 487 |
type processingResult struct { |
| 488 |
event *watch.Event |
| 489 |
err error |
| 490 |
} |
| 491 |
|
| 492 |
type concurrentOrderedEventProcessing struct { |
| 493 |
wc *watchChan |
| 494 |
|
| 495 |
processingQueue chan chan *processingResult |
| 496 |
// Metadata for logging |
| 497 |
objectType string |
| 498 |
groupResource schema.GroupResource |
| 499 |
} |
| 500 |
|
| 501 |
func (p *concurrentOrderedEventProcessing) scheduleEventProcessing(ctx context.Context, wg *sync.WaitGroup) { |
| 502 |
var e *event |
| 503 |
for { |
| 504 |
select { |
| 505 |
case <-ctx.Done(): |
| 506 |
return |
| 507 |
case e = <-p.wc.incomingEventChan: |
| 508 |
} |
| 509 |
processingResponse := make(chan *processingResult, 1) |
| 510 |
select { |
| 511 |
case <-ctx.Done(): |
| 512 |
return |
| 513 |
case p.processingQueue <- processingResponse: |
| 514 |
} |
| 515 |
wg.Add(1) |
| 516 |
go func(e *event, response chan<- *processingResult) { |
| 517 |
defer wg.Done() |
| 518 |
responseEvent, err := p.wc.transform(e) |
| 519 |
select { |
| 520 |
case <-ctx.Done(): |
| 521 |
case response <- &processingResult{event: responseEvent, err: err}: |
| 522 |
} |
| 523 |
}(e, processingResponse) |
| 524 |
} |
| 525 |
} |
| 526 |
|
| 527 |
func (p *concurrentOrderedEventProcessing) collectEventProcessing(ctx context.Context) { |
| 528 |
var processingResponse chan *processingResult |
| 529 |
var r *processingResult |
| 530 |
for { |
| 531 |
select { |
| 532 |
case <-ctx.Done(): |
| 533 |
return |
| 534 |
case processingResponse = <-p.processingQueue: |
| 535 |
} |
| 536 |
select { |
| 537 |
case <-ctx.Done(): |
| 538 |
return |
| 539 |
case r = <-processingResponse: |
| 540 |
} |
| 541 |
if r.err != nil { |
| 542 |
p.wc.sendError(r.err) |
| 543 |
return |
| 544 |
} |
| 545 |
if r.event == nil { |
| 546 |
continue |
| 547 |
} |
| 548 |
if len(p.wc.resultChan) == cap(p.wc.resultChan) { |
| 549 |
klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow dispatching events to watchers", "outgoingEvents", outgoingBufSize, "objectType", p.wc.watcher.objectType, "groupResource", p.wc.watcher.groupResource) |
| 550 |
} |
| 551 |
// If user couldn't receive results fast enough, we also block incoming events from watcher. |
| 552 |
// Because storing events in local will cause more memory usage. |
| 553 |
// The worst case would be closing the fast watcher. |
| 554 |
select { |
| 555 |
case p.wc.resultChan <- *r.event: |
| 556 |
case <-p.wc.ctx.Done(): |
| 557 |
return |
| 558 |
} |
| 559 |
} |
| 560 |
} |
| 561 |
|
| 562 |
func (wc *watchChan) filter(obj runtime.Object) bool { |
| 563 |
if wc.internalPred.Empty() { |
| 564 |
return true |
| 565 |
} |
| 566 |
matched, err := wc.internalPred.Matches(obj) |
| 567 |
return err == nil && matched |
| 568 |
} |
| 569 |
|
| 570 |
func (wc *watchChan) acceptAll() bool { |
| 571 |
return wc.internalPred.Empty() |
| 572 |
} |
| 573 |
|
| 574 |
// transform transforms an event into a result for user if not filtered. |
| 575 |
func (wc *watchChan) transform(e *event) (res *watch.Event, err error) { |
| 576 |
curObj, oldObj, err := wc.prepareObjs(e) |
| 577 |
if err != nil { |
| 578 |
klog.Errorf("failed to prepare current and previous objects: %v", err) |
| 579 |
return nil, err |
| 580 |
} |
| 581 |
|
| 582 |
switch { |
| 583 |
case e.isProgressNotify: |
| 584 |
object := wc.watcher.newFunc() |
| 585 |
if err := wc.watcher.versioner.UpdateObject(object, uint64(e.rev)); err != nil { |
| 586 |
klog.Errorf("failed to propagate object version: %v", err) |
| 587 |
return nil, fmt.Errorf("failed to propagate object resource version: %w", err) |
| 588 |
} |
| 589 |
if e.isInitialEventsEndBookmark { |
| 590 |
if err := storage.AnnotateInitialEventsEndBookmark(object); err != nil { |
| 591 |
return nil, fmt.Errorf("error while accessing object's metadata gr: %v, type: %v, obj: %#v, err: %w", wc.watcher.groupResource, wc.watcher.objectType, object, err) |
| 592 |
} |
| 593 |
} |
| 594 |
res = &watch.Event{ |
| 595 |
Type: watch.Bookmark, |
| 596 |
Object: object, |
| 597 |
} |
| 598 |
case e.isDeleted: |
| 599 |
if !wc.filter(oldObj) { |
| 600 |
return nil, nil |
| 601 |
} |
| 602 |
res = &watch.Event{ |
| 603 |
Type: watch.Deleted, |
| 604 |
Object: oldObj, |
| 605 |
} |
| 606 |
case e.isCreated: |
| 607 |
if !wc.filter(curObj) { |
| 608 |
return nil, nil |
| 609 |
} |
| 610 |
res = &watch.Event{ |
| 611 |
Type: watch.Added, |
| 612 |
Object: curObj, |
| 613 |
} |
| 614 |
default: |
| 615 |
if wc.acceptAll() { |
| 616 |
res = &watch.Event{ |
| 617 |
Type: watch.Modified, |
| 618 |
Object: curObj, |
| 619 |
} |
| 620 |
return res, nil |
| 621 |
} |
| 622 |
curObjPasses := wc.filter(curObj) |
| 623 |
oldObjPasses := wc.filter(oldObj) |
| 624 |
switch { |
| 625 |
case curObjPasses && oldObjPasses: |
| 626 |
res = &watch.Event{ |
| 627 |
Type: watch.Modified, |
| 628 |
Object: curObj, |
| 629 |
} |
| 630 |
case curObjPasses && !oldObjPasses: |
| 631 |
res = &watch.Event{ |
| 632 |
Type: watch.Added, |
| 633 |
Object: curObj, |
| 634 |
} |
| 635 |
case !curObjPasses && oldObjPasses: |
| 636 |
res = &watch.Event{ |
| 637 |
Type: watch.Deleted, |
| 638 |
Object: oldObj, |
| 639 |
} |
| 640 |
} |
| 641 |
} |
| 642 |
return res, nil |
| 643 |
} |
| 644 |
|
| 645 |
func transformErrorToEvent(err error) *watch.Event { |
| 646 |
err = interpretWatchError(err) |
| 647 |
if _, ok := err.(apierrors.APIStatus); !ok { |
| 648 |
err = apierrors.NewInternalError(err) |
| 649 |
} |
| 650 |
status := err.(apierrors.APIStatus).Status() |
| 651 |
return &watch.Event{ |
| 652 |
Type: watch.Error, |
| 653 |
Object: &status, |
| 654 |
} |
| 655 |
} |
| 656 |
|
| 657 |
func (wc *watchChan) sendError(err error) { |
| 658 |
select { |
| 659 |
case wc.errChan <- err: |
| 660 |
case <-wc.ctx.Done(): |
| 661 |
} |
| 662 |
} |
| 663 |
|
| 664 |
func (wc *watchChan) sendEvent(e *event) { |
| 665 |
if len(wc.incomingEventChan) == incomingBufSize { |
| 666 |
klog.V(3).InfoS("Fast watcher, slow processing. Probably caused by slow decoding, user not receiving fast, or other processing logic", "incomingEvents", incomingBufSize, "objectType", wc.watcher.objectType, "groupResource", wc.watcher.groupResource) |
| 667 |
} |
| 668 |
select { |
| 669 |
case wc.incomingEventChan <- e: |
| 670 |
case <-wc.ctx.Done(): |
| 671 |
} |
| 672 |
} |
| 673 |
|
| 674 |
func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtime.Object, err error) { |
| 675 |
if e.isProgressNotify { |
| 676 |
// progressNotify events doesn't contain neither current nor previous object version, |
| 677 |
return nil, nil, nil |
| 678 |
} |
| 679 |
|
| 680 |
if !e.isDeleted { |
| 681 |
data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.value, authenticatedDataString(e.key)) |
| 682 |
if err != nil { |
| 683 |
return nil, nil, err |
| 684 |
} |
| 685 |
curObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev) |
| 686 |
if err != nil { |
| 687 |
return nil, nil, err |
| 688 |
} |
| 689 |
} |
| 690 |
// We need to decode prevValue, only if this is deletion event or |
| 691 |
// the underlying filter doesn't accept all objects (otherwise we |
| 692 |
// know that the filter for previous object will return true and |
| 693 |
// we need the object only to compute whether it was filtered out |
| 694 |
// before). |
| 695 |
if len(e.prevValue) > 0 && (e.isDeleted || !wc.acceptAll()) { |
| 696 |
data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.prevValue, authenticatedDataString(e.key)) |
| 697 |
if err != nil { |
| 698 |
return nil, nil, wc.watcher.transformIfCorruptObjectError(e, err) |
| 699 |
} |
| 700 |
// Note that this sends the *old* object with the etcd revision for the time at |
| 701 |
// which it gets deleted. |
| 702 |
oldObj, err = decodeObj(wc.watcher.codec, wc.watcher.versioner, data, e.rev) |
| 703 |
if err != nil { |
| 704 |
return nil, nil, wc.watcher.transformIfCorruptObjectError(e, err) |
| 705 |
} |
| 706 |
} |
| 707 |
return curObj, oldObj, nil |
| 708 |
} |
| 709 |
|
| 710 |
type corruptObjectDeletedError struct { |
| 711 |
err error |
| 712 |
} |
| 713 |
|
| 714 |
func (e *corruptObjectDeletedError) Error() string { |
| 715 |
return fmt.Sprintf("saw a DELETED event, but object data is corrupt - %v", e.err) |
| 716 |
} |
| 717 |
func (e *corruptObjectDeletedError) Unwrap() error { return e.err } |
| 718 |
|
| 719 |
func (w *watcher) transformIfCorruptObjectError(e *event, err error) error { |
| 720 |
var corruptObjErr *corruptObjectError |
| 721 |
if !e.isDeleted || !errors.As(err, &corruptObjErr) { |
| 722 |
return err |
| 723 |
} |
| 724 |
|
| 725 |
// if we are here it means we received a DELETED event but the object |
| 726 |
// associated with it is corrupt because we failed to transform or |
| 727 |
// decode the data associated with the object. |
| 728 |
// wrap the original error so we can send a proper watch Error event. |
| 729 |
return &corruptObjectDeletedError{err: corruptObjErr} |
| 730 |
} |
| 731 |
|
| 732 |
func decodeObj(codec runtime.Codec, versioner storage.Versioner, data []byte, rev int64) (_ runtime.Object, err error) { |
| 733 |
obj, err := runtime.Decode(codec, []byte(data)) |
| 734 |
if err != nil { |
| 735 |
if fatalOnDecodeError { |
| 736 |
// we are running in a test environment and thus an |
| 737 |
// error here is due to a coder mistake if the defer |
| 738 |
// does not catch it |
| 739 |
panic(err) |
| 740 |
} |
| 741 |
return nil, err |
| 742 |
} |
| 743 |
// ensure resource version is set on the object we load from etcd |
| 744 |
if err := versioner.UpdateObject(obj, uint64(rev)); err != nil { |
| 745 |
return nil, fmt.Errorf("failure to version api object (%d) %#v: %v", rev, obj, err) |
| 746 |
} |
| 747 |
return obj, nil |
| 748 |
} |