| 1 |
/* |
| 2 |
Copyright 2014 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package cacher |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"strconv" |
| 23 |
"strings" |
| 24 |
"testing" |
| 25 |
"time" |
| 26 |
|
| 27 |
"github.com/stretchr/testify/assert" |
| 28 |
"github.com/stretchr/testify/require" |
| 29 |
|
| 30 |
v1 "k8s.io/api/core/v1" |
| 31 |
apiequality "k8s.io/apimachinery/pkg/api/equality" |
| 32 |
"k8s.io/apimachinery/pkg/api/errors" |
| 33 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 34 |
"k8s.io/apimachinery/pkg/fields" |
| 35 |
"k8s.io/apimachinery/pkg/labels" |
| 36 |
"k8s.io/apimachinery/pkg/runtime" |
| 37 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 38 |
"k8s.io/apimachinery/pkg/util/wait" |
| 39 |
"k8s.io/apimachinery/pkg/watch" |
| 40 |
"k8s.io/apiserver/pkg/features" |
| 41 |
"k8s.io/apiserver/pkg/storage" |
| 42 |
"k8s.io/apiserver/pkg/storage/cacher/metrics" |
| 43 |
"k8s.io/apiserver/pkg/storage/cacher/progress" |
| 44 |
utilfeature "k8s.io/apiserver/pkg/util/feature" |
| 45 |
"k8s.io/client-go/tools/cache" |
| 46 |
featuregatetesting "k8s.io/component-base/featuregate/testing" |
| 47 |
k8smetrics "k8s.io/component-base/metrics" |
| 48 |
"k8s.io/component-base/metrics/testutil" |
| 49 |
"k8s.io/utils/clock" |
| 50 |
testingclock "k8s.io/utils/clock/testing" |
| 51 |
) |
| 52 |
|
| 53 |
func makeTestPod(name string, resourceVersion uint64) *v1.Pod { |
| 54 |
return makeTestPodDetails(name, resourceVersion, "some-node", map[string]string{"k8s-app": "my-app"}) |
| 55 |
} |
| 56 |
|
| 57 |
func makeTestPodDetails(name string, resourceVersion uint64, nodeName string, labels map[string]string) *v1.Pod { |
| 58 |
return &v1.Pod{ |
| 59 |
ObjectMeta: metav1.ObjectMeta{ |
| 60 |
Namespace: "ns", |
| 61 |
Name: name, |
| 62 |
ResourceVersion: strconv.FormatUint(resourceVersion, 10), |
| 63 |
Labels: labels, |
| 64 |
}, |
| 65 |
Spec: v1.PodSpec{ |
| 66 |
NodeName: nodeName, |
| 67 |
}, |
| 68 |
} |
| 69 |
} |
| 70 |
|
| 71 |
func makeTestStoreElement(pod *v1.Pod) *storeElement { |
| 72 |
return &storeElement{ |
| 73 |
Key: "prefix/ns/" + pod.Name, |
| 74 |
Object: pod, |
| 75 |
Labels: labels.Set(pod.Labels), |
| 76 |
Fields: fields.Set{"spec.nodeName": pod.Spec.NodeName}, |
| 77 |
} |
| 78 |
} |
| 79 |
|
| 80 |
type testWatchCache struct { |
| 81 |
*watchCache |
| 82 |
|
| 83 |
bookmarkRevision chan int64 |
| 84 |
stopCh chan struct{} |
| 85 |
} |
| 86 |
|
| 87 |
func (w *testWatchCache) getAllEventsSince(resourceVersion uint64, opts storage.ListOptions) ([]*watchCacheEvent, error) { |
| 88 |
cacheInterval, err := w.getCacheIntervalForEvents(resourceVersion, opts) |
| 89 |
if err != nil { |
| 90 |
return nil, err |
| 91 |
} |
| 92 |
|
| 93 |
result := []*watchCacheEvent{} |
| 94 |
for { |
| 95 |
event, err := cacheInterval.Next() |
| 96 |
if err != nil { |
| 97 |
return nil, err |
| 98 |
} |
| 99 |
if event == nil { |
| 100 |
break |
| 101 |
} |
| 102 |
result = append(result, event) |
| 103 |
} |
| 104 |
|
| 105 |
return result, nil |
| 106 |
} |
| 107 |
|
| 108 |
func (w *testWatchCache) getCacheIntervalForEvents(resourceVersion uint64, opts storage.ListOptions) (*watchCacheInterval, error) { |
| 109 |
w.RLock() |
| 110 |
defer w.RUnlock() |
| 111 |
|
| 112 |
return w.getAllEventsSinceLocked(resourceVersion, "", opts) |
| 113 |
} |
| 114 |
|
| 115 |
// newTestWatchCache just adds a fake clock. |
| 116 |
func newTestWatchCache(capacity int, eventFreshDuration time.Duration, indexers *cache.Indexers) *testWatchCache { |
| 117 |
keyFunc := func(obj runtime.Object) (string, error) { |
| 118 |
return storage.NamespaceKeyFunc("prefix", obj) |
| 119 |
} |
| 120 |
getAttrsFunc := func(obj runtime.Object) (labels.Set, fields.Set, error) { |
| 121 |
pod, ok := obj.(*v1.Pod) |
| 122 |
if !ok { |
| 123 |
return nil, nil, fmt.Errorf("not a pod") |
| 124 |
} |
| 125 |
return labels.Set(pod.Labels), fields.Set{"spec.nodeName": pod.Spec.NodeName}, nil |
| 126 |
} |
| 127 |
versioner := storage.APIObjectVersioner{} |
| 128 |
mockHandler := func(*watchCacheEvent) {} |
| 129 |
wc := &testWatchCache{} |
| 130 |
wc.bookmarkRevision = make(chan int64, 1) |
| 131 |
wc.stopCh = make(chan struct{}) |
| 132 |
pr := progress.NewConditionalProgressRequester(wc.RequestWatchProgress, &immediateTickerFactory{}, nil) |
| 133 |
go pr.Run(wc.stopCh) |
| 134 |
wc.watchCache = newWatchCache(keyFunc, mockHandler, getAttrsFunc, versioner, indexers, testingclock.NewFakeClock(time.Now()), eventFreshDuration, schema.GroupResource{Resource: "pods"}, pr) |
| 135 |
// To preserve behavior of tests that assume a given capacity, |
| 136 |
// resize it to th expected size. |
| 137 |
wc.capacity = capacity |
| 138 |
wc.cache = make([]*watchCacheEvent, capacity) |
| 139 |
wc.lowerBoundCapacity = min(capacity, defaultLowerBoundCapacity) |
| 140 |
wc.upperBoundCapacity = max(capacity, defaultUpperBoundCapacity) |
| 141 |
|
| 142 |
return wc |
| 143 |
} |
| 144 |
|
| 145 |
type immediateTickerFactory struct{} |
| 146 |
|
| 147 |
func (t *immediateTickerFactory) NewTimer(d time.Duration) clock.Timer { |
| 148 |
timer := immediateTicker{ |
| 149 |
c: make(chan time.Time), |
| 150 |
} |
| 151 |
timer.Reset(d) |
| 152 |
return &timer |
| 153 |
} |
| 154 |
|
| 155 |
type immediateTicker struct { |
| 156 |
c chan time.Time |
| 157 |
} |
| 158 |
|
| 159 |
func (t *immediateTicker) Reset(d time.Duration) (active bool) { |
| 160 |
select { |
| 161 |
case <-t.c: |
| 162 |
active = true |
| 163 |
default: |
| 164 |
} |
| 165 |
go func() { |
| 166 |
t.c <- time.Now() |
| 167 |
}() |
| 168 |
return active |
| 169 |
} |
| 170 |
|
| 171 |
func (t *immediateTicker) C() <-chan time.Time { |
| 172 |
return t.c |
| 173 |
} |
| 174 |
|
| 175 |
func (t *immediateTicker) Stop() bool { |
| 176 |
select { |
| 177 |
case <-t.c: |
| 178 |
return true |
| 179 |
default: |
| 180 |
return false |
| 181 |
} |
| 182 |
} |
| 183 |
|
| 184 |
func (w *testWatchCache) RequestWatchProgress(ctx context.Context) error { |
| 185 |
go func() { |
| 186 |
select { |
| 187 |
case rev := <-w.bookmarkRevision: |
| 188 |
w.UpdateResourceVersion(fmt.Sprintf("%d", rev)) |
| 189 |
case <-ctx.Done(): |
| 190 |
return |
| 191 |
} |
| 192 |
}() |
| 193 |
return nil |
| 194 |
} |
| 195 |
|
| 196 |
func (w *testWatchCache) Stop() { |
| 197 |
close(w.stopCh) |
| 198 |
} |
| 199 |
|
| 200 |
func TestWatchCacheBasic(t *testing.T) { |
| 201 |
store := newTestWatchCache(2, DefaultEventFreshDuration, &cache.Indexers{}) |
| 202 |
defer store.Stop() |
| 203 |
|
| 204 |
// Test Add/Update/Delete. |
| 205 |
pod1 := makeTestPod("pod", 1) |
| 206 |
if err := store.Add(pod1); err != nil { |
| 207 |
t.Errorf("unexpected error: %v", err) |
| 208 |
} |
| 209 |
if item, ok, _ := store.Get(pod1); !ok { |
| 210 |
t.Errorf("didn't find pod") |
| 211 |
} else { |
| 212 |
expected := makeTestStoreElement(makeTestPod("pod", 1)) |
| 213 |
if !apiequality.Semantic.DeepEqual(expected, item) { |
| 214 |
t.Errorf("expected %v, got %v", expected, item) |
| 215 |
} |
| 216 |
} |
| 217 |
pod2 := makeTestPod("pod", 2) |
| 218 |
if err := store.Update(pod2); err != nil { |
| 219 |
t.Errorf("unexpected error: %v", err) |
| 220 |
} |
| 221 |
if item, ok, _ := store.Get(pod2); !ok { |
| 222 |
t.Errorf("didn't find pod") |
| 223 |
} else { |
| 224 |
expected := makeTestStoreElement(makeTestPod("pod", 2)) |
| 225 |
if !apiequality.Semantic.DeepEqual(expected, item) { |
| 226 |
t.Errorf("expected %v, got %v", expected, item) |
| 227 |
} |
| 228 |
} |
| 229 |
pod3 := makeTestPod("pod", 3) |
| 230 |
if err := store.Delete(pod3); err != nil { |
| 231 |
t.Errorf("unexpected error: %v", err) |
| 232 |
} |
| 233 |
if _, ok, _ := store.Get(pod3); ok { |
| 234 |
t.Errorf("found pod") |
| 235 |
} |
| 236 |
|
| 237 |
// Test List. |
| 238 |
store.Add(makeTestPod("pod1", 4)) |
| 239 |
store.Add(makeTestPod("pod2", 5)) |
| 240 |
store.Add(makeTestPod("pod3", 6)) |
| 241 |
{ |
| 242 |
expected := map[string]storeElement{ |
| 243 |
"prefix/ns/pod1": *makeTestStoreElement(makeTestPod("pod1", 4)), |
| 244 |
"prefix/ns/pod2": *makeTestStoreElement(makeTestPod("pod2", 5)), |
| 245 |
"prefix/ns/pod3": *makeTestStoreElement(makeTestPod("pod3", 6)), |
| 246 |
} |
| 247 |
items := make(map[string]storeElement) |
| 248 |
for _, item := range store.List() { |
| 249 |
elem := item.(*storeElement) |
| 250 |
items[elem.Key] = *elem |
| 251 |
} |
| 252 |
if !apiequality.Semantic.DeepEqual(expected, items) { |
| 253 |
t.Errorf("expected %v, got %v", expected, items) |
| 254 |
} |
| 255 |
} |
| 256 |
|
| 257 |
// Test Replace. |
| 258 |
store.Replace([]interface{}{ |
| 259 |
makeTestPod("pod4", 7), |
| 260 |
makeTestPod("pod5", 8), |
| 261 |
}, "8") |
| 262 |
{ |
| 263 |
expected := map[string]storeElement{ |
| 264 |
"prefix/ns/pod4": *makeTestStoreElement(makeTestPod("pod4", 7)), |
| 265 |
"prefix/ns/pod5": *makeTestStoreElement(makeTestPod("pod5", 8)), |
| 266 |
} |
| 267 |
items := make(map[string]storeElement) |
| 268 |
for _, item := range store.List() { |
| 269 |
elem := item.(*storeElement) |
| 270 |
items[elem.Key] = *elem |
| 271 |
} |
| 272 |
if !apiequality.Semantic.DeepEqual(expected, items) { |
| 273 |
t.Errorf("expected %v, got %v", expected, items) |
| 274 |
} |
| 275 |
} |
| 276 |
} |
| 277 |
|
| 278 |
func TestEvents(t *testing.T) { |
| 279 |
store := newTestWatchCache(5, DefaultEventFreshDuration, &cache.Indexers{}) |
| 280 |
defer store.Stop() |
| 281 |
|
| 282 |
// no dynamic-size cache to fit old tests. |
| 283 |
store.lowerBoundCapacity = 5 |
| 284 |
store.upperBoundCapacity = 5 |
| 285 |
|
| 286 |
store.Add(makeTestPod("pod", 3)) |
| 287 |
|
| 288 |
// Test for Added event. |
| 289 |
{ |
| 290 |
_, err := store.getAllEventsSince(1, storage.ListOptions{Predicate: storage.Everything}) |
| 291 |
if err == nil { |
| 292 |
t.Errorf("expected error too old") |
| 293 |
} |
| 294 |
if _, ok := err.(*errors.StatusError); !ok { |
| 295 |
t.Errorf("expected error to be of type StatusError") |
| 296 |
} |
| 297 |
} |
| 298 |
{ |
| 299 |
result, err := store.getAllEventsSince(2, storage.ListOptions{Predicate: storage.Everything}) |
| 300 |
if err != nil { |
| 301 |
t.Errorf("unexpected error: %v", err) |
| 302 |
} |
| 303 |
if len(result) != 1 { |
| 304 |
t.Fatalf("unexpected events: %v", result) |
| 305 |
} |
| 306 |
if result[0].Type != watch.Added { |
| 307 |
t.Errorf("unexpected event type: %v", result[0].Type) |
| 308 |
} |
| 309 |
pod := makeTestPod("pod", uint64(3)) |
| 310 |
if !apiequality.Semantic.DeepEqual(pod, result[0].Object) { |
| 311 |
t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod) |
| 312 |
} |
| 313 |
if result[0].PrevObject != nil { |
| 314 |
t.Errorf("unexpected item: %v", result[0].PrevObject) |
| 315 |
} |
| 316 |
} |
| 317 |
|
| 318 |
store.Update(makeTestPod("pod", 4)) |
| 319 |
store.Update(makeTestPod("pod", 5)) |
| 320 |
|
| 321 |
// Test with not full cache. |
| 322 |
{ |
| 323 |
_, err := store.getAllEventsSince(1, storage.ListOptions{Predicate: storage.Everything}) |
| 324 |
if err == nil { |
| 325 |
t.Errorf("expected error too old") |
| 326 |
} |
| 327 |
} |
| 328 |
{ |
| 329 |
result, err := store.getAllEventsSince(3, storage.ListOptions{Predicate: storage.Everything}) |
| 330 |
if err != nil { |
| 331 |
t.Errorf("unexpected error: %v", err) |
| 332 |
} |
| 333 |
if len(result) != 2 { |
| 334 |
t.Fatalf("unexpected events: %v", result) |
| 335 |
} |
| 336 |
for i := 0; i < 2; i++ { |
| 337 |
if result[i].Type != watch.Modified { |
| 338 |
t.Errorf("unexpected event type: %v", result[i].Type) |
| 339 |
} |
| 340 |
pod := makeTestPod("pod", uint64(i+4)) |
| 341 |
if !apiequality.Semantic.DeepEqual(pod, result[i].Object) { |
| 342 |
t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod) |
| 343 |
} |
| 344 |
prevPod := makeTestPod("pod", uint64(i+3)) |
| 345 |
if !apiequality.Semantic.DeepEqual(prevPod, result[i].PrevObject) { |
| 346 |
t.Errorf("unexpected item: %v, expected: %v", result[i].PrevObject, prevPod) |
| 347 |
} |
| 348 |
} |
| 349 |
} |
| 350 |
|
| 351 |
for i := 6; i < 10; i++ { |
| 352 |
store.Update(makeTestPod("pod", uint64(i))) |
| 353 |
} |
| 354 |
|
| 355 |
// Test with full cache - there should be elements from 5 to 9. |
| 356 |
{ |
| 357 |
_, err := store.getAllEventsSince(3, storage.ListOptions{Predicate: storage.Everything}) |
| 358 |
if err == nil { |
| 359 |
t.Errorf("expected error too old") |
| 360 |
} |
| 361 |
} |
| 362 |
{ |
| 363 |
result, err := store.getAllEventsSince(4, storage.ListOptions{Predicate: storage.Everything}) |
| 364 |
if err != nil { |
| 365 |
t.Errorf("unexpected error: %v", err) |
| 366 |
} |
| 367 |
if len(result) != 5 { |
| 368 |
t.Fatalf("unexpected events: %v", result) |
| 369 |
} |
| 370 |
for i := 0; i < 5; i++ { |
| 371 |
pod := makeTestPod("pod", uint64(i+5)) |
| 372 |
if !apiequality.Semantic.DeepEqual(pod, result[i].Object) { |
| 373 |
t.Errorf("unexpected item: %v, expected: %v", result[i].Object, pod) |
| 374 |
} |
| 375 |
} |
| 376 |
} |
| 377 |
|
| 378 |
// Test for delete event. |
| 379 |
store.Delete(makeTestPod("pod", uint64(10))) |
| 380 |
|
| 381 |
{ |
| 382 |
result, err := store.getAllEventsSince(9, storage.ListOptions{Predicate: storage.Everything}) |
| 383 |
if err != nil { |
| 384 |
t.Errorf("unexpected error: %v", err) |
| 385 |
} |
| 386 |
if len(result) != 1 { |
| 387 |
t.Fatalf("unexpected events: %v", result) |
| 388 |
} |
| 389 |
if result[0].Type != watch.Deleted { |
| 390 |
t.Errorf("unexpected event type: %v", result[0].Type) |
| 391 |
} |
| 392 |
pod := makeTestPod("pod", uint64(10)) |
| 393 |
if !apiequality.Semantic.DeepEqual(pod, result[0].Object) { |
| 394 |
t.Errorf("unexpected item: %v, expected: %v", result[0].Object, pod) |
| 395 |
} |
| 396 |
prevPod := makeTestPod("pod", uint64(9)) |
| 397 |
if !apiequality.Semantic.DeepEqual(prevPod, result[0].PrevObject) { |
| 398 |
t.Errorf("unexpected item: %v, expected: %v", result[0].PrevObject, prevPod) |
| 399 |
} |
| 400 |
} |
| 401 |
} |
| 402 |
|
| 403 |
func TestMarker(t *testing.T) { |
| 404 |
store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{}) |
| 405 |
defer store.Stop() |
| 406 |
|
| 407 |
// First thing that is called when propagated from storage is Replace. |
| 408 |
store.Replace([]interface{}{ |
| 409 |
makeTestPod("pod1", 5), |
| 410 |
makeTestPod("pod2", 9), |
| 411 |
}, "9") |
| 412 |
|
| 413 |
_, err := store.getAllEventsSince(8, storage.ListOptions{Predicate: storage.Everything}) |
| 414 |
if err == nil || !strings.Contains(err.Error(), "too old resource version") { |
| 415 |
t.Errorf("unexpected error: %v", err) |
| 416 |
} |
| 417 |
// Getting events from 8 should return no events, |
| 418 |
// even though there is a marker there. |
| 419 |
result, err := store.getAllEventsSince(9, storage.ListOptions{Predicate: storage.Everything}) |
| 420 |
if err != nil { |
| 421 |
t.Fatalf("unexpected error: %v", err) |
| 422 |
} |
| 423 |
if len(result) != 0 { |
| 424 |
t.Errorf("unexpected result: %#v, expected no events", result) |
| 425 |
} |
| 426 |
|
| 427 |
pod := makeTestPod("pods", 12) |
| 428 |
store.Add(pod) |
| 429 |
// Getting events from 8 should still work and return one event. |
| 430 |
result, err = store.getAllEventsSince(9, storage.ListOptions{Predicate: storage.Everything}) |
| 431 |
if err != nil { |
| 432 |
t.Fatalf("unexpected error: %v", err) |
| 433 |
} |
| 434 |
if len(result) != 1 || !apiequality.Semantic.DeepEqual(result[0].Object, pod) { |
| 435 |
t.Errorf("unexpected result: %#v, expected %v", result, pod) |
| 436 |
} |
| 437 |
} |
| 438 |
|
| 439 |
func TestWaitUntilFreshAndList(t *testing.T) { |
| 440 |
ctx := context.Background() |
| 441 |
store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{ |
| 442 |
"l:label": func(obj interface{}) ([]string, error) { |
| 443 |
pod, ok := obj.(*v1.Pod) |
| 444 |
if !ok { |
| 445 |
return nil, fmt.Errorf("not a pod %#v", obj) |
| 446 |
} |
| 447 |
if value, ok := pod.Labels["label"]; ok { |
| 448 |
return []string{value}, nil |
| 449 |
} |
| 450 |
return nil, nil |
| 451 |
}, |
| 452 |
"f:spec.nodeName": func(obj interface{}) ([]string, error) { |
| 453 |
pod, ok := obj.(*v1.Pod) |
| 454 |
if !ok { |
| 455 |
return nil, fmt.Errorf("not a pod %#v", obj) |
| 456 |
} |
| 457 |
return []string{pod.Spec.NodeName}, nil |
| 458 |
}, |
| 459 |
}) |
| 460 |
defer store.Stop() |
| 461 |
// In background, update the store. |
| 462 |
go func() { |
| 463 |
store.Add(makeTestPodDetails("pod1", 2, "node1", map[string]string{"label": "value1"})) |
| 464 |
store.Add(makeTestPodDetails("pod2", 3, "node1", map[string]string{"label": "value1"})) |
| 465 |
store.Add(makeTestPodDetails("pod3", 5, "node2", map[string]string{"label": "value2"})) |
| 466 |
}() |
| 467 |
|
| 468 |
// list by empty MatchValues. |
| 469 |
resp, indexUsed, err := store.WaitUntilFreshAndList(ctx, 5, "prefix/", storage.ListOptions{Predicate: storage.Everything}) |
| 470 |
if err != nil { |
| 471 |
t.Fatalf("unexpected error: %v", err) |
| 472 |
} |
| 473 |
if resp.ResourceVersion != 5 { |
| 474 |
t.Errorf("unexpected resourceVersion: %v, expected: 5", resp.ResourceVersion) |
| 475 |
} |
| 476 |
if len(resp.Items) != 3 { |
| 477 |
t.Errorf("unexpected list returned: %#v", resp) |
| 478 |
} |
| 479 |
if indexUsed != "" { |
| 480 |
t.Errorf("Used index %q but expected none to be used", indexUsed) |
| 481 |
} |
| 482 |
|
| 483 |
// list by label index. |
| 484 |
resp, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", storage.ListOptions{Predicate: storage.SelectionPredicate{ |
| 485 |
Label: labels.SelectorFromSet(map[string]string{ |
| 486 |
"label": "value1", |
| 487 |
}), |
| 488 |
Field: fields.SelectorFromSet(map[string]string{ |
| 489 |
"spec.nodeName": "node2", |
| 490 |
}), |
| 491 |
IndexLabels: []string{"label"}, |
| 492 |
}}) |
| 493 |
if err != nil { |
| 494 |
t.Fatalf("unexpected error: %v", err) |
| 495 |
} |
| 496 |
if resp.ResourceVersion != 5 { |
| 497 |
t.Errorf("unexpected resourceVersion: %v, expected: 5", resp.ResourceVersion) |
| 498 |
} |
| 499 |
if len(resp.Items) != 2 { |
| 500 |
t.Errorf("unexpected list returned: %#v", resp) |
| 501 |
} |
| 502 |
if indexUsed != "l:label" { |
| 503 |
t.Errorf("Used index %q but expected %q", indexUsed, "l:label") |
| 504 |
} |
| 505 |
|
| 506 |
// list with spec.nodeName index. |
| 507 |
resp, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", storage.ListOptions{Predicate: storage.SelectionPredicate{ |
| 508 |
Label: labels.SelectorFromSet(map[string]string{ |
| 509 |
"not-exist-label": "whatever", |
| 510 |
}), |
| 511 |
Field: fields.SelectorFromSet(map[string]string{ |
| 512 |
"spec.nodeName": "node2", |
| 513 |
}), |
| 514 |
IndexFields: []string{"spec.nodeName"}, |
| 515 |
}}) |
| 516 |
if err != nil { |
| 517 |
t.Fatalf("unexpected error: %v", err) |
| 518 |
} |
| 519 |
if resp.ResourceVersion != 5 { |
| 520 |
t.Errorf("unexpected resourceVersion: %v, expected: 5", resp.ResourceVersion) |
| 521 |
} |
| 522 |
if len(resp.Items) != 1 { |
| 523 |
t.Errorf("unexpected list returned: %#v", resp) |
| 524 |
} |
| 525 |
if indexUsed != "f:spec.nodeName" { |
| 526 |
t.Errorf("Used index %q but expected %q", indexUsed, "f:spec.nodeName") |
| 527 |
} |
| 528 |
|
| 529 |
// list with index not exists. |
| 530 |
resp, indexUsed, err = store.WaitUntilFreshAndList(ctx, 5, "prefix/", storage.ListOptions{Predicate: storage.SelectionPredicate{ |
| 531 |
Label: labels.SelectorFromSet(map[string]string{ |
| 532 |
"not-exist-label": "whatever", |
| 533 |
}), |
| 534 |
Field: fields.Everything(), |
| 535 |
IndexLabels: []string{"label"}, |
| 536 |
}}) |
| 537 |
if err != nil { |
| 538 |
t.Fatalf("unexpected error: %v", err) |
| 539 |
} |
| 540 |
if resp.ResourceVersion != 5 { |
| 541 |
t.Errorf("unexpected resourceVersion: %v, expected: 5", resp.ResourceVersion) |
| 542 |
} |
| 543 |
if len(resp.Items) != 3 { |
| 544 |
t.Errorf("unexpected list returned: %#v", resp) |
| 545 |
} |
| 546 |
if indexUsed != "" { |
| 547 |
t.Errorf("Used index %q but expected none to be used", indexUsed) |
| 548 |
} |
| 549 |
} |
| 550 |
|
| 551 |
func TestWaitUntilFreshAndListFromCache(t *testing.T) { |
| 552 |
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, true) |
| 553 |
forceRequestWatchProgressSupport(t) |
| 554 |
ctx := context.Background() |
| 555 |
store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{}) |
| 556 |
defer store.Stop() |
| 557 |
// In background, update the store. |
| 558 |
go func() { |
| 559 |
store.Add(makeTestPod("pod1", 2)) |
| 560 |
store.bookmarkRevision <- 3 |
| 561 |
}() |
| 562 |
|
| 563 |
// list from future revision. Requires watch cache to request bookmark to get it. |
| 564 |
resp, indexUsed, err := store.WaitUntilFreshAndList(ctx, 3, "prefix/", storage.ListOptions{Predicate: storage.Everything}) |
| 565 |
if err != nil { |
| 566 |
t.Fatalf("unexpected error: %v", err) |
| 567 |
} |
| 568 |
if resp.ResourceVersion != 3 { |
| 569 |
t.Errorf("unexpected resourceVersion: %v, expected: 6", resp.ResourceVersion) |
| 570 |
} |
| 571 |
if len(resp.Items) != 1 { |
| 572 |
t.Errorf("unexpected list returned: %#v", resp) |
| 573 |
} |
| 574 |
if indexUsed != "" { |
| 575 |
t.Errorf("Used index %q but expected none to be used", indexUsed) |
| 576 |
} |
| 577 |
} |
| 578 |
|
| 579 |
func TestWaitUntilFreshAndGet(t *testing.T) { |
| 580 |
ctx := context.Background() |
| 581 |
store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{}) |
| 582 |
defer store.Stop() |
| 583 |
|
| 584 |
// In background, update the store. |
| 585 |
go func() { |
| 586 |
store.Add(makeTestPod("foo", 2)) |
| 587 |
store.Add(makeTestPod("bar", 5)) |
| 588 |
}() |
| 589 |
|
| 590 |
obj, exists, resourceVersion, err := store.WaitUntilFreshAndGet(ctx, 5, "prefix/ns/bar") |
| 591 |
if err != nil { |
| 592 |
t.Fatalf("unexpected error: %v", err) |
| 593 |
} |
| 594 |
if resourceVersion != 5 { |
| 595 |
t.Errorf("unexpected resourceVersion: %v, expected: 5", resourceVersion) |
| 596 |
} |
| 597 |
if !exists { |
| 598 |
t.Fatalf("no results returned: %#v", obj) |
| 599 |
} |
| 600 |
expected := makeTestStoreElement(makeTestPod("bar", 5)) |
| 601 |
if !apiequality.Semantic.DeepEqual(expected, obj) { |
| 602 |
t.Errorf("expected %v, got %v", expected, obj) |
| 603 |
} |
| 604 |
} |
| 605 |
|
| 606 |
func TestWaitUntilFreshAndListTimeout(t *testing.T) { |
| 607 |
tcs := []struct { |
| 608 |
name string |
| 609 |
ConsistentListFromCache bool |
| 610 |
}{ |
| 611 |
{ |
| 612 |
name: "FromStorage", |
| 613 |
ConsistentListFromCache: false, |
| 614 |
}, |
| 615 |
{ |
| 616 |
name: "FromCache", |
| 617 |
ConsistentListFromCache: true, |
| 618 |
}, |
| 619 |
} |
| 620 |
for _, tc := range tcs { |
| 621 |
t.Run(tc.name, func(t *testing.T) { |
| 622 |
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, tc.ConsistentListFromCache) |
| 623 |
ctx := context.Background() |
| 624 |
store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{}) |
| 625 |
defer store.Stop() |
| 626 |
fc := store.clock.(*testingclock.FakeClock) |
| 627 |
|
| 628 |
// In background, step clock after the below call starts the timer. |
| 629 |
go func() { |
| 630 |
for !fc.HasWaiters() { |
| 631 |
time.Sleep(time.Millisecond) |
| 632 |
} |
| 633 |
store.Add(makeTestPod("foo", 2)) |
| 634 |
store.bookmarkRevision <- 3 |
| 635 |
fc.Step(blockTimeout) |
| 636 |
|
| 637 |
// Add an object to make sure the test would |
| 638 |
// eventually fail instead of just waiting |
| 639 |
// forever. |
| 640 |
time.Sleep(30 * time.Second) |
| 641 |
store.Add(makeTestPod("bar", 4)) |
| 642 |
}() |
| 643 |
|
| 644 |
_, _, err := store.WaitUntilFreshAndList(ctx, 4, "", storage.ListOptions{Predicate: storage.Everything}) |
| 645 |
if !errors.IsTimeout(err) { |
| 646 |
t.Errorf("expected timeout error but got: %v", err) |
| 647 |
} |
| 648 |
if !storage.IsTooLargeResourceVersion(err) { |
| 649 |
t.Errorf("expected 'Too large resource version' cause in error but got: %v", err) |
| 650 |
} |
| 651 |
}) |
| 652 |
} |
| 653 |
} |
| 654 |
|
| 655 |
type testLW struct { |
| 656 |
ListFunc func(options metav1.ListOptions) (runtime.Object, error) |
| 657 |
WatchFunc func(options metav1.ListOptions) (watch.Interface, error) |
| 658 |
} |
| 659 |
|
| 660 |
func (t *testLW) List(options metav1.ListOptions) (runtime.Object, error) { |
| 661 |
return t.ListFunc(options) |
| 662 |
} |
| 663 |
func (t *testLW) Watch(options metav1.ListOptions) (watch.Interface, error) { |
| 664 |
return t.WatchFunc(options) |
| 665 |
} |
| 666 |
|
| 667 |
func TestReflectorForWatchCache(t *testing.T) { |
| 668 |
ctx := context.Background() |
| 669 |
store := newTestWatchCache(5, DefaultEventFreshDuration, &cache.Indexers{}) |
| 670 |
defer store.Stop() |
| 671 |
|
| 672 |
{ |
| 673 |
resp, _, err := store.WaitUntilFreshAndList(ctx, 0, "", storage.ListOptions{Predicate: storage.Everything}) |
| 674 |
if err != nil { |
| 675 |
t.Fatalf("unexpected error: %v", err) |
| 676 |
} |
| 677 |
if resp.ResourceVersion != 0 { |
| 678 |
t.Errorf("unexpected resource version: %d", resp.ResourceVersion) |
| 679 |
} |
| 680 |
} |
| 681 |
|
| 682 |
lw := &testLW{ |
| 683 |
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) { |
| 684 |
fw := watch.NewFake() |
| 685 |
go fw.Stop() |
| 686 |
return fw, nil |
| 687 |
}, |
| 688 |
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { |
| 689 |
return &v1.PodList{ListMeta: metav1.ListMeta{ResourceVersion: "10"}}, nil |
| 690 |
}, |
| 691 |
} |
| 692 |
r := cache.NewReflector(lw, &v1.Pod{}, store, 0) |
| 693 |
r.ListAndWatch(wait.NeverStop) |
| 694 |
|
| 695 |
{ |
| 696 |
resp, _, err := store.WaitUntilFreshAndList(ctx, 10, "", storage.ListOptions{Predicate: storage.Everything}) |
| 697 |
if err != nil { |
| 698 |
t.Fatalf("unexpected error: %v", err) |
| 699 |
} |
| 700 |
if resp.ResourceVersion != 10 { |
| 701 |
t.Errorf("unexpected resource version: %d", resp.ResourceVersion) |
| 702 |
} |
| 703 |
} |
| 704 |
} |
| 705 |
|
| 706 |
func TestDynamicCache(t *testing.T) { |
| 707 |
tests := []struct { |
| 708 |
name string |
| 709 |
eventCount int |
| 710 |
cacheCapacity int |
| 711 |
startIndex int |
| 712 |
// interval is time duration between adjacent events. |
| 713 |
lowerBoundCapacity int |
| 714 |
upperBoundCapacity int |
| 715 |
interval time.Duration |
| 716 |
expectCapacity int |
| 717 |
expectStartIndex int |
| 718 |
}{ |
| 719 |
{ |
| 720 |
name: "[capacity not equals 4*n] events inside DefaultEventFreshDuration cause cache expanding", |
| 721 |
eventCount: 5, |
| 722 |
cacheCapacity: 5, |
| 723 |
lowerBoundCapacity: 5 / 2, |
| 724 |
upperBoundCapacity: 5 * 2, |
| 725 |
interval: DefaultEventFreshDuration / 6, |
| 726 |
expectCapacity: 10, |
| 727 |
expectStartIndex: 0, |
| 728 |
}, |
| 729 |
{ |
| 730 |
name: "[capacity not equals 4*n] events outside DefaultEventFreshDuration without change cache capacity", |
| 731 |
eventCount: 5, |
| 732 |
cacheCapacity: 5, |
| 733 |
lowerBoundCapacity: 5 / 2, |
| 734 |
upperBoundCapacity: 5 * 2, |
| 735 |
interval: DefaultEventFreshDuration / 4, |
| 736 |
expectCapacity: 5, |
| 737 |
expectStartIndex: 0, |
| 738 |
}, |
| 739 |
{ |
| 740 |
name: "[capacity not equals 4*n] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking", |
| 741 |
eventCount: 5, |
| 742 |
cacheCapacity: 5, |
| 743 |
lowerBoundCapacity: 5 / 2, |
| 744 |
upperBoundCapacity: 5 * 2, |
| 745 |
interval: DefaultEventFreshDuration + time.Second, |
| 746 |
expectCapacity: 2, |
| 747 |
expectStartIndex: 3, |
| 748 |
}, |
| 749 |
{ |
| 750 |
name: "[capacity not equals 4*n] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking with given lowerBoundCapacity", |
| 751 |
eventCount: 5, |
| 752 |
cacheCapacity: 5, |
| 753 |
lowerBoundCapacity: 3, |
| 754 |
upperBoundCapacity: 5 * 2, |
| 755 |
interval: DefaultEventFreshDuration + time.Second, |
| 756 |
expectCapacity: 3, |
| 757 |
expectStartIndex: 2, |
| 758 |
}, |
| 759 |
{ |
| 760 |
name: "[capacity not equals 4*n] events inside DefaultEventFreshDuration cause cache expanding with given upperBoundCapacity", |
| 761 |
eventCount: 5, |
| 762 |
cacheCapacity: 5, |
| 763 |
lowerBoundCapacity: 5 / 2, |
| 764 |
upperBoundCapacity: 8, |
| 765 |
interval: DefaultEventFreshDuration / 6, |
| 766 |
expectCapacity: 8, |
| 767 |
expectStartIndex: 0, |
| 768 |
}, |
| 769 |
{ |
| 770 |
name: "[capacity not equals 4*n] [startIndex not equal 0] events inside DefaultEventFreshDuration cause cache expanding", |
| 771 |
eventCount: 5, |
| 772 |
cacheCapacity: 5, |
| 773 |
startIndex: 3, |
| 774 |
lowerBoundCapacity: 5 / 2, |
| 775 |
upperBoundCapacity: 5 * 2, |
| 776 |
interval: DefaultEventFreshDuration / 6, |
| 777 |
expectCapacity: 10, |
| 778 |
expectStartIndex: 3, |
| 779 |
}, |
| 780 |
{ |
| 781 |
name: "[capacity not equals 4*n] [startIndex not equal 0] events outside DefaultEventFreshDuration without change cache capacity", |
| 782 |
eventCount: 5, |
| 783 |
cacheCapacity: 5, |
| 784 |
startIndex: 3, |
| 785 |
lowerBoundCapacity: 5 / 2, |
| 786 |
upperBoundCapacity: 5 * 2, |
| 787 |
interval: DefaultEventFreshDuration / 4, |
| 788 |
expectCapacity: 5, |
| 789 |
expectStartIndex: 3, |
| 790 |
}, |
| 791 |
{ |
| 792 |
name: "[capacity not equals 4*n] [startIndex not equal 0] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking", |
| 793 |
eventCount: 5, |
| 794 |
cacheCapacity: 5, |
| 795 |
startIndex: 3, |
| 796 |
lowerBoundCapacity: 5 / 2, |
| 797 |
upperBoundCapacity: 5 * 2, |
| 798 |
interval: DefaultEventFreshDuration + time.Second, |
| 799 |
expectCapacity: 2, |
| 800 |
expectStartIndex: 6, |
| 801 |
}, |
| 802 |
{ |
| 803 |
name: "[capacity not equals 4*n] [startIndex not equal 0] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking with given lowerBoundCapacity", |
| 804 |
eventCount: 5, |
| 805 |
cacheCapacity: 5, |
| 806 |
startIndex: 3, |
| 807 |
lowerBoundCapacity: 3, |
| 808 |
upperBoundCapacity: 5 * 2, |
| 809 |
interval: DefaultEventFreshDuration + time.Second, |
| 810 |
expectCapacity: 3, |
| 811 |
expectStartIndex: 5, |
| 812 |
}, |
| 813 |
{ |
| 814 |
name: "[capacity not equals 4*n] [startIndex not equal 0] events inside DefaultEventFreshDuration cause cache expanding with given upperBoundCapacity", |
| 815 |
eventCount: 5, |
| 816 |
cacheCapacity: 5, |
| 817 |
startIndex: 3, |
| 818 |
lowerBoundCapacity: 5 / 2, |
| 819 |
upperBoundCapacity: 8, |
| 820 |
interval: DefaultEventFreshDuration / 6, |
| 821 |
expectCapacity: 8, |
| 822 |
expectStartIndex: 3, |
| 823 |
}, |
| 824 |
{ |
| 825 |
name: "[capacity equals 4*n] events inside DefaultEventFreshDuration cause cache expanding", |
| 826 |
eventCount: 8, |
| 827 |
cacheCapacity: 8, |
| 828 |
lowerBoundCapacity: 8 / 2, |
| 829 |
upperBoundCapacity: 8 * 2, |
| 830 |
interval: DefaultEventFreshDuration / 9, |
| 831 |
expectCapacity: 16, |
| 832 |
expectStartIndex: 0, |
| 833 |
}, |
| 834 |
{ |
| 835 |
name: "[capacity equals 4*n] events outside DefaultEventFreshDuration without change cache capacity", |
| 836 |
eventCount: 8, |
| 837 |
cacheCapacity: 8, |
| 838 |
lowerBoundCapacity: 8 / 2, |
| 839 |
upperBoundCapacity: 8 * 2, |
| 840 |
interval: DefaultEventFreshDuration / 8, |
| 841 |
expectCapacity: 8, |
| 842 |
expectStartIndex: 0, |
| 843 |
}, |
| 844 |
{ |
| 845 |
name: "[capacity equals 4*n] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking", |
| 846 |
eventCount: 8, |
| 847 |
cacheCapacity: 8, |
| 848 |
lowerBoundCapacity: 8 / 2, |
| 849 |
upperBoundCapacity: 8 * 2, |
| 850 |
interval: DefaultEventFreshDuration/2 + time.Second, |
| 851 |
expectCapacity: 4, |
| 852 |
expectStartIndex: 4, |
| 853 |
}, |
| 854 |
{ |
| 855 |
name: "[capacity equals 4*n] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking with given lowerBoundCapacity", |
| 856 |
eventCount: 8, |
| 857 |
cacheCapacity: 8, |
| 858 |
lowerBoundCapacity: 7, |
| 859 |
upperBoundCapacity: 8 * 2, |
| 860 |
interval: DefaultEventFreshDuration/2 + time.Second, |
| 861 |
expectCapacity: 7, |
| 862 |
expectStartIndex: 1, |
| 863 |
}, |
| 864 |
{ |
| 865 |
name: "[capacity equals 4*n] events inside DefaultEventFreshDuration cause cache expanding with given upperBoundCapacity", |
| 866 |
eventCount: 8, |
| 867 |
cacheCapacity: 8, |
| 868 |
lowerBoundCapacity: 8 / 2, |
| 869 |
upperBoundCapacity: 10, |
| 870 |
interval: DefaultEventFreshDuration / 9, |
| 871 |
expectCapacity: 10, |
| 872 |
expectStartIndex: 0, |
| 873 |
}, |
| 874 |
{ |
| 875 |
name: "[capacity equals 4*n] [startIndex not equal 0] events inside DefaultEventFreshDuration cause cache expanding", |
| 876 |
eventCount: 8, |
| 877 |
cacheCapacity: 8, |
| 878 |
startIndex: 3, |
| 879 |
lowerBoundCapacity: 8 / 2, |
| 880 |
upperBoundCapacity: 8 * 2, |
| 881 |
interval: DefaultEventFreshDuration / 9, |
| 882 |
expectCapacity: 16, |
| 883 |
expectStartIndex: 3, |
| 884 |
}, |
| 885 |
{ |
| 886 |
name: "[capacity equals 4*n] [startIndex not equal 0] events outside DefaultEventFreshDuration without change cache capacity", |
| 887 |
eventCount: 8, |
| 888 |
cacheCapacity: 8, |
| 889 |
startIndex: 3, |
| 890 |
lowerBoundCapacity: 8 / 2, |
| 891 |
upperBoundCapacity: 8 * 2, |
| 892 |
interval: DefaultEventFreshDuration / 8, |
| 893 |
expectCapacity: 8, |
| 894 |
expectStartIndex: 3, |
| 895 |
}, |
| 896 |
{ |
| 897 |
name: "[capacity equals 4*n] [startIndex not equal 0] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking", |
| 898 |
eventCount: 8, |
| 899 |
cacheCapacity: 8, |
| 900 |
startIndex: 3, |
| 901 |
lowerBoundCapacity: 8 / 2, |
| 902 |
upperBoundCapacity: 8 * 2, |
| 903 |
interval: DefaultEventFreshDuration/2 + time.Second, |
| 904 |
expectCapacity: 4, |
| 905 |
expectStartIndex: 7, |
| 906 |
}, |
| 907 |
{ |
| 908 |
name: "[capacity equals 4*n] [startIndex not equal 0] quarter of recent events outside DefaultEventFreshDuration cause cache shrinking with given lowerBoundCapacity", |
| 909 |
eventCount: 8, |
| 910 |
cacheCapacity: 8, |
| 911 |
startIndex: 3, |
| 912 |
lowerBoundCapacity: 7, |
| 913 |
upperBoundCapacity: 8 * 2, |
| 914 |
interval: DefaultEventFreshDuration/2 + time.Second, |
| 915 |
expectCapacity: 7, |
| 916 |
expectStartIndex: 4, |
| 917 |
}, |
| 918 |
{ |
| 919 |
name: "[capacity equals 4*n] [startIndex not equal 0] events inside DefaultEventFreshDuration cause cache expanding with given upperBoundCapacity", |
| 920 |
eventCount: 8, |
| 921 |
cacheCapacity: 8, |
| 922 |
startIndex: 3, |
| 923 |
lowerBoundCapacity: 8 / 2, |
| 924 |
upperBoundCapacity: 10, |
| 925 |
interval: DefaultEventFreshDuration / 9, |
| 926 |
expectCapacity: 10, |
| 927 |
expectStartIndex: 3, |
| 928 |
}, |
| 929 |
} |
| 930 |
|
| 931 |
for _, test := range tests { |
| 932 |
t.Run(test.name, func(t *testing.T) { |
| 933 |
store := newTestWatchCache(test.cacheCapacity, DefaultEventFreshDuration, &cache.Indexers{}) |
| 934 |
defer store.Stop() |
| 935 |
store.cache = make([]*watchCacheEvent, test.cacheCapacity) |
| 936 |
store.startIndex = test.startIndex |
| 937 |
store.lowerBoundCapacity = test.lowerBoundCapacity |
| 938 |
store.upperBoundCapacity = test.upperBoundCapacity |
| 939 |
loadEventWithDuration(store, test.eventCount, test.interval) |
| 940 |
nextInterval := store.clock.Now().Add(time.Duration(test.interval.Nanoseconds() * int64(test.eventCount))) |
| 941 |
store.resizeCacheLocked(nextInterval) |
| 942 |
if store.capacity != test.expectCapacity { |
| 943 |
t.Errorf("expect capacity %d, but get %d", test.expectCapacity, store.capacity) |
| 944 |
} |
| 945 |
|
| 946 |
// check cache's startIndex, endIndex and all elements. |
| 947 |
if store.startIndex != test.expectStartIndex { |
| 948 |
t.Errorf("expect startIndex %d, but get %d", test.expectStartIndex, store.startIndex) |
| 949 |
} |
| 950 |
if store.endIndex != test.startIndex+test.eventCount { |
| 951 |
t.Errorf("expect endIndex %d get %d", test.startIndex+test.eventCount, store.endIndex) |
| 952 |
} |
| 953 |
if !checkCacheElements(store) { |
| 954 |
t.Errorf("some elements locations in cache is wrong") |
| 955 |
} |
| 956 |
}) |
| 957 |
} |
| 958 |
} |
| 959 |
|
| 960 |
func loadEventWithDuration(cache *testWatchCache, count int, interval time.Duration) { |
| 961 |
for i := 0; i < count; i++ { |
| 962 |
event := &watchCacheEvent{ |
| 963 |
Key: fmt.Sprintf("event-%d", i+cache.startIndex), |
| 964 |
RecordTime: cache.clock.Now().Add(time.Duration(interval.Nanoseconds() * int64(i))), |
| 965 |
} |
| 966 |
cache.cache[(i+cache.startIndex)%cache.capacity] = event |
| 967 |
} |
| 968 |
cache.endIndex = cache.startIndex + count |
| 969 |
} |
| 970 |
|
| 971 |
func checkCacheElements(cache *testWatchCache) bool { |
| 972 |
for i := cache.startIndex; i < cache.endIndex; i++ { |
| 973 |
location := i % cache.capacity |
| 974 |
if cache.cache[location].Key != fmt.Sprintf("event-%d", i) { |
| 975 |
return false |
| 976 |
} |
| 977 |
} |
| 978 |
return true |
| 979 |
} |
| 980 |
|
| 981 |
func TestCacheIncreaseDoesNotBreakWatch(t *testing.T) { |
| 982 |
store := newTestWatchCache(2, DefaultEventFreshDuration, &cache.Indexers{}) |
| 983 |
defer store.Stop() |
| 984 |
|
| 985 |
now := store.clock.Now() |
| 986 |
addEvent := func(key string, rv uint64, t time.Time) { |
| 987 |
event := &watchCacheEvent{ |
| 988 |
Key: key, |
| 989 |
ResourceVersion: rv, |
| 990 |
RecordTime: t, |
| 991 |
} |
| 992 |
store.updateCache(event) |
| 993 |
} |
| 994 |
|
| 995 |
// Initial LIST comes from the moment of RV=10. |
| 996 |
store.Replace(nil, "10") |
| 997 |
|
| 998 |
addEvent("key1", 20, now) |
| 999 |
|
| 1000 |
// Force "key1" to rotate our of cache. |
| 1001 |
later := now.Add(2 * DefaultEventFreshDuration) |
| 1002 |
addEvent("key2", 30, later) |
| 1003 |
addEvent("key3", 40, later) |
| 1004 |
|
| 1005 |
// Force cache resize. |
| 1006 |
addEvent("key4", 50, later.Add(time.Second)) |
| 1007 |
|
| 1008 |
_, err := store.getAllEventsSince(15, storage.ListOptions{Predicate: storage.Everything}) |
| 1009 |
if err == nil || !strings.Contains(err.Error(), "too old resource version") { |
| 1010 |
t.Errorf("unexpected error: %v", err) |
| 1011 |
} |
| 1012 |
} |
| 1013 |
|
| 1014 |
func TestSuggestedWatchChannelSize(t *testing.T) { |
| 1015 |
testCases := []struct { |
| 1016 |
name string |
| 1017 |
capacity int |
| 1018 |
indexExists bool |
| 1019 |
triggerUsed bool |
| 1020 |
eventsFreshDuration time.Duration |
| 1021 |
expected int |
| 1022 |
}{ |
| 1023 |
{ |
| 1024 |
name: "capacity=100, indexExists, triggerUsed", |
| 1025 |
capacity: 100, |
| 1026 |
indexExists: true, |
| 1027 |
triggerUsed: true, |
| 1028 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1029 |
expected: 10, |
| 1030 |
}, |
| 1031 |
{ |
| 1032 |
name: "capacity=100, indexExists, !triggerUsed", |
| 1033 |
capacity: 100, |
| 1034 |
indexExists: true, |
| 1035 |
triggerUsed: false, |
| 1036 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1037 |
expected: 10, |
| 1038 |
}, |
| 1039 |
{ |
| 1040 |
name: "capacity=100, !indexExists", |
| 1041 |
capacity: 100, |
| 1042 |
indexExists: false, |
| 1043 |
triggerUsed: false, |
| 1044 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1045 |
expected: 10, |
| 1046 |
}, |
| 1047 |
{ |
| 1048 |
name: "capacity=750, indexExists, triggerUsed", |
| 1049 |
capacity: 750, |
| 1050 |
indexExists: true, |
| 1051 |
triggerUsed: true, |
| 1052 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1053 |
expected: 10, |
| 1054 |
}, |
| 1055 |
{ |
| 1056 |
name: "capacity=750, indexExists, !triggerUsed", |
| 1057 |
capacity: 750, |
| 1058 |
indexExists: true, |
| 1059 |
triggerUsed: false, |
| 1060 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1061 |
expected: 10, |
| 1062 |
}, |
| 1063 |
{ |
| 1064 |
name: "capacity=750, !indexExists", |
| 1065 |
capacity: 750, |
| 1066 |
indexExists: false, |
| 1067 |
triggerUsed: false, |
| 1068 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1069 |
expected: 10, |
| 1070 |
}, |
| 1071 |
{ |
| 1072 |
name: "capacity=7500, indexExists, triggerUsed", |
| 1073 |
capacity: 7500, |
| 1074 |
indexExists: true, |
| 1075 |
triggerUsed: true, |
| 1076 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1077 |
expected: 10, |
| 1078 |
}, |
| 1079 |
{ |
| 1080 |
name: "capacity=7500, indexExists, !triggerUsed", |
| 1081 |
capacity: 7500, |
| 1082 |
indexExists: true, |
| 1083 |
triggerUsed: false, |
| 1084 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1085 |
expected: 100, |
| 1086 |
}, |
| 1087 |
{ |
| 1088 |
name: "capacity=7500, indexExists, !triggerUsed, eventsFreshDuration=2m30s", |
| 1089 |
capacity: 7500, |
| 1090 |
indexExists: true, |
| 1091 |
triggerUsed: false, |
| 1092 |
eventsFreshDuration: 2 * DefaultEventFreshDuration, |
| 1093 |
expected: 50, |
| 1094 |
}, |
| 1095 |
{ |
| 1096 |
name: "capacity=7500, !indexExists", |
| 1097 |
capacity: 7500, |
| 1098 |
indexExists: false, |
| 1099 |
triggerUsed: false, |
| 1100 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1101 |
expected: 100, |
| 1102 |
}, |
| 1103 |
{ |
| 1104 |
name: "capacity=7500, !indexExists, eventsFreshDuration=2m30s", |
| 1105 |
capacity: 7500, |
| 1106 |
indexExists: false, |
| 1107 |
triggerUsed: false, |
| 1108 |
eventsFreshDuration: 2 * DefaultEventFreshDuration, |
| 1109 |
expected: 50, |
| 1110 |
}, |
| 1111 |
{ |
| 1112 |
name: "capacity=75000, indexExists, triggerUsed", |
| 1113 |
capacity: 75000, |
| 1114 |
indexExists: true, |
| 1115 |
triggerUsed: true, |
| 1116 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1117 |
expected: 10, |
| 1118 |
}, |
| 1119 |
{ |
| 1120 |
name: "capacity=75000, indexExists, !triggerUsed", |
| 1121 |
capacity: 75000, |
| 1122 |
indexExists: true, |
| 1123 |
triggerUsed: false, |
| 1124 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1125 |
expected: 1000, |
| 1126 |
}, |
| 1127 |
{ |
| 1128 |
name: "capacity=75000, indexExists, !triggerUsed, eventsFreshDuration=2m30s", |
| 1129 |
capacity: 75000, |
| 1130 |
indexExists: true, |
| 1131 |
triggerUsed: false, |
| 1132 |
eventsFreshDuration: 2 * DefaultEventFreshDuration, |
| 1133 |
expected: 500, |
| 1134 |
}, |
| 1135 |
{ |
| 1136 |
name: "capacity=75000, !indexExists", |
| 1137 |
capacity: 75000, |
| 1138 |
indexExists: false, |
| 1139 |
triggerUsed: false, |
| 1140 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1141 |
expected: 100, |
| 1142 |
}, |
| 1143 |
{ |
| 1144 |
name: "capacity=750000, indexExists, triggerUsed", |
| 1145 |
capacity: 750000, |
| 1146 |
indexExists: true, |
| 1147 |
triggerUsed: true, |
| 1148 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1149 |
expected: 10, |
| 1150 |
}, |
| 1151 |
{ |
| 1152 |
name: "capacity=750000, indexExists, !triggerUsed", |
| 1153 |
capacity: 750000, |
| 1154 |
indexExists: true, |
| 1155 |
triggerUsed: false, |
| 1156 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1157 |
expected: 1000, |
| 1158 |
}, |
| 1159 |
{ |
| 1160 |
name: "capacity=750000, !indexExists", |
| 1161 |
capacity: 750000, |
| 1162 |
indexExists: false, |
| 1163 |
triggerUsed: false, |
| 1164 |
eventsFreshDuration: DefaultEventFreshDuration, |
| 1165 |
expected: 100, |
| 1166 |
}, |
| 1167 |
} |
| 1168 |
|
| 1169 |
for _, test := range testCases { |
| 1170 |
t.Run(test.name, func(t *testing.T) { |
| 1171 |
store := newTestWatchCache(test.capacity, test.eventsFreshDuration, &cache.Indexers{}) |
| 1172 |
defer store.Stop() |
| 1173 |
got := store.suggestedWatchChannelSize(test.indexExists, test.triggerUsed) |
| 1174 |
if got != test.expected { |
| 1175 |
t.Errorf("unexpected channel size got: %v, expected: %v", got, test.expected) |
| 1176 |
} |
| 1177 |
}) |
| 1178 |
} |
| 1179 |
} |
| 1180 |
|
| 1181 |
func TestCapacityUpperBound(t *testing.T) { |
| 1182 |
testCases := []struct { |
| 1183 |
name string |
| 1184 |
eventFreshDuration time.Duration |
| 1185 |
expected int |
| 1186 |
}{ |
| 1187 |
{ |
| 1188 |
name: "default eventFreshDuration", |
| 1189 |
eventFreshDuration: DefaultEventFreshDuration, // 75s |
| 1190 |
expected: defaultUpperBoundCapacity, // 100 * 1024 |
| 1191 |
}, |
| 1192 |
{ |
| 1193 |
name: "lower eventFreshDuration, capacity limit unchanged", |
| 1194 |
eventFreshDuration: 45 * time.Second, // 45s |
| 1195 |
expected: defaultUpperBoundCapacity, // 100 * 1024 |
| 1196 |
}, |
| 1197 |
{ |
| 1198 |
name: "higher eventFreshDuration, capacity limit scaled up", |
| 1199 |
eventFreshDuration: 4 * DefaultEventFreshDuration, // 4 * 75s |
| 1200 |
expected: 4 * defaultUpperBoundCapacity, // 4 * 100 * 1024 |
| 1201 |
}, |
| 1202 |
{ |
| 1203 |
name: "higher eventFreshDuration, capacity limit scaled and rounded up", |
| 1204 |
eventFreshDuration: 3 * DefaultEventFreshDuration, // 3 * 75s |
| 1205 |
expected: 4 * defaultUpperBoundCapacity, // 4 * 100 * 1024 |
| 1206 |
}, |
| 1207 |
{ |
| 1208 |
name: "higher eventFreshDuration, capacity limit scaled up and capped", |
| 1209 |
eventFreshDuration: DefaultEventFreshDuration << 20, // 2^20 * 75s |
| 1210 |
expected: defaultUpperBoundCapacity << 14, // 2^14 * 100 * 1024 |
| 1211 |
}, |
| 1212 |
} |
| 1213 |
|
| 1214 |
for _, test := range testCases { |
| 1215 |
t.Run(test.name, func(t *testing.T) { |
| 1216 |
capacity := capacityUpperBound(test.eventFreshDuration) |
| 1217 |
if test.expected != capacity { |
| 1218 |
t.Errorf("expected %v, got %v", test.expected, capacity) |
| 1219 |
} |
| 1220 |
}) |
| 1221 |
} |
| 1222 |
} |
| 1223 |
|
| 1224 |
func BenchmarkWatchCache_updateCache(b *testing.B) { |
| 1225 |
store := newTestWatchCache(defaultUpperBoundCapacity, DefaultEventFreshDuration, &cache.Indexers{}) |
| 1226 |
defer store.Stop() |
| 1227 |
store.cache = store.cache[:0] |
| 1228 |
store.upperBoundCapacity = defaultUpperBoundCapacity |
| 1229 |
loadEventWithDuration(store, defaultUpperBoundCapacity, 0) |
| 1230 |
add := &watchCacheEvent{ |
| 1231 |
Key: fmt.Sprintf("event-%d", defaultUpperBoundCapacity), |
| 1232 |
RecordTime: store.clock.Now(), |
| 1233 |
} |
| 1234 |
b.ResetTimer() |
| 1235 |
for i := 0; i < b.N; i++ { |
| 1236 |
store.updateCache(add) |
| 1237 |
} |
| 1238 |
} |
| 1239 |
|
| 1240 |
func TestHistogramCacheReadWait(t *testing.T) { |
| 1241 |
registry := k8smetrics.NewKubeRegistry() |
| 1242 |
if err := registry.Register(metrics.WatchCacheReadWait); err != nil { |
| 1243 |
t.Errorf("unexpected error: %v", err) |
| 1244 |
} |
| 1245 |
ctx := context.Background() |
| 1246 |
testedMetrics := "apiserver_watch_cache_read_wait_seconds" |
| 1247 |
store := newTestWatchCache(2, DefaultEventFreshDuration, &cache.Indexers{}) |
| 1248 |
defer store.Stop() |
| 1249 |
|
| 1250 |
// In background, update the store. |
| 1251 |
go func() { |
| 1252 |
if err := store.Add(makeTestPod("foo", 2)); err != nil { |
| 1253 |
t.Errorf("unexpected error: %v", err) |
| 1254 |
} |
| 1255 |
if err := store.Add(makeTestPod("bar", 5)); err != nil { |
| 1256 |
t.Errorf("unexpected error: %v", err) |
| 1257 |
} |
| 1258 |
}() |
| 1259 |
|
| 1260 |
testCases := []struct { |
| 1261 |
desc string |
| 1262 |
resourceVersion uint64 |
| 1263 |
want string |
| 1264 |
}{ |
| 1265 |
{ |
| 1266 |
desc: "resourceVersion is non-zero", |
| 1267 |
resourceVersion: 5, |
| 1268 |
want: ` |
| 1269 |
# HELP apiserver_watch_cache_read_wait_seconds [ALPHA] Histogram of time spent waiting for a watch cache to become fresh. |
| 1270 |
# TYPE apiserver_watch_cache_read_wait_seconds histogram |
| 1271 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.005"} 1 |
| 1272 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.025"} 1 |
| 1273 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.05"} 1 |
| 1274 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.1"} 1 |
| 1275 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.2"} 1 |
| 1276 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.4"} 1 |
| 1277 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.6"} 1 |
| 1278 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.8"} 1 |
| 1279 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1"} 1 |
| 1280 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.25"} 1 |
| 1281 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.5"} 1 |
| 1282 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="2"} 1 |
| 1283 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="3"} 1 |
| 1284 |
apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="+Inf"} 1 |
| 1285 |
apiserver_watch_cache_read_wait_seconds_sum{resource="pods"} 0 |
| 1286 |
apiserver_watch_cache_read_wait_seconds_count{resource="pods"} 1 |
| 1287 |
`, |
| 1288 |
}, |
| 1289 |
{ |
| 1290 |
desc: "resourceVersion is 0", |
| 1291 |
resourceVersion: 0, |
| 1292 |
want: ``, |
| 1293 |
}, |
| 1294 |
} |
| 1295 |
|
| 1296 |
for _, test := range testCases { |
| 1297 |
t.Run(test.desc, func(t *testing.T) { |
| 1298 |
defer registry.Reset() |
| 1299 |
if _, _, _, err := store.WaitUntilFreshAndGet(ctx, test.resourceVersion, "prefix/ns/bar"); err != nil { |
| 1300 |
t.Errorf("unexpected error: %v", err) |
| 1301 |
} |
| 1302 |
if err := testutil.GatherAndCompare(registry, strings.NewReader(test.want), testedMetrics); err != nil { |
| 1303 |
t.Errorf("unexpected error: %v", err) |
| 1304 |
} |
| 1305 |
}) |
| 1306 |
} |
| 1307 |
} |
| 1308 |
|
| 1309 |
func TestCacheSnapshots(t *testing.T) { |
| 1310 |
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ListFromCacheSnapshot, true) |
| 1311 |
|
| 1312 |
store := newTestWatchCache(3, DefaultEventFreshDuration, &cache.Indexers{}) |
| 1313 |
defer store.Stop() |
| 1314 |
store.upperBoundCapacity = 3 |
| 1315 |
store.lowerBoundCapacity = 1 |
| 1316 |
clock := store.clock.(*testingclock.FakeClock) |
| 1317 |
|
| 1318 |
_, found := store.snapshots.GetLessOrEqual(100) |
| 1319 |
assert.False(t, found, "Expected empty cache to not include any snapshots") |
| 1320 |
|
| 1321 |
t.Log("Test cache on rev 100") |
| 1322 |
require.NoError(t, store.Add(makeTestPod("foo", 100))) |
| 1323 |
require.NoError(t, store.Update(makeTestPod("foo", 200))) |
| 1324 |
clock.Step(time.Second) |
| 1325 |
require.NoError(t, store.Delete(makeTestPod("foo", 300))) |
| 1326 |
|
| 1327 |
t.Log("Test cache on rev 100") |
| 1328 |
_, found = store.snapshots.GetLessOrEqual(99) |
| 1329 |
assert.False(t, found, "Expected store to not include rev 99") |
| 1330 |
lister, found := store.snapshots.GetLessOrEqual(100) |
| 1331 |
assert.True(t, found, "Expected store to not include rev 100") |
| 1332 |
elements := lister.ListPrefix("", "") |
| 1333 |
assert.Len(t, elements, 1) |
| 1334 |
assert.Equal(t, makeTestPod("foo", 100), elements[0].(*storeElement).Object) |
| 1335 |
|
| 1336 |
t.Log("Overflow cache to remove rev 100") |
| 1337 |
require.NoError(t, store.Add(makeTestPod("foo", 400))) |
| 1338 |
_, found = store.snapshots.GetLessOrEqual(100) |
| 1339 |
assert.False(t, found, "Expected overfilled cache to delete oldest rev 100") |
| 1340 |
|
| 1341 |
t.Log("Test cache on rev 200") |
| 1342 |
lister, found = store.snapshots.GetLessOrEqual(200) |
| 1343 |
assert.True(t, found, "Expected store to still keep rev 200") |
| 1344 |
elements = lister.ListPrefix("", "") |
| 1345 |
assert.Len(t, elements, 1) |
| 1346 |
assert.Equal(t, makeTestPod("foo", 200), elements[0].(*storeElement).Object) |
| 1347 |
|
| 1348 |
t.Log("Test cache on rev 300") |
| 1349 |
lister, found = store.snapshots.GetLessOrEqual(300) |
| 1350 |
assert.True(t, found, "Expected store to still keep rev 300") |
| 1351 |
elements = lister.ListPrefix("", "") |
| 1352 |
assert.Empty(t, elements) |
| 1353 |
|
| 1354 |
t.Log("Test cache on rev 400") |
| 1355 |
lister, found = store.snapshots.GetLessOrEqual(400) |
| 1356 |
assert.True(t, found, "Expected store to still keep rev 400") |
| 1357 |
elements = lister.ListPrefix("", "") |
| 1358 |
assert.Len(t, elements, 1) |
| 1359 |
assert.Equal(t, makeTestPod("foo", 400), elements[0].(*storeElement).Object) |
| 1360 |
|
| 1361 |
t.Log("Add event outside the event fresh window to force cache capacity downsize") |
| 1362 |
assert.Equal(t, 3, store.capacity) |
| 1363 |
clock.Step(DefaultEventFreshDuration + 1) |
| 1364 |
require.NoError(t, store.Update(makeTestPod("foo", 500))) |
| 1365 |
assert.Equal(t, 1, store.capacity) |
| 1366 |
assert.Equal(t, 1, store.snapshots.Len()) |
| 1367 |
_, found = store.snapshots.GetLessOrEqual(499) |
| 1368 |
assert.False(t, found, "Expected overfilled cache to delete events below 500") |
| 1369 |
|
| 1370 |
t.Log("Test cache on rev 500") |
| 1371 |
lister, found = store.snapshots.GetLessOrEqual(500) |
| 1372 |
assert.True(t, found, "Expected store to still keep rev 500") |
| 1373 |
elements = lister.ListPrefix("", "") |
| 1374 |
assert.Len(t, elements, 1) |
| 1375 |
assert.Equal(t, makeTestPod("foo", 500), elements[0].(*storeElement).Object) |
| 1376 |
|
| 1377 |
t.Log("Add event to force capacity upsize") |
| 1378 |
require.NoError(t, store.Update(makeTestPod("foo", 600))) |
| 1379 |
assert.Equal(t, 2, store.capacity) |
| 1380 |
assert.Equal(t, 2, store.snapshots.Len()) |
| 1381 |
|
| 1382 |
t.Log("Test cache on rev 600") |
| 1383 |
lister, found = store.snapshots.GetLessOrEqual(600) |
| 1384 |
assert.True(t, found, "Expected replace to be snapshotted") |
| 1385 |
elements = lister.ListPrefix("", "") |
| 1386 |
assert.Len(t, elements, 1) |
| 1387 |
assert.Equal(t, makeTestPod("foo", 600), elements[0].(*storeElement).Object) |
| 1388 |
|
| 1389 |
t.Log("Replace cache to remove history") |
| 1390 |
_, found = store.snapshots.GetLessOrEqual(500) |
| 1391 |
assert.True(t, found, "Confirm that cache stores history before replace") |
| 1392 |
err := store.Replace([]interface{}{makeTestPod("foo", 600)}, "700") |
| 1393 |
require.NoError(t, err) |
| 1394 |
_, found = store.snapshots.GetLessOrEqual(500) |
| 1395 |
assert.False(t, found, "Expected replace to remove history") |
| 1396 |
_, found = store.snapshots.GetLessOrEqual(600) |
| 1397 |
assert.False(t, found, "Expected replace to remove history") |
| 1398 |
|
| 1399 |
t.Log("Test cache on rev 700") |
| 1400 |
lister, found = store.snapshots.GetLessOrEqual(700) |
| 1401 |
assert.True(t, found, "Expected replace to be snapshotted") |
| 1402 |
elements = lister.ListPrefix("", "") |
| 1403 |
assert.Len(t, elements, 1) |
| 1404 |
assert.Equal(t, makeTestPod("foo", 600), elements[0].(*storeElement).Object) |
| 1405 |
} |