Debian Codegraph @

pkg/storage/etcd3/store_test.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package etcd3
18
19 import (
20 "context"
21 "crypto/rand"
22 "fmt"
23 "io/ioutil"
24 "os"
25 "reflect"
26 "strings"
27 "sync/atomic"
28 "testing"
29
30 "github.com/go-logr/logr"
31 "github.com/stretchr/testify/require"
32 clientv3 "go.etcd.io/etcd/client/v3"
33 "go.etcd.io/etcd/client/v3/kubernetes"
34 "go.etcd.io/etcd/server/v3/embed"
35 "google.golang.org/grpc/grpclog"
36
37 "k8s.io/apimachinery/pkg/api/apitesting"
38 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
39 "k8s.io/apimachinery/pkg/fields"
40 "k8s.io/apimachinery/pkg/labels"
41 "k8s.io/apimachinery/pkg/runtime"
42 "k8s.io/apimachinery/pkg/runtime/schema"
43 "k8s.io/apimachinery/pkg/runtime/serializer"
44 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
45 "k8s.io/apiserver/pkg/apis/example"
46 examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
47 "k8s.io/apiserver/pkg/storage"
48 "k8s.io/apiserver/pkg/storage/etcd3/testserver"
49 storagetesting "k8s.io/apiserver/pkg/storage/testing"
50 "k8s.io/apiserver/pkg/storage/value"
51 "k8s.io/klog/v2"
52 )
53
54 var scheme = runtime.NewScheme()
55 var codecs = serializer.NewCodecFactory(scheme)
56
57 const defaultTestPrefix = "test!"
58
59 func init() {
60 metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
61 utilruntime.Must(example.AddToScheme(scheme))
62 utilruntime.Must(examplev1.AddToScheme(scheme))
63
64 grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, os.Stderr))
65 }
66
67 func newPod() runtime.Object {
68 return &example.Pod{}
69 }
70
71 func newPodList() runtime.Object {
72 return &example.PodList{}
73 }
74
75 func checkStorageInvariants(etcdClient *clientv3.Client, codec runtime.Codec) storagetesting.KeyValidation {
76 return func(ctx context.Context, t *testing.T, key string) {
77 getResp, err := etcdClient.KV.Get(ctx, key)
78 if err != nil {
79 t.Fatalf("etcdClient.KV.Get failed: %v", err)
80 }
81 if len(getResp.Kvs) == 0 {
82 t.Fatalf("expecting non empty result on key: %s", key)
83 }
84 decoded, err := runtime.Decode(codec, getResp.Kvs[0].Value[len(defaultTestPrefix):])
85 if err != nil {
86 t.Fatalf("expecting successful decode of object from %v\n%v", err, string(getResp.Kvs[0].Value))
87 }
88 obj := decoded.(*example.Pod)
89 if obj.ResourceVersion != "" {
90 t.Errorf("stored object should have empty resource version")
91 }
92 if obj.SelfLink != "" {
93 t.Errorf("stored output should have empty selfLink")
94 }
95 }
96 }
97
98 func TestCreate(t *testing.T) {
99 ctx, store, etcdClient := testSetup(t)
100 storagetesting.RunTestCreate(ctx, t, store, checkStorageInvariants(etcdClient.Client, store.codec))
101 }
102
103 func TestCreateWithTTL(t *testing.T) {
104 ctx, store, _ := testSetup(t)
105 storagetesting.RunTestCreateWithTTL(ctx, t, store)
106 }
107
108 func TestCreateWithKeyExist(t *testing.T) {
109 ctx, store, _ := testSetup(t)
110 storagetesting.RunTestCreateWithKeyExist(ctx, t, store)
111 }
112
113 func TestGet(t *testing.T) {
114 ctx, store, _ := testSetup(t)
115 storagetesting.RunTestGet(ctx, t, store)
116 }
117
118 func TestUnconditionalDelete(t *testing.T) {
119 ctx, store, _ := testSetup(t)
120 storagetesting.RunTestUnconditionalDelete(ctx, t, store)
121 }
122
123 func TestConditionalDelete(t *testing.T) {
124 ctx, store, _ := testSetup(t)
125 storagetesting.RunTestConditionalDelete(ctx, t, store)
126 }
127
128 func TestDeleteWithSuggestion(t *testing.T) {
129 ctx, store, _ := testSetup(t)
130 storagetesting.RunTestDeleteWithSuggestion(ctx, t, store)
131 }
132
133 func TestDeleteWithSuggestionAndConflict(t *testing.T) {
134 ctx, store, _ := testSetup(t)
135 storagetesting.RunTestDeleteWithSuggestionAndConflict(ctx, t, store)
136 }
137
138 func TestDeleteWithSuggestionOfDeletedObject(t *testing.T) {
139 ctx, store, _ := testSetup(t)
140 storagetesting.RunTestDeleteWithSuggestionOfDeletedObject(ctx, t, store)
141 }
142
143 func TestValidateDeletionWithSuggestion(t *testing.T) {
144 ctx, store, _ := testSetup(t)
145 storagetesting.RunTestValidateDeletionWithSuggestion(ctx, t, store)
146 }
147
148 func TestValidateDeletionWithOnlySuggestionValid(t *testing.T) {
149 ctx, store, _ := testSetup(t)
150 storagetesting.RunTestValidateDeletionWithOnlySuggestionValid(ctx, t, store)
151 }
152
153 func TestDeleteWithConflict(t *testing.T) {
154 ctx, store, _ := testSetup(t)
155 storagetesting.RunTestDeleteWithConflict(ctx, t, store)
156 }
157
158 func TestPreconditionalDeleteWithSuggestion(t *testing.T) {
159 ctx, store, _ := testSetup(t)
160 storagetesting.RunTestPreconditionalDeleteWithSuggestion(ctx, t, store)
161 }
162
163 func TestPreconditionalDeleteWithSuggestionPass(t *testing.T) {
164 ctx, store, _ := testSetup(t)
165 storagetesting.RunTestPreconditionalDeleteWithOnlySuggestionPass(ctx, t, store)
166 }
167
168 func TestListPaging(t *testing.T) {
169 ctx, store, _ := testSetup(t)
170 storagetesting.RunTestListPaging(ctx, t, store)
171 }
172
173 func TestGetListNonRecursive(t *testing.T) {
174 ctx, store, client := testSetup(t)
175 storagetesting.RunTestGetListNonRecursive(ctx, t, increaseRV(client.Client), store)
176 }
177
178 func TestGetListRecursivePrefix(t *testing.T) {
179 ctx, store, _ := testSetup(t)
180 storagetesting.RunTestGetListRecursivePrefix(ctx, t, store)
181 }
182
183 type storeWithPrefixTransformer struct {
184 *store
185 }
186
187 func (s *storeWithPrefixTransformer) UpdatePrefixTransformer(modifier storagetesting.PrefixTransformerModifier) func() {
188 originalTransformer := s.transformer.(*storagetesting.PrefixTransformer)
189 transformer := *originalTransformer
190 s.transformer = modifier(&transformer)
191 s.watcher.transformer = modifier(&transformer)
192 return func() {
193 s.transformer = originalTransformer
194 s.watcher.transformer = originalTransformer
195 }
196 }
197
198 type corruptedTransformer struct {
199 value.Transformer
200 }
201
202 func (f *corruptedTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error) {
203 return nil, true, &corruptObjectError{err: fmt.Errorf("bits flipped"), errType: untransformable}
204 }
205
206 type storeWithCorruptedTransformer struct {
207 *store
208 }
209
210 func (s *storeWithCorruptedTransformer) CorruptTransformer() func() {
211 ct := &corruptedTransformer{Transformer: s.transformer}
212 s.transformer = ct
213 s.watcher.transformer = ct
214 return func() {
215 s.transformer = ct.Transformer
216 s.watcher.transformer = ct.Transformer
217 }
218 }
219
220 func TestGuaranteedUpdate(t *testing.T) {
221 ctx, store, client := testSetup(t)
222 storagetesting.RunTestGuaranteedUpdate(ctx, t, &storeWithPrefixTransformer{store}, checkStorageInvariants(client.Client, store.codec))
223 }
224
225 func TestGuaranteedUpdateWithTTL(t *testing.T) {
226 ctx, store, _ := testSetup(t)
227 storagetesting.RunTestGuaranteedUpdateWithTTL(ctx, t, store)
228 }
229
230 func TestGuaranteedUpdateChecksStoredData(t *testing.T) {
231 ctx, store, _ := testSetup(t)
232 storagetesting.RunTestGuaranteedUpdateChecksStoredData(ctx, t, &storeWithPrefixTransformer{store})
233 }
234
235 func TestGuaranteedUpdateWithConflict(t *testing.T) {
236 ctx, store, _ := testSetup(t)
237 storagetesting.RunTestGuaranteedUpdateWithConflict(ctx, t, store)
238 }
239
240 func TestGuaranteedUpdateWithSuggestionAndConflict(t *testing.T) {
241 ctx, store, _ := testSetup(t)
242 storagetesting.RunTestGuaranteedUpdateWithSuggestionAndConflict(ctx, t, store)
243 }
244
245 func TestTransformationFailure(t *testing.T) {
246 ctx, store, _ := testSetup(t)
247 storagetesting.RunTestTransformationFailure(ctx, t, &storeWithPrefixTransformer{store})
248 }
249
250 func TestList(t *testing.T) {
251 ctx, store, client := testSetup(t)
252 storagetesting.RunTestList(ctx, t, store, increaseRV(client.Client), false)
253 }
254
255 func TestConsistentList(t *testing.T) {
256 ctx, store, client := testSetup(t)
257 storagetesting.RunTestConsistentList(ctx, t, store, increaseRV(client.Client), false, true, false)
258 }
259
260 func checkStorageCallsInvariants(transformer *storagetesting.PrefixTransformer, recorder *clientRecorder) storagetesting.CallsValidation {
261 return func(t *testing.T, pageSize, estimatedProcessedObjects uint64) {
262 if reads := transformer.GetReadsAndReset(); reads != estimatedProcessedObjects {
263 t.Errorf("unexpected reads: %d, expected: %d", reads, estimatedProcessedObjects)
264 }
265 estimatedGetCalls := uint64(1)
266 if pageSize != 0 {
267 // We expect that kube-apiserver will be increasing page sizes
268 // if not full pages are received, so we should see significantly less
269 // than 1000 pages (which would be result of talking to etcd with page size
270 // copied from pred.Limit).
271 // The expected number of calls is n+1 where n is the smallest n so that:
272 // pageSize + pageSize * 2 + pageSize * 4 + ... + pageSize * 2^n >= podCount.
273 // For pageSize = 1, podCount = 1000, we get n+1 = 10, 2 ^ 10 = 1024.
274 currLimit := pageSize
275 for sum := uint64(1); sum < estimatedProcessedObjects; {
276 currLimit *= 2
277 if currLimit > maxLimit {
278 currLimit = maxLimit
279 }
280 sum += currLimit
281 estimatedGetCalls++
282 }
283 }
284 if reads := recorder.GetReadsAndReset(); reads != estimatedGetCalls {
285 t.Fatalf("unexpected reads: %d, want: %d", reads, estimatedGetCalls)
286 }
287 }
288 }
289
290 func TestListContinuation(t *testing.T) {
291 ctx, store, client := testSetup(t, withRecorder())
292 validation := checkStorageCallsInvariants(
293 store.transformer.(*storagetesting.PrefixTransformer), client.KV.(*clientRecorder))
294 storagetesting.RunTestListContinuation(ctx, t, store, validation)
295 }
296
297 func TestListPaginationRareObject(t *testing.T) {
298 ctx, store, client := testSetup(t, withRecorder())
299 validation := checkStorageCallsInvariants(
300 store.transformer.(*storagetesting.PrefixTransformer), client.KV.(*clientRecorder))
301 storagetesting.RunTestListPaginationRareObject(ctx, t, store, validation)
302 }
303
304 func TestListContinuationWithFilter(t *testing.T) {
305 ctx, store, client := testSetup(t, withRecorder())
306 validation := checkStorageCallsInvariants(
307 store.transformer.(*storagetesting.PrefixTransformer), client.KV.(*clientRecorder))
308 storagetesting.RunTestListContinuationWithFilter(ctx, t, store, validation)
309 }
310
311 func TestNamespaceScopedList(t *testing.T) {
312 ctx, store, _ := testSetup(t)
313 storagetesting.RunTestNamespaceScopedList(ctx, t, store)
314 }
315
316 func compactStorage(client *clientv3.Client) storagetesting.Compaction {
317 return func(ctx context.Context, t *testing.T, resourceVersion string) {
318 versioner := storage.APIObjectVersioner{}
319 rv, err := versioner.ParseResourceVersion(resourceVersion)
320 if err != nil {
321 t.Fatal(err)
322 }
323 if _, err = client.Compact(ctx, int64(rv)); err != nil {
324 t.Fatalf("Unable to compact, %v", err)
325 }
326 }
327 }
328
329 func increaseRV(client *clientv3.Client) storagetesting.IncreaseRVFunc {
330 return func(ctx context.Context, t *testing.T) {
331 if _, err := client.KV.Put(ctx, "increaseRV", "ok"); err != nil {
332 t.Fatalf("Could not update increaseRV: %v", err)
333 }
334 }
335 }
336
337 func TestListInconsistentContinuation(t *testing.T) {
338 ctx, store, client := testSetup(t)
339 storagetesting.RunTestListInconsistentContinuation(ctx, t, store, compactStorage(client.Client))
340 }
341
342 func TestListResourceVersionMatch(t *testing.T) {
343 ctx, store, _ := testSetup(t)
344 storagetesting.RunTestListResourceVersionMatch(ctx, t, &storeWithPrefixTransformer{store})
345 }
346
347 func TestCount(t *testing.T) {
348 ctx, store, _ := testSetup(t)
349 storagetesting.RunTestCount(ctx, t, store)
350 }
351
352 // =======================================================================
353 // Implementation-specific tests are following.
354 // The following tests are exercising the details of the implementation
355 // not the actual user-facing contract of storage interface.
356 // As such, they may focus e.g. on non-functional aspects like performance
357 // impact.
358 // =======================================================================
359
360 func TestPrefix(t *testing.T) {
361 testcases := map[string]string{
362 "custom/prefix": "/custom/prefix/",
363 "/custom//prefix//": "/custom/prefix/",
364 "/registry": "/registry/",
365 }
366 for configuredPrefix, effectivePrefix := range testcases {
367 _, store, _ := testSetup(t, withPrefix(configuredPrefix))
368 if store.pathPrefix != effectivePrefix {
369 t.Errorf("configured prefix of %s, expected effective prefix of %s, got %s", configuredPrefix, effectivePrefix, store.pathPrefix)
370 }
371 }
372 }
373
374 func Test_growSlice(t *testing.T) {
375 type args struct {
376 initialCapacity int
377 initialLen int
378 v reflect.Value
379 maxCapacity int
380 sizes []int
381 }
382 tests := []struct {
383 name string
384 args args
385 cap int
386 len int
387 }{
388 {
389 name: "empty",
390 args: args{v: reflect.ValueOf([]example.Pod{})},
391 cap: 0,
392 },
393 {
394 name: "no sizes",
395 args: args{v: reflect.ValueOf([]example.Pod{}), maxCapacity: 10},
396 cap: 10,
397 },
398 {
399 name: "above maxCapacity",
400 args: args{v: reflect.ValueOf([]example.Pod{}), maxCapacity: 10, sizes: []int{1, 12}},
401 cap: 10,
402 },
403 {
404 name: "takes max",
405 args: args{v: reflect.ValueOf([]example.Pod{}), maxCapacity: 10, sizes: []int{8, 4}},
406 cap: 8,
407 },
408 {
409 name: "with existing capacity above max",
410 args: args{initialCapacity: 12, maxCapacity: 10, sizes: []int{8, 4}},
411 cap: 12,
412 },
413 {
414 name: "with existing capacity below max",
415 args: args{initialCapacity: 5, maxCapacity: 10, sizes: []int{8, 4}},
416 cap: 8,
417 },
418 {
419 name: "with existing capacity and length above max",
420 args: args{initialCapacity: 12, initialLen: 5, maxCapacity: 10, sizes: []int{8, 4}},
421 cap: 12,
422 len: 5,
423 },
424 {
425 name: "with existing capacity and length below max",
426 args: args{initialCapacity: 5, initialLen: 3, maxCapacity: 10, sizes: []int{8, 4}},
427 cap: 8,
428 len: 3,
429 },
430 }
431 for _, tt := range tests {
432 t.Run(tt.name, func(t *testing.T) {
433 if tt.args.initialCapacity > 0 {
434 val := make([]example.Pod, tt.args.initialLen, tt.args.initialCapacity)
435 for i := 0; i < tt.args.initialLen; i++ {
436 val[i].Name = fmt.Sprintf("test-%d", i)
437 }
438 tt.args.v = reflect.ValueOf(val)
439 }
440 // reflection requires that the value be addressable in order to call set,
441 // so we must ensure the value we created is available on the heap (not a problem
442 // for normal usage)
443 if !tt.args.v.CanAddr() {
444 x := reflect.New(tt.args.v.Type())
445 x.Elem().Set(tt.args.v)
446 tt.args.v = x.Elem()
447 }
448 growSlice(tt.args.v, tt.args.maxCapacity, tt.args.sizes...)
449 if tt.cap != tt.args.v.Cap() {
450 t.Errorf("Unexpected capacity: got=%d want=%d", tt.args.v.Cap(), tt.cap)
451 }
452 if tt.len != tt.args.v.Len() {
453 t.Errorf("Unexpected length: got=%d want=%d", tt.args.v.Len(), tt.len)
454 }
455 for i := 0; i < tt.args.v.Len(); i++ {
456 nameWanted := fmt.Sprintf("test-%d", i)
457 val := tt.args.v.Index(i).Interface()
458 pod, ok := val.(example.Pod)
459 if !ok || pod.Name != nameWanted {
460 t.Errorf("Unexpected element value: got=%s, want=%s", pod.Name, nameWanted)
461 }
462 }
463 })
464 }
465 }
466
467 func TestLeaseMaxObjectCount(t *testing.T) {
468 ctx, store, _ := testSetup(t, withLeaseConfig(LeaseManagerConfig{
469 ReuseDurationSeconds: defaultLeaseReuseDurationSeconds,
470 MaxObjectCount: 2,
471 }))
472
473 obj := &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}
474 out := &example.Pod{}
475
476 testCases := []struct {
477 key string
478 expectAttachedCount int64
479 }{
480 {
481 key: "testkey1",
482 expectAttachedCount: 1,
483 },
484 {
485 key: "testkey2",
486 expectAttachedCount: 2,
487 },
488 {
489 key: "testkey3",
490 // We assume each time has 1 object attached to the lease
491 // so after granting a new lease, the recorded count is set to 1
492 expectAttachedCount: 1,
493 },
494 }
495
496 for _, tc := range testCases {
497 err := store.Create(ctx, tc.key, obj, out, 120)
498 if err != nil {
499 t.Fatalf("Set failed: %v", err)
500 }
501 if store.leaseManager.leaseAttachedObjectCount != tc.expectAttachedCount {
502 t.Errorf("Lease manager recorded count %v should be %v", store.leaseManager.leaseAttachedObjectCount, tc.expectAttachedCount)
503 }
504 }
505 }
506
507 // ===================================================
508 // Test-setup related function are following.
509 // ===================================================
510
511 func newTestLeaseManagerConfig() LeaseManagerConfig {
512 cfg := NewDefaultLeaseManagerConfig()
513 // As 30s is the default timeout for testing in global configuration,
514 // we cannot wait longer than that in a single time: change it to 1s
515 // for testing purposes. See wait.ForeverTestTimeout
516 cfg.ReuseDurationSeconds = 1
517 return cfg
518 }
519
520 func newTestTransformer() value.Transformer {
521 return storagetesting.NewPrefixTransformer([]byte(defaultTestPrefix), false)
522 }
523
524 type clientRecorder struct {
525 reads uint64
526 clientv3.KV
527 }
528
529 func (r *clientRecorder) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
530 atomic.AddUint64(&r.reads, 1)
531 return r.KV.Get(ctx, key, opts...)
532 }
533
534 func (r *clientRecorder) GetReadsAndReset() uint64 {
535 return atomic.SwapUint64(&r.reads, 0)
536 }
537
538 type setupOptions struct {
539 client func(testing.TB) *kubernetes.Client
540 codec runtime.Codec
541 newFunc func() runtime.Object
542 newListFunc func() runtime.Object
543 prefix string
544 resourcePrefix string
545 groupResource schema.GroupResource
546 transformer value.Transformer
547 leaseConfig LeaseManagerConfig
548
549 recorderEnabled bool
550 }
551
552 type setupOption func(*setupOptions)
553
554 func withClientConfig(config *embed.Config) setupOption {
555 return func(options *setupOptions) {
556 options.client = func(t testing.TB) *kubernetes.Client {
557 return testserver.RunEtcd(t, config)
558 }
559 }
560 }
561
562 func withPrefix(prefix string) setupOption {
563 return func(options *setupOptions) {
564 options.prefix = prefix
565 }
566 }
567
568 func withLeaseConfig(leaseConfig LeaseManagerConfig) setupOption {
569 return func(options *setupOptions) {
570 options.leaseConfig = leaseConfig
571 }
572 }
573
574 func withRecorder() setupOption {
575 return func(options *setupOptions) {
576 options.recorderEnabled = true
577 }
578 }
579
580 func withDefaults(options *setupOptions) {
581 options.client = func(t testing.TB) *kubernetes.Client {
582 return testserver.RunEtcd(t, nil)
583 }
584 options.codec = apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)
585 options.newFunc = newPod
586 options.newListFunc = newPodList
587 options.prefix = ""
588 options.resourcePrefix = "/pods"
589 options.groupResource = schema.GroupResource{Resource: "pods"}
590 options.transformer = newTestTransformer()
591 options.leaseConfig = newTestLeaseManagerConfig()
592 }
593
594 var _ setupOption = withDefaults
595
596 func testSetup(t testing.TB, opts ...setupOption) (context.Context, *store, *kubernetes.Client) {
597 setupOpts := setupOptions{}
598 opts = append([]setupOption{withDefaults}, opts...)
599 for _, opt := range opts {
600 opt(&setupOpts)
601 }
602 client := setupOpts.client(t)
603 if setupOpts.recorderEnabled {
604 client.KV = &clientRecorder{KV: client.KV}
605 }
606 versioner := storage.APIObjectVersioner{}
607 store := newStore(
608 client,
609 setupOpts.codec,
610 setupOpts.newFunc,
611 setupOpts.newListFunc,
612 setupOpts.prefix,
613 setupOpts.resourcePrefix,
614 setupOpts.groupResource,
615 setupOpts.transformer,
616 setupOpts.leaseConfig,
617 NewDefaultDecoder(setupOpts.codec, versioner),
618 versioner,
619 )
620 ctx := context.Background()
621 return ctx, store, client
622 }
623
624 func TestValidateKey(t *testing.T) {
625 validKeys := []string{
626 "/foo/bar/baz/a.b.c/",
627 "/foo",
628 "foo/bar/baz",
629 "/foo/bar..baz/",
630 "/foo/bar..",
631 "foo",
632 "foo/bar",
633 "/foo/bar/",
634 }
635 invalidKeys := []string{
636 "/foo/bar/../a.b.c/",
637 "..",
638 "/..",
639 "../",
640 "/foo/bar/..",
641 "../foo/bar",
642 "/../foo",
643 "/foo/bar/../",
644 ".",
645 "/.",
646 "./",
647 "/./",
648 "/foo/.",
649 "./bar",
650 "/foo/./bar/",
651 }
652 const (
653 pathPrefix = "/first/second"
654 expectPrefix = pathPrefix + "/"
655 )
656 _, store, _ := testSetup(t, withPrefix(pathPrefix))
657
658 for _, key := range validKeys {
659 k, err := store.prepareKey(key)
660 if err != nil {
661 t.Errorf("key %q should be valid; unexpected error: %v", key, err)
662 } else if !strings.HasPrefix(k, expectPrefix) {
663 t.Errorf("key %q should have prefix %q", k, expectPrefix)
664 }
665 }
666
667 for _, key := range invalidKeys {
668 _, err := store.prepareKey(key)
669 if err == nil {
670 t.Errorf("key %q should be invalid", key)
671 }
672 }
673 }
674
675 func TestInvalidKeys(t *testing.T) {
676 const invalidKey = "/foo/bar/../baz"
677 expectedError := fmt.Sprintf("invalid key: %q", invalidKey)
678
679 expectInvalidKey := func(methodName string, err error) {
680 if err == nil {
681 t.Errorf("[%s] expected invalid key error; got nil", methodName)
682 } else if err.Error() != expectedError {
683 t.Errorf("[%s] expected invalid key error; got %v", methodName, err)
684 }
685 }
686
687 ctx, store, _ := testSetup(t)
688 expectInvalidKey("Create", store.Create(ctx, invalidKey, nil, nil, 0))
689 expectInvalidKey("Delete", store.Delete(ctx, invalidKey, nil, nil, nil, nil, storage.DeleteOptions{}))
690 _, watchErr := store.Watch(ctx, invalidKey, storage.ListOptions{})
691 expectInvalidKey("Watch", watchErr)
692 expectInvalidKey("Get", store.Get(ctx, invalidKey, storage.GetOptions{}, nil))
693 expectInvalidKey("GetList", store.GetList(ctx, invalidKey, storage.ListOptions{}, nil))
694 expectInvalidKey("GuaranteedUpdate", store.GuaranteedUpdate(ctx, invalidKey, nil, true, nil, nil, nil))
695 _, countErr := store.Count(invalidKey)
696 expectInvalidKey("Count", countErr)
697 }
698
699 func BenchmarkStore_GetList(b *testing.B) {
700 generateBigPod := func(index int, total int, expect int) runtime.Object {
701 l := map[string]string{}
702 if index%(total/expect) == 0 {
703 l["foo"] = "bar"
704 }
705 terminationGracePeriodSeconds := int64(42)
706 activeDeadlineSeconds := int64(42)
707 pod := &examplev1.Pod{
708 ObjectMeta: metav1.ObjectMeta{
709 Labels: l,
710 },
711 Spec: examplev1.PodSpec{
712 RestartPolicy: examplev1.RestartPolicy("Always"),
713 TerminationGracePeriodSeconds: &terminationGracePeriodSeconds,
714 ActiveDeadlineSeconds: &activeDeadlineSeconds,
715 NodeSelector: map[string]string{},
716 ServiceAccountName: "demo-sa",
717 },
718 }
719 pod.Name = fmt.Sprintf("object-%d", index)
720 data := make([]byte, 1024*2, 1024*2) // 2k labels
721 rand.Read(data)
722 pod.Spec.NodeSelector["key"] = string(data)
723 return pod
724 }
725 testCases := []struct {
726 name string
727 objectNum int
728 expectNum int
729 selector labels.Selector
730 newObjectFunc func(index int, total int, expect int) runtime.Object
731 newListObjectFunc func() runtime.Object
732 }{
733 {
734 name: "pick 50 pods out of 5000 pod",
735 objectNum: 5000,
736 expectNum: 50,
737 selector: labels.SelectorFromSet(map[string]string{"foo": "bar"}),
738 newObjectFunc: generateBigPod,
739 newListObjectFunc: func() runtime.Object { return &examplev1.PodList{} },
740 },
741 {
742 name: "pick 500 pods out of 5000 pod",
743 objectNum: 5000,
744 expectNum: 500,
745 selector: labels.SelectorFromSet(map[string]string{"foo": "bar"}),
746 newObjectFunc: generateBigPod,
747 newListObjectFunc: func() runtime.Object { return &examplev1.PodList{} },
748 },
749 {
750 name: "pick 1000 pods out of 5000 pod",
751 objectNum: 5000,
752 expectNum: 1000,
753 selector: labels.SelectorFromSet(map[string]string{"foo": "bar"}),
754 newObjectFunc: generateBigPod,
755 newListObjectFunc: func() runtime.Object { return &examplev1.PodList{} },
756 },
757 {
758 name: "pick 2500 pods out of 5000 pod",
759 objectNum: 5000,
760 expectNum: 2500,
761 selector: labels.SelectorFromSet(map[string]string{"foo": "bar"}),
762 newObjectFunc: generateBigPod,
763 newListObjectFunc: func() runtime.Object { return &examplev1.PodList{} },
764 },
765 {
766 name: "pick 5000 pods out of 5000 pod",
767 objectNum: 5000,
768 expectNum: 5000,
769 selector: labels.SelectorFromSet(map[string]string{"foo": "bar"}),
770 newObjectFunc: generateBigPod,
771 newListObjectFunc: func() runtime.Object { return &examplev1.PodList{} },
772 },
773 }
774 for _, tc := range testCases {
775 b.Run(tc.name, func(b *testing.B) {
776 // booting etcd instance
777 ctx, store, etcdClient := testSetup(b)
778 defer etcdClient.Close()
779
780 // make fake objects..
781 dir := "/testing"
782 originalRevision := ""
783 for i := 0; i < tc.objectNum; i++ {
784 obj := tc.newObjectFunc(i, tc.objectNum, tc.expectNum)
785 o := obj.(metav1.Object)
786 key := fmt.Sprintf("/testing/testkey/%s", o.GetName())
787 out := tc.newObjectFunc(i, tc.objectNum, tc.expectNum)
788 if err := store.Create(ctx, key, obj, out, 0); err != nil {
789 b.Fatalf("Set failed: %v", err)
790 }
791 originalRevision = out.(metav1.Object).GetResourceVersion()
792 }
793
794 // prepare result and pred
795 pred := storage.SelectionPredicate{
796 Label: tc.selector,
797 Field: fields.Everything(),
798 GetAttrs: func(obj runtime.Object) (labels.Set, fields.Set, error) {
799 pod, ok := obj.(*examplev1.Pod)
800 if !ok {
801 return nil, nil, fmt.Errorf("not a pod")
802 }
803 return pod.ObjectMeta.Labels, fields.Set{
804 "metadata.name": pod.Name,
805 }, nil
806 },
807 }
808
809 // now we start benchmarking
810 b.ResetTimer()
811 for i := 0; i < b.N; i++ {
812 list := tc.newListObjectFunc()
813 if err := store.GetList(ctx, dir, storage.ListOptions{Predicate: pred, Recursive: true}, list); err != nil {
814 b.Errorf("Unexpected List error: %v", err)
815 }
816 listObject := list.(*examplev1.PodList)
817 if originalRevision != listObject.GetResourceVersion() {
818 b.Fatalf("original revision (%s) did not match final revision after linearized reads (%s)", originalRevision, listObject.GetResourceVersion())
819 }
820 if len(listObject.Items) != tc.expectNum {
821 b.Fatalf("expect (%d) items but got (%d)", tc.expectNum, len(listObject.Items))
822 }
823 }
824 })
825 }
826 }
827
828 func BenchmarkStoreListCreate(b *testing.B) {
829 klog.SetLogger(logr.Discard())
830 b.Run("RV=NotOlderThan", func(b *testing.B) {
831 ctx, store, _ := testSetup(b)
832 storagetesting.RunBenchmarkStoreListCreate(ctx, b, store, metav1.ResourceVersionMatchNotOlderThan)
833 })
834 b.Run("RV=ExactMatch", func(b *testing.B) {
835 ctx, store, _ := testSetup(b)
836 storagetesting.RunBenchmarkStoreListCreate(ctx, b, store, metav1.ResourceVersionMatchExact)
837 })
838 }
839
840 func BenchmarkStoreList(b *testing.B) {
841 klog.SetLogger(logr.Discard())
842 // Based on https://github.com/kubernetes/community/blob/master/sig-scalability/configs-and-limits/thresholds.md
843 dimensions := []struct {
844 namespaceCount int
845 podPerNamespaceCount int
846 nodeCount int
847 }{
848 {
849 namespaceCount: 10_000,
850 podPerNamespaceCount: 15,
851 nodeCount: 5_000,
852 },
853 {
854 namespaceCount: 50,
855 podPerNamespaceCount: 3_000,
856 nodeCount: 5_000,
857 },
858 {
859 namespaceCount: 100,
860 podPerNamespaceCount: 1_100,
861 nodeCount: 1000,
862 },
863 }
864 for _, dims := range dimensions {
865 b.Run(fmt.Sprintf("Namespaces=%d/Pods=%d/Nodes=%d", dims.namespaceCount, dims.namespaceCount*dims.podPerNamespaceCount, dims.nodeCount), func(b *testing.B) {
866 data := storagetesting.PrepareBenchchmarkData(dims.namespaceCount, dims.podPerNamespaceCount, dims.nodeCount)
867 ctx, store, _ := testSetup(b)
868 var out example.Pod
869 for _, pod := range data.Pods {
870 err := store.Create(ctx, computePodKey(pod), pod, &out, 0)
871 if err != nil {
872 b.Fatal(err)
873 }
874 }
875 storagetesting.RunBenchmarkStoreList(ctx, b, store, data, false)
876 })
877 }
878 }
879
880 func computePodKey(obj *example.Pod) string {
881 return fmt.Sprintf("/pods/%s/%s", obj.Namespace, obj.Name)
882 }
883
884 func TestGetCurrentResourceVersion(t *testing.T) {
885 ctx, store, _ := testSetup(t)
886
887 makePod := func(name string) *example.Pod {
888 return &example.Pod{
889 ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: name},
890 }
891 }
892 createPod := func(obj *example.Pod) *example.Pod {
893 key := "pods/" + obj.Namespace + "/" + obj.Name
894 out := &example.Pod{}
895 err := store.Create(context.TODO(), key, obj, out, 0)
896 require.NoError(t, err)
897 return out
898 }
899 getPod := func(name, ns string) *example.Pod {
900 key := "pods/" + ns + "/" + name
901 out := &example.Pod{}
902 err := store.Get(context.TODO(), key, storage.GetOptions{}, out)
903 require.NoError(t, err)
904 return out
905 }
906
907 // create a pod and make sure its RV is equal to the one maintained by etcd
908 pod := createPod(makePod("pod-1"))
909 currentStorageRV, err := store.GetCurrentResourceVersion(context.TODO())
910 require.NoError(t, err)
911 podRV, err := store.versioner.ParseResourceVersion(pod.ResourceVersion)
912 require.NoError(t, err)
913 require.Equal(t, currentStorageRV, podRV, "expected the global etcd RV to be equal to pod's RV")
914
915 // now make unrelated write and make sure the target function returns global etcd RV
916 resp, err := store.client.KV.Put(ctx, "compact_rev_key", pod.ResourceVersion)
917 require.NoError(t, err)
918 currentStorageRV, err = store.GetCurrentResourceVersion(context.TODO())
919 require.NoError(t, err)
920 require.NoError(t, err)
921 require.Equal(t, currentStorageRV, uint64(resp.Header.Revision), "expected the global etcd RV to be equal to replicaset's RV")
922
923 // ensure that the pod's RV hasn't been changed
924 currentPod := getPod(pod.Name, pod.Namespace)
925 currentPodRV, err := store.versioner.ParseResourceVersion(currentPod.ResourceVersion)
926 require.NoError(t, err)
927 require.Equal(t, currentPodRV, podRV, "didn't expect to see the pod's RV changed")
928 }