Debian Codegraph @

pkg/storage/storagebackend/factory/factory_test.go

raw
1 /*
2 Copyright 2022 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package factory
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "strings"
24 "sync"
25 "sync/atomic"
26 "testing"
27 "time"
28
29 clientv3 "go.etcd.io/etcd/client/v3"
30 "go.etcd.io/etcd/client/v3/kubernetes"
31 "k8s.io/apiserver/pkg/storage/etcd3/testserver"
32 "k8s.io/apiserver/pkg/storage/storagebackend"
33 )
34
35 type mockKV struct {
36 get func(ctx context.Context) (*clientv3.GetResponse, error)
37 }
38
39 func (mkv mockKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
40 return nil, nil
41 }
42 func (mkv mockKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
43 return mkv.get(ctx)
44 }
45 func (mockKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {
46 return nil, nil
47 }
48 func (mockKV) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {
49 return nil, nil
50 }
51 func (mockKV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) {
52 return clientv3.OpResponse{}, nil
53 }
54 func (mockKV) Txn(ctx context.Context) clientv3.Txn {
55 return nil
56 }
57
58 func TestCreateHealthcheck(t *testing.T) {
59 etcdConfig := testserver.NewTestConfig(t)
60 client := testserver.RunEtcd(t, etcdConfig)
61 newETCD3ClientFn := newETCD3Client
62 defer func() {
63 newETCD3Client = newETCD3ClientFn
64 }()
65 tests := []struct {
66 name string
67 cfg storagebackend.Config
68 want error
69 responseTime time.Duration
70 }{
71 {
72 name: "ok if response time lower than default timeout",
73 cfg: storagebackend.Config{
74 Type: storagebackend.StorageTypeETCD3,
75 Transport: storagebackend.TransportConfig{},
76 },
77 responseTime: 1 * time.Second,
78 want: nil,
79 },
80 {
81 name: "ok if response time lower than custom timeout",
82 cfg: storagebackend.Config{
83 Type: storagebackend.StorageTypeETCD3,
84 Transport: storagebackend.TransportConfig{},
85 HealthcheckTimeout: 5 * time.Second,
86 },
87 responseTime: 3 * time.Second,
88 want: nil,
89 },
90 {
91 name: "timeouts if response time higher than default timeout",
92 cfg: storagebackend.Config{
93 Type: storagebackend.StorageTypeETCD3,
94 Transport: storagebackend.TransportConfig{},
95 },
96 responseTime: 3 * time.Second,
97 want: context.DeadlineExceeded,
98 },
99 {
100 name: "timeouts if response time higher than custom timeout",
101 cfg: storagebackend.Config{
102 Type: storagebackend.StorageTypeETCD3,
103 Transport: storagebackend.TransportConfig{},
104 HealthcheckTimeout: 3 * time.Second,
105 },
106 responseTime: 5 * time.Second,
107 want: context.DeadlineExceeded,
108 },
109 }
110
111 for _, tc := range tests {
112 t.Run(tc.name, func(t *testing.T) {
113 ready := make(chan struct{})
114 tc.cfg.Transport.ServerList = client.Endpoints()
115 newETCD3Client = func(c storagebackend.TransportConfig) (*kubernetes.Client, error) {
116 defer close(ready)
117 dummyKV := mockKV{
118 get: func(ctx context.Context) (*clientv3.GetResponse, error) {
119 select {
120 case <-ctx.Done():
121 return nil, ctx.Err()
122 case <-time.After(tc.responseTime):
123 return nil, nil
124 }
125 },
126 }
127 client.KV = dummyKV
128 return client, nil
129 }
130 stop := make(chan struct{})
131 defer close(stop)
132
133 healthcheck, err := CreateHealthCheck(tc.cfg, stop)
134 if err != nil {
135 t.Fatal(err)
136 }
137 // Wait for healthcheck to establish connection
138 <-ready
139 got := healthcheck()
140
141 if !errors.Is(got, tc.want) {
142 t.Errorf("healthcheck() missmatch want %v got %v", tc.want, got)
143 }
144 })
145 }
146 }
147
148 func TestCreateReadycheck(t *testing.T) {
149 etcdConfig := testserver.NewTestConfig(t)
150 client := testserver.RunEtcd(t, etcdConfig)
151 newETCD3ClientFn := newETCD3Client
152 defer func() {
153 newETCD3Client = newETCD3ClientFn
154 }()
155 tests := []struct {
156 name string
157 cfg storagebackend.Config
158 want error
159 responseTime time.Duration
160 }{
161 {
162 name: "ok if response time lower than default timeout",
163 cfg: storagebackend.Config{
164 Type: storagebackend.StorageTypeETCD3,
165 Transport: storagebackend.TransportConfig{},
166 },
167 responseTime: 1 * time.Second,
168 want: nil,
169 },
170 {
171 name: "ok if response time lower than custom timeout",
172 cfg: storagebackend.Config{
173 Type: storagebackend.StorageTypeETCD3,
174 Transport: storagebackend.TransportConfig{},
175 ReadycheckTimeout: 5 * time.Second,
176 },
177 responseTime: 3 * time.Second,
178 want: nil,
179 },
180 {
181 name: "timeouts if response time higher than default timeout",
182 cfg: storagebackend.Config{
183 Type: storagebackend.StorageTypeETCD3,
184 Transport: storagebackend.TransportConfig{},
185 },
186 responseTime: 3 * time.Second,
187 want: context.DeadlineExceeded,
188 },
189 {
190 name: "timeouts if response time higher than custom timeout",
191 cfg: storagebackend.Config{
192 Type: storagebackend.StorageTypeETCD3,
193 Transport: storagebackend.TransportConfig{},
194 ReadycheckTimeout: 3 * time.Second,
195 },
196 responseTime: 5 * time.Second,
197 want: context.DeadlineExceeded,
198 },
199 {
200 name: "timeouts if response time higher than default timeout with custom healthcheck timeout",
201 cfg: storagebackend.Config{
202 Type: storagebackend.StorageTypeETCD3,
203 Transport: storagebackend.TransportConfig{},
204 HealthcheckTimeout: 10 * time.Second,
205 },
206 responseTime: 3 * time.Second,
207 want: context.DeadlineExceeded,
208 },
209 }
210
211 for _, tc := range tests {
212 t.Run(tc.name, func(t *testing.T) {
213 ready := make(chan struct{})
214 tc.cfg.Transport.ServerList = client.Endpoints()
215 newETCD3Client = func(c storagebackend.TransportConfig) (*kubernetes.Client, error) {
216 defer close(ready)
217 dummyKV := mockKV{
218 get: func(ctx context.Context) (*clientv3.GetResponse, error) {
219 select {
220 case <-ctx.Done():
221 return nil, ctx.Err()
222 case <-time.After(tc.responseTime):
223 return nil, nil
224 }
225 },
226 }
227 client.KV = dummyKV
228 return client, nil
229 }
230 stop := make(chan struct{})
231 defer close(stop)
232
233 healthcheck, err := CreateReadyCheck(tc.cfg, stop)
234 if err != nil {
235 t.Fatal(err)
236 }
237 // Wait for healthcheck to establish connection
238 <-ready
239
240 got := healthcheck()
241
242 if !errors.Is(got, tc.want) {
243 t.Errorf("healthcheck() missmatch want %v got %v", tc.want, got)
244 }
245 })
246 }
247 }
248
249 func TestRateLimitHealthcheck(t *testing.T) {
250 etcdConfig := testserver.NewTestConfig(t)
251 client := testserver.RunEtcd(t, etcdConfig)
252 newETCD3ClientFn := newETCD3Client
253 defer func() {
254 newETCD3Client = newETCD3ClientFn
255 }()
256
257 cfg := storagebackend.Config{
258 Type: storagebackend.StorageTypeETCD3,
259 Transport: storagebackend.TransportConfig{},
260 HealthcheckTimeout: 5 * time.Second,
261 }
262 cfg.Transport.ServerList = client.Endpoints()
263 tests := []struct {
264 name string
265 want error
266 }{
267 {
268 name: "etcd ok",
269 },
270 {
271 name: "etcd down",
272 want: errors.New("etcd down"),
273 },
274 }
275 for _, tc := range tests {
276 t.Run(tc.name, func(t *testing.T) {
277
278 ready := make(chan struct{})
279
280 var counter uint64
281 newETCD3Client = func(c storagebackend.TransportConfig) (*kubernetes.Client, error) {
282 defer close(ready)
283 dummyKV := mockKV{
284 get: func(ctx context.Context) (*clientv3.GetResponse, error) {
285 atomic.AddUint64(&counter, 1)
286 select {
287 case <-ctx.Done():
288 return nil, ctx.Err()
289 default:
290 return nil, tc.want
291 }
292 },
293 }
294 client.KV = dummyKV
295 return client, nil
296 }
297
298 stop := make(chan struct{})
299 defer close(stop)
300 healthcheck, err := CreateHealthCheck(cfg, stop)
301 if err != nil {
302 t.Fatal(err)
303 }
304 // Wait for healthcheck to establish connection
305 <-ready
306 // run a first request to obtain the state
307 err = healthcheck()
308 if !errors.Is(err, tc.want) {
309 t.Errorf("healthcheck() mismatch want %v got %v", tc.want, err)
310 }
311
312 // run multiple request in parallel, they should have the same state that the first one
313 var wg sync.WaitGroup
314 for i := 0; i < 100; i++ {
315 wg.Add(1)
316 go func() {
317 defer wg.Done()
318 err := healthcheck()
319 if !errors.Is(err, tc.want) {
320 t.Errorf("healthcheck() mismatch want %v got %v", tc.want, err)
321 }
322
323 }()
324 }
325
326 // check the counter once the requests have finished
327 wg.Wait()
328 if counter != 1 {
329 t.Errorf("healthcheck() called etcd %d times, expected only one call", counter)
330 }
331
332 // wait until the rate limit allows new connections
333 time.Sleep(cfg.HealthcheckTimeout / 2)
334
335 // a new run on request should increment the counter only once
336 // run multiple request in parallel, they should have the same state that the first one
337 for i := 0; i < 100; i++ {
338 wg.Add(1)
339 go func() {
340 defer wg.Done()
341 err := healthcheck()
342 if !errors.Is(err, tc.want) {
343 t.Errorf("healthcheck() mismatch want %v got %v", tc.want, err)
344 }
345
346 }()
347 }
348 wg.Wait()
349
350 if counter != 2 {
351 t.Errorf("healthcheck() called etcd %d times, expected only two calls", counter)
352 }
353 })
354 }
355
356 }
357
358 func TestTimeTravelHealthcheck(t *testing.T) {
359 etcdConfig := testserver.NewTestConfig(t)
360 client := testserver.RunEtcd(t, etcdConfig)
361 newETCD3ClientFn := newETCD3Client
362 defer func() {
363 newETCD3Client = newETCD3ClientFn
364 }()
365
366 cfg := storagebackend.Config{
367 Type: storagebackend.StorageTypeETCD3,
368 Transport: storagebackend.TransportConfig{},
369 HealthcheckTimeout: 5 * time.Second,
370 }
371 cfg.Transport.ServerList = client.Endpoints()
372
373 ready := make(chan struct{})
374 signal := make(chan struct{})
375
376 var counter uint64
377 newETCD3Client = func(c storagebackend.TransportConfig) (*kubernetes.Client, error) {
378 defer close(ready)
379 dummyKV := mockKV{
380 get: func(ctx context.Context) (*clientv3.GetResponse, error) {
381 atomic.AddUint64(&counter, 1)
382 val := atomic.LoadUint64(&counter)
383 // the first request wait for a custom timeout to trigger an error.
384 // We don't use the context timeout because we want to check that
385 // the cached answer is not overridden, and since the rate limit is
386 // based on cfg.HealthcheckTimeout / 2, the timeout will race with
387 // the race limiter to server the new request from the cache or allow
388 // it to go through
389 if val == 1 {
390 select {
391 case <-ctx.Done():
392 return nil, ctx.Err()
393 case <-time.After((2 * cfg.HealthcheckTimeout) / 3):
394 return nil, fmt.Errorf("etcd down")
395 }
396 }
397 // subsequent requests will always work
398 return nil, nil
399 },
400 }
401 client.KV = dummyKV
402 return client, nil
403 }
404
405 stop := make(chan struct{})
406 defer close(stop)
407 healthcheck, err := CreateHealthCheck(cfg, stop)
408 if err != nil {
409 t.Fatal(err)
410 }
411 // Wait for healthcheck to establish connection
412 <-ready
413 // run a first request that fails after 2 seconds
414 go func() {
415 err := healthcheck()
416 if !strings.Contains(err.Error(), "etcd down") {
417 t.Errorf("healthcheck() mismatch want %v got %v", fmt.Errorf("etcd down"), err)
418 }
419 close(signal)
420 }()
421
422 // wait until the rate limit allows new connections
423 time.Sleep(cfg.HealthcheckTimeout / 2)
424
425 select {
426 case <-signal:
427 t.Errorf("first request should not return yet")
428 default:
429 }
430
431 // a new run on request should succeed and increment the counter
432 err = healthcheck()
433 if err != nil {
434 t.Errorf("unexpected error: %v", err)
435 }
436 c := atomic.LoadUint64(&counter)
437 if c != 2 {
438 t.Errorf("healthcheck() called etcd %d times, expected only two calls", c)
439 }
440
441 // cached request should be success and not be overridden by the late error
442 <-signal
443 err = healthcheck()
444 if err != nil {
445 t.Errorf("unexpected error: %v", err)
446 }
447 c = atomic.LoadUint64(&counter)
448 if c != 2 {
449 t.Errorf("healthcheck() called etcd %d times, expected only two calls", c)
450 }
451
452 }