Debian Codegraph @

pkg/server/filters/priority-and-fairness_test.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package filters
18
19 import (
20 "context"
21 "fmt"
22 "net/http"
23 "net/http/httptest"
24 "net/url"
25 "os"
26 "reflect"
27 "strings"
28 "sync"
29 "testing"
30 "time"
31
32 flowcontrol "k8s.io/api/flowcontrol/v1"
33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34 "k8s.io/apimachinery/pkg/runtime"
35 "k8s.io/apimachinery/pkg/types"
36 "k8s.io/apimachinery/pkg/util/sets"
37 "k8s.io/apimachinery/pkg/util/wait"
38 "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
39 "k8s.io/apiserver/pkg/authentication/user"
40 apifilters "k8s.io/apiserver/pkg/endpoints/filters"
41 epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
42 apirequest "k8s.io/apiserver/pkg/endpoints/request"
43 "k8s.io/apiserver/pkg/server/mux"
44 utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
45 fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
46 fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
47 fcrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
48 "k8s.io/client-go/informers"
49 clientset "k8s.io/client-go/kubernetes"
50 "k8s.io/client-go/kubernetes/fake"
51 "k8s.io/component-base/metrics/legacyregistry"
52 "k8s.io/component-base/metrics/testutil"
53 "k8s.io/klog/v2"
54 clocktesting "k8s.io/utils/clock/testing"
55 "k8s.io/utils/ptr"
56
57 "github.com/google/go-cmp/cmp"
58 )
59
60 func TestMain(m *testing.M) {
61 klog.InitFlags(nil)
62 os.Exit(m.Run())
63 }
64
65 type mockDecision int
66
67 const (
68 decisionNoQueuingExecute mockDecision = iota
69 decisionQueuingExecute
70 decisionCancelWait
71 decisionReject
72 decisionSkipFilter
73 )
74
75 var defaultRequestWorkEstimator = func(req *http.Request, fsName, plName string) fcrequest.WorkEstimate {
76 return fcrequest.WorkEstimate{InitialSeats: 1}
77 }
78
79 type fakeApfFilter struct {
80 mockDecision mockDecision
81 postEnqueue func()
82 postDequeue func()
83
84 utilflowcontrol.WatchTracker
85 utilflowcontrol.MaxSeatsTracker
86 }
87
88 func (t fakeApfFilter) Handle(ctx context.Context,
89 requestDigest utilflowcontrol.RequestDigest,
90 noteFn func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string),
91 workEstimator func() fcrequest.WorkEstimate,
92 queueNoteFn fq.QueueNoteFn,
93 execFn func(),
94 ) {
95 if t.mockDecision == decisionSkipFilter {
96 panic("Handle should not be invoked")
97 }
98 noteFn(bootstrap.SuggestedFlowSchemaGlobalDefault, bootstrap.SuggestedPriorityLevelConfigurationGlobalDefault, requestDigest.User.GetName())
99 switch t.mockDecision {
100 case decisionNoQueuingExecute:
101 execFn()
102 case decisionQueuingExecute:
103 queueNoteFn(true)
104 t.postEnqueue()
105 queueNoteFn(false)
106 t.postDequeue()
107 execFn()
108 case decisionCancelWait:
109 queueNoteFn(true)
110 t.postEnqueue()
111 queueNoteFn(false)
112 t.postDequeue()
113 case decisionReject:
114 return
115 }
116 }
117
118 func (t fakeApfFilter) Run(stopCh <-chan struct{}) error {
119 return nil
120 }
121
122 func (t fakeApfFilter) Install(c *mux.PathRecorderMux) {
123 }
124
125 func newApfServerWithSingleRequest(t *testing.T, decision mockDecision) *httptest.Server {
126 onExecuteFunc := func() {
127 if decision == decisionCancelWait {
128 t.Errorf("execute should not be invoked")
129 }
130 // atomicReadOnlyExecuting can be either 0 or 1 as we test one request at a time.
131 currentValue := atomicReadOnlyExecuting.Load()
132 if decision != decisionSkipFilter && currentValue != 1 {
133 t.Errorf("Wanted %d requests executing, got %d", 1, currentValue)
134 }
135 }
136
137 postExecuteFunc := func() {}
138 // atomicReadOnlyWaiting can be either 0 or 1 as we test one request at a time.
139 postEnqueueFunc := func() {
140 currentValue := atomicReadOnlyWaiting.Load()
141 if currentValue != 1 {
142 t.Errorf("Wanted %d requests in queue, got %d", 1, currentValue)
143 }
144 }
145
146 postDequeueFunc := func() {
147 currentValue := atomicReadOnlyWaiting.Load()
148 if currentValue != 0 {
149 t.Errorf("Wanted %d requests in queue, got %d", 0, currentValue)
150 }
151 }
152 return newApfServerWithHooks(t, decision, onExecuteFunc, postExecuteFunc, postEnqueueFunc, postDequeueFunc)
153 }
154
155 func newApfServerWithHooks(t *testing.T, decision mockDecision, onExecute, postExecute, postEnqueue, postDequeue func()) *httptest.Server {
156 fakeFilter := fakeApfFilter{
157 mockDecision: decision,
158 postEnqueue: postEnqueue,
159 postDequeue: postDequeue,
160 WatchTracker: utilflowcontrol.NewWatchTracker(),
161 MaxSeatsTracker: utilflowcontrol.NewMaxSeatsTracker(),
162 }
163 return newApfServerWithFilter(t, fakeFilter, time.Minute/4, onExecute, postExecute)
164 }
165
166 func newApfServerWithFilter(t *testing.T, flowControlFilter utilflowcontrol.Interface, defaultWaitLimit time.Duration, onExecute, postExecute func()) *httptest.Server {
167 epmetrics.Register()
168 fcmetrics.Register()
169 apfServer := httptest.NewServer(newApfHandlerWithFilter(t, flowControlFilter, defaultWaitLimit, onExecute, postExecute))
170 return apfServer
171 }
172
173 func newApfHandlerWithFilter(t *testing.T, flowControlFilter utilflowcontrol.Interface, defaultWaitLimit time.Duration, onExecute, postExecute func()) http.Handler {
174 requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")}
175 longRunningRequestCheck := BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString("proxy"))
176
177 apfHandler := WithPriorityAndFairness(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
178 onExecute()
179 }), longRunningRequestCheck, flowControlFilter, defaultRequestWorkEstimator, defaultWaitLimit)
180
181 handler := apifilters.WithRequestInfo(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
182 r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{
183 Groups: []string{user.AllUnauthenticated},
184 }))
185 func() {
186 // the APF handler completes its run, either normally or
187 // with a panic, in either case, all APF book keeping must
188 // be completed by now. Also, whether the request is
189 // executed or rejected, we expect the counter to be zero.
190 // TODO: all test(s) using this filter must run
191 // serially to each other
192 defer func() {
193 currentValue := atomicReadOnlyExecuting.Load()
194 if currentValue != 0 {
195 t.Errorf("Wanted %d requests executing, got %d", 0, currentValue)
196 }
197 }()
198 apfHandler.ServeHTTP(w, r)
199 postExecute()
200 }()
201 }), requestInfoFactory)
202
203 return handler
204 }
205
206 func TestApfSkipLongRunningRequest(t *testing.T) {
207 server := newApfServerWithSingleRequest(t, decisionSkipFilter)
208 defer server.Close()
209
210 ctx, cancel := context.WithCancel(context.Background())
211 defer cancel()
212 StartPriorityAndFairnessWatermarkMaintenance(ctx.Done())
213
214 // send a watch request to test skipping long running request
215 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/foos/foo/proxy", server.URL), http.StatusOK); err != nil {
216 // request should not be rejected
217 t.Error(err)
218 }
219 }
220
221 func TestApfRejectRequest(t *testing.T) {
222 server := newApfServerWithSingleRequest(t, decisionReject)
223 defer server.Close()
224
225 ctx, cancel := context.WithCancel(context.Background())
226 defer cancel()
227 StartPriorityAndFairnessWatermarkMaintenance(ctx.Done())
228
229 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusTooManyRequests); err != nil {
230 t.Error(err)
231 }
232
233 checkForExpectedMetrics(t, []string{
234 "apiserver_request_terminations_total",
235 "apiserver_request_total",
236 })
237 }
238
239 func TestApfExemptRequest(t *testing.T) {
240 server := newApfServerWithSingleRequest(t, decisionNoQueuingExecute)
241 defer server.Close()
242
243 ctx, cancel := context.WithCancel(context.Background())
244 defer cancel()
245 StartPriorityAndFairnessWatermarkMaintenance(ctx.Done())
246
247 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil {
248 t.Error(err)
249 }
250
251 checkForExpectedMetrics(t, []string{
252 "apiserver_current_inflight_requests",
253 "apiserver_flowcontrol_read_vs_write_current_requests",
254 })
255 }
256
257 func TestApfExecuteRequest(t *testing.T) {
258 server := newApfServerWithSingleRequest(t, decisionQueuingExecute)
259 defer server.Close()
260
261 ctx, cancel := context.WithCancel(context.Background())
262 defer cancel()
263 StartPriorityAndFairnessWatermarkMaintenance(ctx.Done())
264
265 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil {
266 t.Error(err)
267 }
268
269 checkForExpectedMetrics(t, []string{
270 "apiserver_current_inflight_requests",
271 "apiserver_current_inqueue_requests",
272 "apiserver_flowcontrol_read_vs_write_current_requests",
273 })
274 }
275
276 func TestApfExecuteMultipleRequests(t *testing.T) {
277 concurrentRequests := 5
278 preStartExecute, postStartExecute := &sync.WaitGroup{}, &sync.WaitGroup{}
279 preEnqueue, postEnqueue := &sync.WaitGroup{}, &sync.WaitGroup{}
280 preDequeue, postDequeue := &sync.WaitGroup{}, &sync.WaitGroup{}
281 finishExecute := &sync.WaitGroup{}
282 for _, wg := range []*sync.WaitGroup{preStartExecute, postStartExecute, preEnqueue, postEnqueue, preDequeue, postDequeue, finishExecute} {
283 wg.Add(concurrentRequests)
284 }
285
286 onExecuteFunc := func() {
287 preStartExecute.Done()
288 preStartExecute.Wait()
289 currentValue := atomicReadOnlyExecuting.Load()
290 if int(currentValue) != concurrentRequests {
291 t.Errorf("Wanted %d requests executing, got %d", concurrentRequests, currentValue)
292 }
293 postStartExecute.Done()
294 postStartExecute.Wait()
295 }
296
297 postEnqueueFunc := func() {
298 preEnqueue.Done()
299 preEnqueue.Wait()
300 currentValue := atomicReadOnlyWaiting.Load()
301 if int(currentValue) != concurrentRequests {
302 t.Errorf("Wanted %d requests in queue, got %d", concurrentRequests, currentValue)
303 }
304 postEnqueue.Done()
305 postEnqueue.Wait()
306 }
307
308 postDequeueFunc := func() {
309 preDequeue.Done()
310 preDequeue.Wait()
311 currentValue := atomicReadOnlyWaiting.Load()
312 if currentValue != 0 {
313 t.Errorf("Wanted %d requests in queue, got %d", 0, currentValue)
314 }
315 postDequeue.Done()
316 postDequeue.Wait()
317 }
318
319 postExecuteFunc := func() {
320 finishExecute.Done()
321 finishExecute.Wait()
322 }
323
324 server := newApfServerWithHooks(t, decisionQueuingExecute, onExecuteFunc, postExecuteFunc, postEnqueueFunc, postDequeueFunc)
325 defer server.Close()
326
327 ctx, cancel := context.WithCancel(context.Background())
328 defer cancel()
329 StartPriorityAndFairnessWatermarkMaintenance(ctx.Done())
330
331 var wg sync.WaitGroup
332 wg.Add(concurrentRequests)
333 for i := 0; i < concurrentRequests; i++ {
334 go func() {
335 defer wg.Done()
336 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil {
337 t.Error(err)
338 }
339 }()
340 }
341 wg.Wait()
342
343 checkForExpectedMetrics(t, []string{
344 "apiserver_current_inflight_requests",
345 "apiserver_current_inqueue_requests",
346 "apiserver_flowcontrol_read_vs_write_current_requests",
347 })
348 }
349
350 func TestApfCancelWaitRequest(t *testing.T) {
351 server := newApfServerWithSingleRequest(t, decisionCancelWait)
352 defer server.Close()
353
354 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusTooManyRequests); err != nil {
355 t.Error(err)
356 }
357
358 checkForExpectedMetrics(t, []string{
359 "apiserver_current_inflight_requests",
360 "apiserver_request_terminations_total",
361 "apiserver_request_total",
362 })
363 }
364
365 type fakeWatchApfFilter struct {
366 lock sync.Mutex
367 inflight int
368 capacity int
369
370 postExecutePanic bool
371 preExecutePanic bool
372
373 utilflowcontrol.WatchTracker
374 utilflowcontrol.MaxSeatsTracker
375 }
376
377 func newFakeWatchApfFilter(capacity int) *fakeWatchApfFilter {
378 return &fakeWatchApfFilter{
379 capacity: capacity,
380 WatchTracker: utilflowcontrol.NewWatchTracker(),
381 MaxSeatsTracker: utilflowcontrol.NewMaxSeatsTracker(),
382 }
383 }
384
385 func (f *fakeWatchApfFilter) Handle(ctx context.Context,
386 requestDigest utilflowcontrol.RequestDigest,
387 noteFn func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string),
388 _ func() fcrequest.WorkEstimate,
389 _ fq.QueueNoteFn,
390 execFn func(),
391 ) {
392 noteFn(bootstrap.SuggestedFlowSchemaGlobalDefault, bootstrap.SuggestedPriorityLevelConfigurationGlobalDefault, requestDigest.User.GetName())
393 canExecute := false
394 func() {
395 f.lock.Lock()
396 defer f.lock.Unlock()
397 if f.inflight < f.capacity {
398 f.inflight++
399 canExecute = true
400 }
401 }()
402 if !canExecute {
403 return
404 }
405
406 if f.preExecutePanic {
407 panic("pre-exec-panic")
408 }
409 execFn()
410 if f.postExecutePanic {
411 panic("post-exec-panic")
412 }
413
414 f.lock.Lock()
415 defer f.lock.Unlock()
416 f.inflight--
417 }
418
419 func (f *fakeWatchApfFilter) Run(stopCh <-chan struct{}) error {
420 return nil
421 }
422
423 func (t *fakeWatchApfFilter) Install(c *mux.PathRecorderMux) {
424 }
425
426 func (f *fakeWatchApfFilter) wait() error {
427 return wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
428 f.lock.Lock()
429 defer f.lock.Unlock()
430 return f.inflight == 0, nil
431 })
432 }
433
434 func TestApfExecuteWatchRequestsWithInitializationSignal(t *testing.T) {
435 signalsLock := sync.Mutex{}
436 signals := []utilflowcontrol.InitializationSignal{}
437 sendSignals := func() {
438 signalsLock.Lock()
439 defer signalsLock.Unlock()
440 for i := range signals {
441 signals[i].Signal()
442 }
443 signals = signals[:0]
444 }
445
446 newInitializationSignal = func() utilflowcontrol.InitializationSignal {
447 signalsLock.Lock()
448 defer signalsLock.Unlock()
449 signal := utilflowcontrol.NewInitializationSignal()
450 signals = append(signals, signal)
451 return signal
452 }
453 defer func() {
454 newInitializationSignal = utilflowcontrol.NewInitializationSignal
455 }()
456
457 // We test if initialization after receiving initialization signal the
458 // new requests will be allowed to run by:
459 // - sending N requests that will occupy the whole capacity
460 // - sending initialiation signals for them
461 // - ensuring that number of inflight requests will get to zero
462 concurrentRequests := 5
463 firstRunning := sync.WaitGroup{}
464 firstRunning.Add(concurrentRequests)
465 allRunning := sync.WaitGroup{}
466 allRunning.Add(2 * concurrentRequests)
467
468 fakeFilter := newFakeWatchApfFilter(concurrentRequests)
469
470 onExecuteFunc := func() {
471 firstRunning.Done()
472
473 fakeFilter.wait()
474
475 allRunning.Done()
476 allRunning.Wait()
477 }
478
479 postExecuteFunc := func() {}
480
481 server := newApfServerWithFilter(t, fakeFilter, time.Minute/4, onExecuteFunc, postExecuteFunc)
482 defer server.Close()
483
484 var wg sync.WaitGroup
485 wg.Add(2 * concurrentRequests)
486 for i := 0; i < concurrentRequests; i++ {
487 go func() {
488 defer wg.Done()
489 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil {
490 t.Error(err)
491 }
492 }()
493 }
494
495 firstRunning.Wait()
496 sendSignals()
497 fakeFilter.wait()
498 firstRunning.Add(concurrentRequests)
499
500 for i := 0; i < concurrentRequests; i++ {
501 go func() {
502 defer wg.Done()
503 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil {
504 t.Error(err)
505 }
506 }()
507 }
508 firstRunning.Wait()
509 sendSignals()
510 wg.Wait()
511 }
512
513 func TestApfRejectWatchRequestsWithInitializationSignal(t *testing.T) {
514 fakeFilter := newFakeWatchApfFilter(0)
515
516 onExecuteFunc := func() {
517 t.Errorf("Request unexepectedly executing")
518 }
519 postExecuteFunc := func() {}
520
521 server := newApfServerWithFilter(t, fakeFilter, time.Minute/4, onExecuteFunc, postExecuteFunc)
522 defer server.Close()
523
524 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusTooManyRequests); err != nil {
525 t.Error(err)
526 }
527 }
528
529 func TestApfWatchPanic(t *testing.T) {
530 epmetrics.Register()
531 fcmetrics.Register()
532
533 fakeFilter := newFakeWatchApfFilter(1)
534
535 onExecuteFunc := func() {
536 panic("test panic")
537 }
538 postExecuteFunc := func() {}
539
540 apfHandler := newApfHandlerWithFilter(t, fakeFilter, time.Minute/4, onExecuteFunc, postExecuteFunc)
541 handler := func(w http.ResponseWriter, r *http.Request) {
542 defer func() {
543 if err := recover(); err == nil {
544 t.Errorf("expected panic, got %v", err)
545 }
546 }()
547 apfHandler.ServeHTTP(w, r)
548 }
549 server := httptest.NewServer(http.HandlerFunc(handler))
550 defer server.Close()
551
552 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil {
553 t.Errorf("unexpected error: %v", err)
554 }
555 }
556
557 func TestApfWatchHandlePanic(t *testing.T) {
558 epmetrics.Register()
559 fcmetrics.Register()
560 preExecutePanicingFilter := newFakeWatchApfFilter(1)
561 preExecutePanicingFilter.preExecutePanic = true
562
563 postExecutePanicingFilter := newFakeWatchApfFilter(1)
564 postExecutePanicingFilter.postExecutePanic = true
565
566 testCases := []struct {
567 name string
568 filter *fakeWatchApfFilter
569 }{
570 {
571 name: "pre-execute panic",
572 filter: preExecutePanicingFilter,
573 },
574 {
575 name: "post-execute panic",
576 filter: postExecutePanicingFilter,
577 },
578 }
579
580 onExecuteFunc := func() {
581 time.Sleep(5 * time.Second)
582 }
583 postExecuteFunc := func() {}
584
585 for _, test := range testCases {
586 t.Run(test.name, func(t *testing.T) {
587 apfHandler := newApfHandlerWithFilter(t, test.filter, time.Minute/4, onExecuteFunc, postExecuteFunc)
588 handler := func(w http.ResponseWriter, r *http.Request) {
589 defer func() {
590 if err := recover(); err == nil {
591 t.Errorf("expected panic, got %v", err)
592 }
593 }()
594 apfHandler.ServeHTTP(w, r)
595 }
596 server := httptest.NewServer(http.HandlerFunc(handler))
597 defer server.Close()
598
599 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil {
600 t.Errorf("unexpected error: %v", err)
601 }
602 })
603 }
604 }
605
606 // TestContextClosesOnRequestProcessed ensures that the request context is cancelled
607 // automatically even if the server doesn't cancel is explicitly.
608 // This is required to ensure we won't be leaking goroutines that wait for context
609 // cancelling (e.g. in queueset::StartRequest method).
610 // Even though in production we are not using httptest.Server, this logic is shared
611 // across these two.
612 func TestContextClosesOnRequestProcessed(t *testing.T) {
613 epmetrics.Register()
614 fcmetrics.Register()
615 wg := sync.WaitGroup{}
616 wg.Add(1)
617 handler := func(w http.ResponseWriter, r *http.Request) {
618 ctx := r.Context()
619 // asynchronously wait for context being closed
620 go func() {
621 <-ctx.Done()
622 wg.Done()
623 }()
624 }
625 server := httptest.NewServer(http.HandlerFunc(handler))
626 defer server.Close()
627
628 if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil {
629 t.Errorf("unexpected error: %v", err)
630 }
631
632 wg.Wait()
633 }
634
635 type fakeFilterRequestDigest struct {
636 *fakeApfFilter
637 requestDigestGot *utilflowcontrol.RequestDigest
638 workEstimateGot fcrequest.WorkEstimate
639 }
640
641 func (f *fakeFilterRequestDigest) Handle(ctx context.Context,
642 requestDigest utilflowcontrol.RequestDigest,
643 noteFn func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string),
644 workEstimator func() fcrequest.WorkEstimate,
645 _ fq.QueueNoteFn, _ func(),
646 ) {
647 f.requestDigestGot = &requestDigest
648 noteFn(bootstrap.MandatoryFlowSchemaCatchAll, bootstrap.MandatoryPriorityLevelConfigurationCatchAll, "")
649 f.workEstimateGot = workEstimator()
650 }
651
652 func TestApfWithRequestDigest(t *testing.T) {
653 epmetrics.Register()
654 fcmetrics.Register()
655 longRunningFunc := func(_ *http.Request, _ *apirequest.RequestInfo) bool { return false }
656 fakeFilter := &fakeFilterRequestDigest{}
657
658 reqDigestExpected := &utilflowcontrol.RequestDigest{
659 RequestInfo: &apirequest.RequestInfo{Verb: "get"},
660 User: &user.DefaultInfo{Name: "foo"},
661 }
662 workExpected := fcrequest.WorkEstimate{
663 InitialSeats: 5,
664 FinalSeats: 7,
665 AdditionalLatency: 3 * time.Second,
666 }
667
668 handler := WithPriorityAndFairness(http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) {}),
669 longRunningFunc,
670 fakeFilter,
671 func(_ *http.Request, _, _ string) fcrequest.WorkEstimate { return workExpected },
672 time.Minute/4,
673 )
674
675 w := httptest.NewRecorder()
676 req, err := http.NewRequest(http.MethodGet, "/bar", nil)
677 if err != nil {
678 t.Fatalf("Failed to create new http request - %v", err)
679 }
680 req = req.WithContext(apirequest.WithRequestInfo(req.Context(), reqDigestExpected.RequestInfo))
681 req = req.WithContext(apirequest.WithUser(req.Context(), reqDigestExpected.User))
682
683 handler.ServeHTTP(w, req)
684
685 if !reflect.DeepEqual(reqDigestExpected, fakeFilter.requestDigestGot) {
686 t.Errorf("Expected RequestDigest to match, diff: %s", cmp.Diff(reqDigestExpected, fakeFilter.requestDigestGot))
687 }
688 if !reflect.DeepEqual(workExpected, fakeFilter.workEstimateGot) {
689 t.Errorf("Expected WorkEstimate to match, diff: %s", cmp.Diff(workExpected, fakeFilter.workEstimateGot))
690 }
691 }
692
693 func TestPriorityAndFairnessWithPanicRecoveryAndTimeoutFilter(t *testing.T) {
694 epmetrics.Register()
695 fcmetrics.Register()
696 timeFmt := "15:04:05.999"
697
698 t.Run("priority level concurrency is set to 1, request handler panics, next request should not be rejected", func(t *testing.T) {
699 const (
700 userName = "alice"
701 fsName = "test-fs"
702 plName = "test-pl"
703 serverConcurrency, plConcurrencyShares, plConcurrency = 1, 1, 1
704 )
705
706 apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, 0)
707 stopCh := make(chan struct{})
708 controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency)
709
710 headerMatcher := headerMatcher{}
711 // we will raise a panic for the first request.
712 firstRequestPathPanic, secondRequestPathShouldWork := "/request/panic-as-designed", "/request/should-succeed-as-expected"
713 firstHandlerDoneCh, secondHandlerDoneCh := make(chan struct{}), make(chan struct{})
714 requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
715 headerMatcher.inspect(t, w, fsName, plName)
716 switch {
717 case r.URL.Path == firstRequestPathPanic:
718 close(firstHandlerDoneCh)
719 panic(fmt.Errorf("request handler panic'd as designed - %#v", r.RequestURI))
720 case r.URL.Path == secondRequestPathShouldWork:
721 close(secondHandlerDoneCh)
722
723 }
724 })
725
726 // NOTE: the server will enforce a 1m timeout on every incoming
727 // request, and the client enforces a timeout of 2m.
728 handler := newHandlerChain(t, requestHandler, controller, userName, time.Minute)
729 server, requestGetter := newHTTP2ServerWithClient(handler, 2*time.Minute)
730 defer server.Close()
731
732 // we send two requests synchronously, one at a time
733 // - first request is expected to panic as designed
734 // - second request is expected to succeed
735 _, err := requestGetter(firstRequestPathPanic)
736
737 // did the server handler panic, as expected?
738 select {
739 case <-firstHandlerDoneCh:
740 case <-time.After(wait.ForeverTestTimeout):
741 t.Errorf("Expected the server handler to panic for request: %q", firstRequestPathPanic)
742 }
743 if isClientTimeout(err) {
744 t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", firstRequestPathPanic, err.Error())
745 }
746 expectResetStreamError(t, err)
747
748 // the second request should be served successfully.
749 response, err := requestGetter(secondRequestPathShouldWork)
750 if err != nil {
751 t.Fatalf("Expected request: %q to get a response, but got error: %#v", secondRequestPathShouldWork, err)
752 }
753 if response.StatusCode != http.StatusOK {
754 t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#v", http.StatusOK, secondRequestPathShouldWork, response)
755 }
756 select {
757 case <-secondHandlerDoneCh:
758 case <-time.After(wait.ForeverTestTimeout):
759 t.Errorf("Expected the server handler to have completed: %q", secondRequestPathShouldWork)
760 }
761
762 close(stopCh)
763 t.Log("Waiting for the controller to shutdown")
764
765 controllerErr := <-controllerCompletedCh
766 if controllerErr != nil {
767 t.Errorf("Expected no error from the controller, but got: %#v", controllerErr)
768 }
769 })
770
771 t.Run("priority level concurrency is set to 1, request times out and inner handler hasn't written to the response yet", func(t *testing.T) {
772 t.Parallel()
773 const (
774 userName = "alice"
775 fsName = "test-fs"
776 plName = "test-pl"
777 serverConcurrency, plConcurrencyShares, plConcurrency = 1, 1, 1
778 )
779
780 apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, 0)
781 stopCh := make(chan struct{})
782 controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency)
783
784 headerMatcher := headerMatcher{}
785 rquestTimesOutPath := "/request/time-out-as-designed"
786 reqHandlerCompletedCh, callerRoundTripDoneCh := make(chan struct{}), make(chan struct{})
787 requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
788 headerMatcher.inspect(t, w, fsName, plName)
789
790 if r.URL.Path == rquestTimesOutPath {
791 defer close(reqHandlerCompletedCh)
792
793 // this will force the request to time out.
794 <-callerRoundTripDoneCh
795 }
796 })
797
798 // NOTE: the server will enforce a 5s timeout on every
799 // incoming request, and the client enforces a timeout of 1m.
800 handler := newHandlerChain(t, requestHandler, controller, userName, 5*time.Second)
801 server, requestGetter := newHTTP2ServerWithClient(handler, time.Minute)
802 defer server.Close()
803
804 // send a request synchronously with a client timeout of 1m, this minimizes the
805 // chance of a flake in ci, the cient waits long enough for the server to send a
806 // timeout response to the client.
807 var (
808 response *http.Response
809 err error
810 )
811 func() {
812 defer close(callerRoundTripDoneCh)
813
814 t.Logf("Waiting for the request: %q to time out", rquestTimesOutPath)
815 response, err = requestGetter(rquestTimesOutPath)
816 if isClientTimeout(err) {
817 t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", rquestTimesOutPath, err.Error())
818 }
819 }()
820
821 t.Logf("Waiting for the inner handler of the request: %q to complete", rquestTimesOutPath)
822 select {
823 case <-reqHandlerCompletedCh:
824 case <-time.After(wait.ForeverTestTimeout):
825 t.Errorf("Expected the server handler to have completed: %q", rquestTimesOutPath)
826 }
827
828 if err != nil {
829 t.Fatalf("Expected request: %q to get a response, but got error: %#v", rquestTimesOutPath, err)
830 }
831 if response.StatusCode != http.StatusGatewayTimeout {
832 t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#v", http.StatusGatewayTimeout, rquestTimesOutPath, response)
833 }
834
835 close(stopCh)
836 t.Log("Waiting for the controller to shutdown")
837
838 controllerErr := <-controllerCompletedCh
839 if controllerErr != nil {
840 t.Errorf("Expected no error from the controller, but got: %#v", controllerErr)
841 }
842 })
843
844 t.Run("priority level concurrency is set to 1, inner handler panics after the request times out", func(t *testing.T) {
845 t.Parallel()
846 const (
847 userName = "alice"
848 fsName = "test-fs"
849 plName = "test-pl"
850 serverConcurrency, plConcurrencyShares, plConcurrency = 1, 1, 1
851 )
852
853 apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, 0)
854 stopCh := make(chan struct{})
855 controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency)
856
857 headerMatcher := headerMatcher{}
858 reqHandlerErrCh, callerRoundTripDoneCh := make(chan error, 1), make(chan struct{})
859 rquestTimesOutPath := "/request/time-out-as-designed"
860 requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
861 headerMatcher.inspect(t, w, fsName, plName)
862
863 if r.URL.Path == rquestTimesOutPath {
864 <-callerRoundTripDoneCh
865
866 // we expect the timeout handler to have timed out this request by now and any attempt
867 // to write to the response should return a http.ErrHandlerTimeout error.
868 _, innerHandlerWriteErr := w.Write([]byte("foo"))
869 reqHandlerErrCh <- innerHandlerWriteErr
870
871 panic(http.ErrAbortHandler)
872 }
873 })
874
875 // NOTE: the server will enforce a 5s timeout on every
876 // incoming request, and the client enforces a timeout of 1m.
877 handler := newHandlerChain(t, requestHandler, controller, userName, 5*time.Second)
878 server, requestGetter := newHTTP2ServerWithClient(handler, time.Minute)
879 defer server.Close()
880
881 // send a request synchronously with a client timeout of 1m, this minimizes the
882 // chance of a flake in ci, the cient waits long enough for the server to send a
883 // timeout response to the client.
884 var (
885 response *http.Response
886 err error
887 )
888 func() {
889 defer close(callerRoundTripDoneCh)
890 t.Logf("Waiting for the request: %q to time out", rquestTimesOutPath)
891 response, err = requestGetter(rquestTimesOutPath)
892 if isClientTimeout(err) {
893 t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", rquestTimesOutPath, err.Error())
894 }
895 }()
896
897 t.Logf("Waiting for the inner handler of the request: %q to complete", rquestTimesOutPath)
898 select {
899 case innerHandlerWriteErr := <-reqHandlerErrCh:
900 if innerHandlerWriteErr != http.ErrHandlerTimeout {
901 t.Fatalf("Expected error: %#v, but got: %#v", http.ErrHandlerTimeout, err)
902 }
903 case <-time.After(wait.ForeverTestTimeout):
904 t.Errorf("Expected the server handler to have completed: %q", rquestTimesOutPath)
905 }
906
907 if err != nil {
908 t.Fatalf("Expected request: %q to get a response, but got error: %#v", rquestTimesOutPath, err)
909 }
910 if response.StatusCode != http.StatusGatewayTimeout {
911 t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#v", http.StatusGatewayTimeout, rquestTimesOutPath, response)
912 }
913
914 close(stopCh)
915 t.Log("Waiting for the controller to shutdown")
916
917 controllerErr := <-controllerCompletedCh
918 if controllerErr != nil {
919 t.Errorf("Expected no error from the controller, but got: %#v", controllerErr)
920 }
921 })
922
923 t.Run("priority level concurrency is set to 1, inner handler writes to the response before request times out", func(t *testing.T) {
924 t.Parallel()
925 const (
926 userName = "alice"
927 fsName = "test-fs"
928 plName = "test-pl"
929 serverConcurrency, plConcurrencyShares, plConcurrency = 1, 1, 1
930 )
931
932 apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, 0)
933 stopCh := make(chan struct{})
934 controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency)
935
936 headerMatcher := headerMatcher{}
937 rquestTimesOutPath := "/request/time-out-as-designed"
938 reqHandlerErrCh, callerRoundTripDoneCh := make(chan error, 1), make(chan struct{})
939 requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
940 headerMatcher.inspect(t, w, fsName, plName)
941
942 if r.URL.Path == rquestTimesOutPath {
943
944 // inner handler writes header and then let the request time out.
945 w.WriteHeader(http.StatusBadRequest)
946 <-callerRoundTripDoneCh
947
948 // we expect the timeout handler to have timed out this request by now and any attempt
949 // to write to the response should return a http.ErrHandlerTimeout error.
950 _, innerHandlerWriteErr := w.Write([]byte("foo"))
951 reqHandlerErrCh <- innerHandlerWriteErr
952 }
953 })
954
955 // NOTE: the server will enforce a 5s timeout on every
956 // incoming request, and the client enforces a timeout of 1m.
957 handler := newHandlerChain(t, requestHandler, controller, userName, 5*time.Second)
958 server, requestGetter := newHTTP2ServerWithClient(handler, time.Minute)
959 defer server.Close()
960
961 // send a request synchronously with a client timeout of 1m, this minimizes the
962 // chance of a flake in ci, the cient waits long enough for the server to send a
963 // timeout response to the client.
964 var err error
965 func() {
966 defer close(callerRoundTripDoneCh)
967 t.Logf("Waiting for the request: %q to time out", rquestTimesOutPath)
968 _, err = requestGetter(rquestTimesOutPath)
969 if isClientTimeout(err) {
970 t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", rquestTimesOutPath, err.Error())
971 }
972 }()
973
974 t.Logf("Waiting for the inner handler of the request: %q to complete", rquestTimesOutPath)
975 select {
976 case innerHandlerWriteErr := <-reqHandlerErrCh:
977 if innerHandlerWriteErr != http.ErrHandlerTimeout {
978 t.Fatalf("Expected error: %#v, but got: %#v", http.ErrHandlerTimeout, err)
979 }
980 case <-time.After(wait.ForeverTestTimeout):
981 t.Errorf("Expected the server handler to have completed: %q", rquestTimesOutPath)
982 }
983
984 expectResetStreamError(t, err)
985
986 close(stopCh)
987 t.Log("Waiting for the controller to shutdown")
988
989 controllerErr := <-controllerCompletedCh
990 if controllerErr != nil {
991 t.Errorf("Expected no error from the controller, but got: %#v", controllerErr)
992 }
993 })
994
995 t.Run("priority level concurrency is set to 1, queue length is 1, first request should time out and second (enqueued) request should time out as well", func(t *testing.T) {
996 t.Parallel()
997
998 type result struct {
999 err error
1000 response *http.Response
1001 }
1002
1003 const (
1004 userName = "alice"
1005 fsName = "test-fs"
1006 plName = "test-pl"
1007 serverConcurrency, plConcurrencyShares, plConcurrency, queueLength = 1, 1, 1, 1
1008 )
1009
1010 apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, queueLength)
1011 stopCh := make(chan struct{})
1012 controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency)
1013
1014 headerMatcher := headerMatcher{}
1015 firstRequestTimesOutPath, secondRequestEnqueuedPath := "/request/first/time-out-as-designed", "/request/second/enqueued-as-designed"
1016 firstReqHandlerErrCh, firstReqInProgressCh := make(chan error, 1), make(chan struct{})
1017 firstReqRoundTripDoneCh, secondReqRoundTripDoneCh := make(chan struct{}), make(chan struct{})
1018 requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1019 headerMatcher.inspect(t, w, fsName, plName)
1020 switch {
1021 case r.URL.Path == firstRequestTimesOutPath:
1022 close(firstReqInProgressCh)
1023 <-firstReqRoundTripDoneCh
1024
1025 // make sure we wait until the caller of the second request returns, this is to
1026 // ensure that second request never has a chance to be executed (to avoid flakes)
1027 <-secondReqRoundTripDoneCh
1028
1029 // we expect the timeout handler to have timed out this request by now and any attempt
1030 // to write to the response should return a http.ErrHandlerTimeout error.
1031 _, firstRequestInnerHandlerWriteErr := w.Write([]byte("foo"))
1032 firstReqHandlerErrCh <- firstRequestInnerHandlerWriteErr
1033
1034 case r.URL.Path == secondRequestEnqueuedPath:
1035 // we expect the concurrency to be set to 1 and so this request should never be executed.
1036 t.Errorf("Expected second request to be enqueued: %q", secondRequestEnqueuedPath)
1037 }
1038 })
1039
1040 // NOTE: the server will enforce a 5s timeout on every
1041 // incoming request, and the client enforces a timeout of 1m.
1042 handler := newHandlerChain(t, requestHandler, controller, userName, 5*time.Second)
1043 server, requestGetter := newHTTP2ServerWithClient(handler, time.Minute)
1044 defer server.Close()
1045
1046 // This test involves two requests sent to the same priority level, which has 1 queue and
1047 // a concurrency limit of 1. The handler chain include the timeout filter.
1048 // Each request is sent from a separate goroutine, with a client-side timeout of 1m, on
1049 // the other hand, the server enforces a timeout of 5s (via the timeout filter).
1050 // The first request should get dispatched immediately; execution (a) starts with closing
1051 // the channel that triggers the second client goroutine to send its request and then (b)
1052 // waits for both client goroutines to have gotten a response (expected to be timeouts).
1053 // The second request sits in the queue until the timeout filter does its thing, which
1054 // it does concurrently to both requests. For the first request this should make the client
1055 // get a timeout response without directly affecting execution. For the second request, the
1056 // fact that the timeout filter closes the request's Context.Done() causes the request to be
1057 // promptly ejected from its queue. The goroutine doing the APF handling writes an HTTP
1058 // response message with status 429.
1059 // The timeout handler invokes its inner handler in one goroutine while reacting to the
1060 // passage of time in its original goroutine. That reaction to a time out consists of either
1061 // (a) writing an HTTP response message with status 504 to indicate the timeout or (b) doing an
1062 // HTTP/2 stream close; the latter is done if either the connection has been "hijacked" or some
1063 // other goroutine (e.g., the one running the inner handler) has started to write a response.
1064 // In the scenario tested here, there is thus a race between two goroutines to respond to
1065 // the second request and any of their responses is allowed by the test.
1066 firstReqResultCh, secondReqResultCh := make(chan result, 1), make(chan result, 1)
1067 go func() {
1068 defer close(firstReqRoundTripDoneCh)
1069 t.Logf("At %s, Sending request: %q", time.Now().Format(timeFmt), firstRequestTimesOutPath)
1070 resp, err := requestGetter(firstRequestTimesOutPath)
1071 t.Logf("At %s, RoundTrip of request: %q has completed", time.Now().Format(timeFmt), firstRequestTimesOutPath)
1072 firstReqResultCh <- result{err: err, response: resp}
1073 }()
1074 go func() {
1075 // we must wait for the "first" request to start executing before
1076 // we can initiate the "second".
1077 defer close(secondReqRoundTripDoneCh)
1078
1079 <-firstReqInProgressCh
1080 t.Logf("At %s, Sending request: %q", time.Now().Format(timeFmt), secondRequestEnqueuedPath)
1081 resp, err := requestGetter(secondRequestEnqueuedPath)
1082 t.Logf("At %s, RoundTrip of request: %q has completed", time.Now().Format(timeFmt), secondRequestEnqueuedPath)
1083 secondReqResultCh <- result{err: err, response: resp}
1084 }()
1085
1086 firstReqResult := <-firstReqResultCh
1087 if isClientTimeout(firstReqResult.err) {
1088 t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", firstRequestTimesOutPath, fmtError(firstReqResult.err))
1089 }
1090 t.Logf("Waiting for the inner handler of the request: %q to complete", firstRequestTimesOutPath)
1091 select {
1092 case firstRequestInnerHandlerWriteErr := <-firstReqHandlerErrCh:
1093 if firstRequestInnerHandlerWriteErr != http.ErrHandlerTimeout {
1094 t.Fatalf("Expected error: %#v, but got: %s", http.ErrHandlerTimeout, fmtError(firstRequestInnerHandlerWriteErr))
1095 }
1096 case <-time.After(wait.ForeverTestTimeout):
1097 t.Errorf("Expected the server handler to have completed: %q", firstRequestTimesOutPath)
1098 }
1099
1100 // first request is expected to time out.
1101 if isStreamReset(firstReqResult.err) || firstReqResult.response.StatusCode != http.StatusGatewayTimeout {
1102 // got what was expected
1103 } else if firstReqResult.err != nil {
1104 t.Fatalf("Expected request: %q to get a response or stream reset, but got error: %s", firstRequestTimesOutPath, fmtError(firstReqResult.err))
1105 } else if firstReqResult.response.StatusCode != http.StatusGatewayTimeout {
1106 t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#+v", http.StatusGatewayTimeout, firstRequestTimesOutPath, firstReqResult.response)
1107 }
1108
1109 // second request is expected to either be rejected (ideal behavior) or time out (current approximation of the ideal behavior)
1110 secondReqResult := <-secondReqResultCh
1111 if isClientTimeout(secondReqResult.err) {
1112 t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", secondRequestEnqueuedPath, fmtError(secondReqResult.err))
1113 }
1114 if isStreamReset(secondReqResult.err) || secondReqResult.response.StatusCode == http.StatusTooManyRequests || secondReqResult.response.StatusCode == http.StatusGatewayTimeout {
1115 // got what was expected
1116 } else if secondReqResult.err != nil {
1117 t.Fatalf("Expected request: %q to get a response or stream reset, but got error: %s", secondRequestEnqueuedPath, fmtError(secondReqResult.err))
1118 } else if !(secondReqResult.response.StatusCode == http.StatusTooManyRequests || secondReqResult.response.StatusCode == http.StatusGatewayTimeout) {
1119 t.Errorf("Expected HTTP status code: %d or %d for request: %q, but got: %#+v", http.StatusTooManyRequests, http.StatusGatewayTimeout, secondRequestEnqueuedPath, secondReqResult.response)
1120 }
1121
1122 close(stopCh)
1123 t.Log("Waiting for the controller to shutdown")
1124
1125 controllerErr := <-controllerCompletedCh
1126 if controllerErr != nil {
1127 t.Errorf("Expected no error from the controller, but got: %s", fmtError(controllerErr))
1128 }
1129 })
1130 }
1131
1132 func fmtError(err error) string {
1133 return fmt.Sprintf("%#+v=%q", err, err.Error())
1134 }
1135
1136 func startAPFController(t *testing.T, stopCh <-chan struct{}, apfConfiguration []runtime.Object, serverConcurrency int,
1137 plName string, plConcurrency int) (utilflowcontrol.Interface, <-chan error) {
1138 clientset := newClientset(t, apfConfiguration...)
1139 // this test does not rely on resync, so resync period is set to zero
1140 factory := informers.NewSharedInformerFactory(clientset, 0)
1141 controller := utilflowcontrol.New(factory, clientset.FlowcontrolV1(), serverConcurrency)
1142
1143 factory.Start(stopCh)
1144
1145 // wait for the informer cache to sync.
1146 timeout, cancel := context.WithTimeout(context.TODO(), 5*time.Second)
1147 defer cancel()
1148 cacheSyncDone := factory.WaitForCacheSync(timeout.Done())
1149 if names := unsyncedInformers(cacheSyncDone); len(names) > 0 {
1150 t.Fatalf("WaitForCacheSync did not successfully complete, resources=%#v", names)
1151 }
1152
1153 controllerCompletedCh := make(chan error)
1154 var controllerErr error
1155 go func() {
1156 controllerErr = controller.Run(stopCh)
1157 controllerCompletedCh <- controllerErr
1158 }()
1159
1160 // make sure that apf controller syncs the priority level configuration object we are using in this test.
1161 // read the metrics and ensure the concurrency limit for our priority level is set to the expected value.
1162 pollErr := wait.PollImmediate(100*time.Millisecond, 5*time.Second, func() (done bool, err error) {
1163 if err := gaugeValueMatch("apiserver_flowcontrol_nominal_limit_seats", map[string]string{"priority_level": plName}, plConcurrency); err != nil {
1164 t.Logf("polling retry - error: %s", err)
1165 return false, nil
1166 }
1167 return true, nil
1168 })
1169 if pollErr != nil {
1170 t.Fatalf("expected the apf controller to sync the priotity level configuration object: %s", plName)
1171 }
1172
1173 return controller, controllerCompletedCh
1174 }
1175
1176 // returns a started http2 server, with a client function to send request to the server.
1177 func newHTTP2ServerWithClient(handler http.Handler, clientTimeout time.Duration) (*httptest.Server, func(path string) (*http.Response, error)) {
1178 server := httptest.NewUnstartedServer(handler)
1179 server.EnableHTTP2 = true
1180 server.StartTLS()
1181 cli := server.Client()
1182 cli.Timeout = clientTimeout
1183 return server, func(path string) (*http.Response, error) {
1184 return cli.Get(server.URL + path)
1185 }
1186 }
1187
1188 type headerMatcher struct{}
1189
1190 // verifies that the expected flow schema and priority level UIDs are attached to the header.
1191 func (m *headerMatcher) inspect(t *testing.T, w http.ResponseWriter, expectedFS, expectedPL string) {
1192 t.Helper()
1193 err := func() error {
1194 if w == nil {
1195 return fmt.Errorf("expected a non nil HTTP response")
1196 }
1197
1198 key := flowcontrol.ResponseHeaderMatchedFlowSchemaUID
1199 if value := w.Header().Get(key); expectedFS != value {
1200 return fmt.Errorf("expected HTTP header %s to have value %q, but got: %q", key, expectedFS, value)
1201 }
1202
1203 key = flowcontrol.ResponseHeaderMatchedPriorityLevelConfigurationUID
1204 if value := w.Header().Get(key); expectedPL != value {
1205 return fmt.Errorf("expected HTTP header %s to have value %q, but got %q", key, expectedPL, value)
1206 }
1207 return nil
1208 }()
1209 if err == nil {
1210 return
1211 }
1212 t.Errorf("Expected APF headers to match, but got: %v", err)
1213 }
1214
1215 // when a request panics, http2 resets the stream with an INTERNAL_ERROR message
1216 func expectResetStreamError(t *testing.T, err error) {
1217 if err == nil {
1218 t.Fatalf("expected the server to send an error, but got nil")
1219 }
1220 if isStreamReset(err) {
1221 return
1222 }
1223 t.Fatalf("expected a stream reset error, but got %#+v=%s", err, err.Error())
1224 }
1225
1226 func newClientset(t *testing.T, objects ...runtime.Object) clientset.Interface {
1227 clientset := fake.NewSimpleClientset(objects...)
1228 if clientset == nil {
1229 t.Fatal("unable to create fake client set")
1230 }
1231 return clientset
1232 }
1233
1234 // builds a chain of handlers that include the panic recovery and timeout filter, so we can simulate the behavior of
1235 // a real apiserver.
1236 // the specified user is added as the authenticated user to the request context.
1237 func newHandlerChain(t *testing.T, handler http.Handler, filter utilflowcontrol.Interface, userName string, requestTimeout time.Duration) http.Handler {
1238 requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")}
1239 longRunningRequestCheck := BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString("proxy"))
1240
1241 apfHandler := WithPriorityAndFairness(handler, longRunningRequestCheck, filter, defaultRequestWorkEstimator, time.Minute/4)
1242
1243 // add the handler in the chain that adds the specified user to the request context
1244 handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1245 r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{
1246 Name: userName,
1247 Groups: []string{user.AllAuthenticated},
1248 }))
1249
1250 apfHandler.ServeHTTP(w, r)
1251 })
1252
1253 handler = WithTimeoutForNonLongRunningRequests(handler, longRunningRequestCheck)
1254 // we don't have any request with invalid timeout, so leaving audit policy and sink nil.
1255 handler = apifilters.WithRequestDeadline(handler, nil, nil, longRunningRequestCheck, nil, requestTimeout)
1256 handler = apifilters.WithRequestInfo(handler, requestInfoFactory)
1257 handler = WithPanicRecovery(handler, requestInfoFactory)
1258 handler = apifilters.WithAuditInit(handler)
1259 return handler
1260 }
1261
1262 func unsyncedInformers(status map[reflect.Type]bool) []string {
1263 names := make([]string, 0)
1264
1265 for objType, synced := range status {
1266 if !synced {
1267 names = append(names, objType.Name())
1268 }
1269 }
1270
1271 return names
1272 }
1273
1274 func newConfiguration(fsName, plName, user string, concurrency int32, queueLength int32) []runtime.Object {
1275 fs := &flowcontrol.FlowSchema{
1276 ObjectMeta: metav1.ObjectMeta{
1277 Name: fsName,
1278 UID: types.UID(fsName),
1279 },
1280 Spec: flowcontrol.FlowSchemaSpec{
1281 MatchingPrecedence: 1,
1282 PriorityLevelConfiguration: flowcontrol.PriorityLevelConfigurationReference{
1283 Name: plName,
1284 },
1285 DistinguisherMethod: &flowcontrol.FlowDistinguisherMethod{
1286 Type: flowcontrol.FlowDistinguisherMethodByUserType,
1287 },
1288 Rules: []flowcontrol.PolicyRulesWithSubjects{
1289 {
1290 Subjects: []flowcontrol.Subject{
1291 {
1292 Kind: flowcontrol.SubjectKindUser,
1293 User: &flowcontrol.UserSubject{
1294 Name: user,
1295 },
1296 },
1297 },
1298 NonResourceRules: []flowcontrol.NonResourcePolicyRule{
1299 {
1300 Verbs: []string{flowcontrol.VerbAll},
1301 NonResourceURLs: []string{flowcontrol.NonResourceAll},
1302 },
1303 },
1304 },
1305 },
1306 },
1307 }
1308
1309 var (
1310 responseType flowcontrol.LimitResponseType = flowcontrol.LimitResponseTypeReject
1311 qcfg *flowcontrol.QueuingConfiguration
1312 )
1313 if queueLength > 0 {
1314 responseType = flowcontrol.LimitResponseTypeQueue
1315 qcfg = &flowcontrol.QueuingConfiguration{
1316 Queues: 1,
1317 QueueLengthLimit: queueLength,
1318 HandSize: 1,
1319 }
1320 }
1321 pl := &flowcontrol.PriorityLevelConfiguration{
1322 ObjectMeta: metav1.ObjectMeta{
1323 Name: plName,
1324 UID: types.UID(plName),
1325 },
1326 Spec: flowcontrol.PriorityLevelConfigurationSpec{
1327 Type: flowcontrol.PriorityLevelEnablementLimited,
1328 Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
1329 NominalConcurrencyShares: ptr.To(concurrency),
1330 LimitResponse: flowcontrol.LimitResponse{
1331 Type: responseType,
1332 Queuing: qcfg,
1333 },
1334 },
1335 },
1336 }
1337
1338 return []runtime.Object{fs, pl}
1339 }
1340
1341 // gathers and checks the metrics.
1342 func checkForExpectedMetrics(t *testing.T, expectedMetrics []string) {
1343 metricsFamily, err := legacyregistry.DefaultGatherer.Gather()
1344 if err != nil {
1345 t.Fatalf("Failed to gather metrics %v", err)
1346 }
1347
1348 metrics := map[string]interface{}{}
1349 for _, mf := range metricsFamily {
1350 metrics[*mf.Name] = mf
1351 }
1352
1353 for _, metricName := range expectedMetrics {
1354 if _, ok := metrics[metricName]; !ok {
1355 if !ok {
1356 t.Errorf("Scraped metrics did not include expected metric %s", metricName)
1357 }
1358 }
1359 }
1360 }
1361
1362 // gaugeValueMatch ensures that the value of gauge metrics matching the labelFilter is as expected.
1363 func gaugeValueMatch(name string, labelFilter map[string]string, wantValue int) error {
1364 metrics, err := legacyregistry.DefaultGatherer.Gather()
1365 if err != nil {
1366 return fmt.Errorf("failed to gather metrics: %s", err)
1367 }
1368
1369 sum := 0
1370 familyMatch, labelMatch := false, false
1371 for _, mf := range metrics {
1372 if mf.GetName() != name {
1373 continue
1374 }
1375
1376 familyMatch = true
1377 for _, metric := range mf.GetMetric() {
1378 if !testutil.LabelsMatch(metric, labelFilter) {
1379 continue
1380 }
1381
1382 labelMatch = true
1383 sum += int(metric.GetGauge().GetValue())
1384 }
1385 }
1386 if !familyMatch {
1387 return fmt.Errorf("expected to find the metric family: %s in the gathered result", name)
1388 }
1389 if !labelMatch {
1390 return fmt.Errorf("expected to find metrics with matching labels: %#+v", labelFilter)
1391 }
1392 if wantValue != sum {
1393 return fmt.Errorf("expected the sum to be: %d, but got: %d for gauge metric: %s with labels %#+v", wantValue, sum, name, labelFilter)
1394 }
1395
1396 return nil
1397 }
1398
1399 func isClientTimeout(err error) bool {
1400 if urlErr, ok := err.(*url.Error); ok {
1401 return urlErr.Timeout()
1402 }
1403 return false
1404 }
1405
1406 func isStreamReset(err error) bool {
1407 if err == nil {
1408 return false
1409 }
1410 if urlErr, ok := err.(*url.Error); ok {
1411 // Sadly, the client does not receive a more specific indication
1412 // of stream reset.
1413 return strings.Contains(urlErr.Err.Error(), "INTERNAL_ERROR")
1414 }
1415 return false
1416 }
1417
1418 func TestGetRequestWaitContext(t *testing.T) {
1419 tests := []struct {
1420 name string
1421 defaultRequestWaitLimit time.Duration
1422 parent func(t time.Time) (context.Context, context.CancelFunc)
1423 newReqWaitCtxExpected bool
1424 reqWaitLimitExpected time.Duration
1425 }{
1426 {
1427 name: "context deadline has exceeded",
1428 parent: func(time.Time) (context.Context, context.CancelFunc) {
1429 ctx, cancel := context.WithCancel(context.Background())
1430 cancel()
1431 return ctx, cancel
1432 },
1433 },
1434 {
1435 name: "context has a deadline, 'received at' is not set, wait limit should be one fourth of the remaining deadline from now",
1436 parent: func(now time.Time) (context.Context, context.CancelFunc) {
1437 return context.WithDeadline(context.Background(), now.Add(60*time.Second))
1438 },
1439 newReqWaitCtxExpected: true,
1440 reqWaitLimitExpected: 15 * time.Second,
1441 },
1442 {
1443 name: "context has a deadline, 'received at' is set, wait limit should be one fourth of the deadline starting from the 'received at' time",
1444 parent: func(now time.Time) (context.Context, context.CancelFunc) {
1445 ctx := apirequest.WithReceivedTimestamp(context.Background(), now.Add(-10*time.Second))
1446 return context.WithDeadline(ctx, now.Add(50*time.Second))
1447 },
1448 newReqWaitCtxExpected: true,
1449 reqWaitLimitExpected: 5 * time.Second, // from now
1450 },
1451 {
1452 name: "context does not have any deadline, 'received at' is not set, default wait limit should be in effect from now",
1453 defaultRequestWaitLimit: 15 * time.Second,
1454 parent: func(time.Time) (context.Context, context.CancelFunc) {
1455 return context.WithCancel(context.Background())
1456 },
1457 newReqWaitCtxExpected: true,
1458 reqWaitLimitExpected: 15 * time.Second,
1459 },
1460 {
1461 name: "context does not have any deadline, 'received at' is set, default wait limit should be in effect starting from the 'received at' time",
1462 defaultRequestWaitLimit: 15 * time.Second,
1463 parent: func(now time.Time) (context.Context, context.CancelFunc) {
1464 ctx := apirequest.WithReceivedTimestamp(context.Background(), now.Add(-10*time.Second))
1465 return context.WithCancel(ctx)
1466 },
1467 newReqWaitCtxExpected: true,
1468 reqWaitLimitExpected: 5 * time.Second, // from now
1469 },
1470 {
1471 name: "context has a deadline, wait limit should not exceed the hard limit of 1m",
1472 parent: func(now time.Time) (context.Context, context.CancelFunc) {
1473 // let 1/4th of the remaining deadline exceed the hard limit
1474 return context.WithDeadline(context.Background(), now.Add(8*time.Minute))
1475 },
1476 newReqWaitCtxExpected: true,
1477 reqWaitLimitExpected: time.Minute,
1478 },
1479 {
1480 name: "context has no deadline, wait limit should not exceed the hard limit of 1m",
1481 defaultRequestWaitLimit: 2 * time.Minute, // it exceeds the hard limit
1482 parent: func(now time.Time) (context.Context, context.CancelFunc) {
1483 return context.WithCancel(context.Background())
1484 },
1485 newReqWaitCtxExpected: true,
1486 reqWaitLimitExpected: time.Minute,
1487 },
1488 }
1489
1490 for _, test := range tests {
1491 t.Run(test.name, func(t *testing.T) {
1492 now := time.Now()
1493 parent, cancel := test.parent(now)
1494 defer cancel()
1495
1496 clock := clocktesting.NewFakePassiveClock(now)
1497 newReqWaitCtxGot, cancelGot := getRequestWaitContext(parent, test.defaultRequestWaitLimit, clock)
1498 if cancelGot == nil {
1499 t.Errorf("Expected a non nil context.CancelFunc")
1500 return
1501 }
1502 defer cancelGot()
1503
1504 switch {
1505 case test.newReqWaitCtxExpected:
1506 deadlineGot, ok := newReqWaitCtxGot.Deadline()
1507 if !ok {
1508 t.Errorf("Expected the new wait limit context to have a deadline")
1509 }
1510 if waitLimitGot := deadlineGot.Sub(now); test.reqWaitLimitExpected != waitLimitGot {
1511 t.Errorf("Expected request wait limit %s, but got: %s", test.reqWaitLimitExpected, waitLimitGot)
1512 }
1513 default:
1514 if parent != newReqWaitCtxGot {
1515 t.Errorf("Expected the parent context to be returned: want: %#v, got %#v", parent, newReqWaitCtxGot)
1516 }
1517 }
1518 })
1519 }
1520 }