| 1 |
/* |
| 2 |
Copyright 2016 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package filters |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"net/http" |
| 23 |
"net/http/httptest" |
| 24 |
"net/url" |
| 25 |
"os" |
| 26 |
"reflect" |
| 27 |
"strings" |
| 28 |
"sync" |
| 29 |
"testing" |
| 30 |
"time" |
| 31 |
|
| 32 |
flowcontrol "k8s.io/api/flowcontrol/v1" |
| 33 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 34 |
"k8s.io/apimachinery/pkg/runtime" |
| 35 |
"k8s.io/apimachinery/pkg/types" |
| 36 |
"k8s.io/apimachinery/pkg/util/sets" |
| 37 |
"k8s.io/apimachinery/pkg/util/wait" |
| 38 |
"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap" |
| 39 |
"k8s.io/apiserver/pkg/authentication/user" |
| 40 |
apifilters "k8s.io/apiserver/pkg/endpoints/filters" |
| 41 |
epmetrics "k8s.io/apiserver/pkg/endpoints/metrics" |
| 42 |
apirequest "k8s.io/apiserver/pkg/endpoints/request" |
| 43 |
"k8s.io/apiserver/pkg/server/mux" |
| 44 |
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" |
| 45 |
fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing" |
| 46 |
fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics" |
| 47 |
fcrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" |
| 48 |
"k8s.io/client-go/informers" |
| 49 |
clientset "k8s.io/client-go/kubernetes" |
| 50 |
"k8s.io/client-go/kubernetes/fake" |
| 51 |
"k8s.io/component-base/metrics/legacyregistry" |
| 52 |
"k8s.io/component-base/metrics/testutil" |
| 53 |
"k8s.io/klog/v2" |
| 54 |
clocktesting "k8s.io/utils/clock/testing" |
| 55 |
"k8s.io/utils/ptr" |
| 56 |
|
| 57 |
"github.com/google/go-cmp/cmp" |
| 58 |
) |
| 59 |
|
| 60 |
func TestMain(m *testing.M) { |
| 61 |
klog.InitFlags(nil) |
| 62 |
os.Exit(m.Run()) |
| 63 |
} |
| 64 |
|
| 65 |
type mockDecision int |
| 66 |
|
| 67 |
const ( |
| 68 |
decisionNoQueuingExecute mockDecision = iota |
| 69 |
decisionQueuingExecute |
| 70 |
decisionCancelWait |
| 71 |
decisionReject |
| 72 |
decisionSkipFilter |
| 73 |
) |
| 74 |
|
| 75 |
var defaultRequestWorkEstimator = func(req *http.Request, fsName, plName string) fcrequest.WorkEstimate { |
| 76 |
return fcrequest.WorkEstimate{InitialSeats: 1} |
| 77 |
} |
| 78 |
|
| 79 |
type fakeApfFilter struct { |
| 80 |
mockDecision mockDecision |
| 81 |
postEnqueue func() |
| 82 |
postDequeue func() |
| 83 |
|
| 84 |
utilflowcontrol.WatchTracker |
| 85 |
utilflowcontrol.MaxSeatsTracker |
| 86 |
} |
| 87 |
|
| 88 |
func (t fakeApfFilter) Handle(ctx context.Context, |
| 89 |
requestDigest utilflowcontrol.RequestDigest, |
| 90 |
noteFn func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string), |
| 91 |
workEstimator func() fcrequest.WorkEstimate, |
| 92 |
queueNoteFn fq.QueueNoteFn, |
| 93 |
execFn func(), |
| 94 |
) { |
| 95 |
if t.mockDecision == decisionSkipFilter { |
| 96 |
panic("Handle should not be invoked") |
| 97 |
} |
| 98 |
noteFn(bootstrap.SuggestedFlowSchemaGlobalDefault, bootstrap.SuggestedPriorityLevelConfigurationGlobalDefault, requestDigest.User.GetName()) |
| 99 |
switch t.mockDecision { |
| 100 |
case decisionNoQueuingExecute: |
| 101 |
execFn() |
| 102 |
case decisionQueuingExecute: |
| 103 |
queueNoteFn(true) |
| 104 |
t.postEnqueue() |
| 105 |
queueNoteFn(false) |
| 106 |
t.postDequeue() |
| 107 |
execFn() |
| 108 |
case decisionCancelWait: |
| 109 |
queueNoteFn(true) |
| 110 |
t.postEnqueue() |
| 111 |
queueNoteFn(false) |
| 112 |
t.postDequeue() |
| 113 |
case decisionReject: |
| 114 |
return |
| 115 |
} |
| 116 |
} |
| 117 |
|
| 118 |
func (t fakeApfFilter) Run(stopCh <-chan struct{}) error { |
| 119 |
return nil |
| 120 |
} |
| 121 |
|
| 122 |
func (t fakeApfFilter) Install(c *mux.PathRecorderMux) { |
| 123 |
} |
| 124 |
|
| 125 |
func newApfServerWithSingleRequest(t *testing.T, decision mockDecision) *httptest.Server { |
| 126 |
onExecuteFunc := func() { |
| 127 |
if decision == decisionCancelWait { |
| 128 |
t.Errorf("execute should not be invoked") |
| 129 |
} |
| 130 |
// atomicReadOnlyExecuting can be either 0 or 1 as we test one request at a time. |
| 131 |
currentValue := atomicReadOnlyExecuting.Load() |
| 132 |
if decision != decisionSkipFilter && currentValue != 1 { |
| 133 |
t.Errorf("Wanted %d requests executing, got %d", 1, currentValue) |
| 134 |
} |
| 135 |
} |
| 136 |
|
| 137 |
postExecuteFunc := func() {} |
| 138 |
// atomicReadOnlyWaiting can be either 0 or 1 as we test one request at a time. |
| 139 |
postEnqueueFunc := func() { |
| 140 |
currentValue := atomicReadOnlyWaiting.Load() |
| 141 |
if currentValue != 1 { |
| 142 |
t.Errorf("Wanted %d requests in queue, got %d", 1, currentValue) |
| 143 |
} |
| 144 |
} |
| 145 |
|
| 146 |
postDequeueFunc := func() { |
| 147 |
currentValue := atomicReadOnlyWaiting.Load() |
| 148 |
if currentValue != 0 { |
| 149 |
t.Errorf("Wanted %d requests in queue, got %d", 0, currentValue) |
| 150 |
} |
| 151 |
} |
| 152 |
return newApfServerWithHooks(t, decision, onExecuteFunc, postExecuteFunc, postEnqueueFunc, postDequeueFunc) |
| 153 |
} |
| 154 |
|
| 155 |
func newApfServerWithHooks(t *testing.T, decision mockDecision, onExecute, postExecute, postEnqueue, postDequeue func()) *httptest.Server { |
| 156 |
fakeFilter := fakeApfFilter{ |
| 157 |
mockDecision: decision, |
| 158 |
postEnqueue: postEnqueue, |
| 159 |
postDequeue: postDequeue, |
| 160 |
WatchTracker: utilflowcontrol.NewWatchTracker(), |
| 161 |
MaxSeatsTracker: utilflowcontrol.NewMaxSeatsTracker(), |
| 162 |
} |
| 163 |
return newApfServerWithFilter(t, fakeFilter, time.Minute/4, onExecute, postExecute) |
| 164 |
} |
| 165 |
|
| 166 |
func newApfServerWithFilter(t *testing.T, flowControlFilter utilflowcontrol.Interface, defaultWaitLimit time.Duration, onExecute, postExecute func()) *httptest.Server { |
| 167 |
epmetrics.Register() |
| 168 |
fcmetrics.Register() |
| 169 |
apfServer := httptest.NewServer(newApfHandlerWithFilter(t, flowControlFilter, defaultWaitLimit, onExecute, postExecute)) |
| 170 |
return apfServer |
| 171 |
} |
| 172 |
|
| 173 |
func newApfHandlerWithFilter(t *testing.T, flowControlFilter utilflowcontrol.Interface, defaultWaitLimit time.Duration, onExecute, postExecute func()) http.Handler { |
| 174 |
requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")} |
| 175 |
longRunningRequestCheck := BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString("proxy")) |
| 176 |
|
| 177 |
apfHandler := WithPriorityAndFairness(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 178 |
onExecute() |
| 179 |
}), longRunningRequestCheck, flowControlFilter, defaultRequestWorkEstimator, defaultWaitLimit) |
| 180 |
|
| 181 |
handler := apifilters.WithRequestInfo(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 182 |
r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{ |
| 183 |
Groups: []string{user.AllUnauthenticated}, |
| 184 |
})) |
| 185 |
func() { |
| 186 |
// the APF handler completes its run, either normally or |
| 187 |
// with a panic, in either case, all APF book keeping must |
| 188 |
// be completed by now. Also, whether the request is |
| 189 |
// executed or rejected, we expect the counter to be zero. |
| 190 |
// TODO: all test(s) using this filter must run |
| 191 |
// serially to each other |
| 192 |
defer func() { |
| 193 |
currentValue := atomicReadOnlyExecuting.Load() |
| 194 |
if currentValue != 0 { |
| 195 |
t.Errorf("Wanted %d requests executing, got %d", 0, currentValue) |
| 196 |
} |
| 197 |
}() |
| 198 |
apfHandler.ServeHTTP(w, r) |
| 199 |
postExecute() |
| 200 |
}() |
| 201 |
}), requestInfoFactory) |
| 202 |
|
| 203 |
return handler |
| 204 |
} |
| 205 |
|
| 206 |
func TestApfSkipLongRunningRequest(t *testing.T) { |
| 207 |
server := newApfServerWithSingleRequest(t, decisionSkipFilter) |
| 208 |
defer server.Close() |
| 209 |
|
| 210 |
ctx, cancel := context.WithCancel(context.Background()) |
| 211 |
defer cancel() |
| 212 |
StartPriorityAndFairnessWatermarkMaintenance(ctx.Done()) |
| 213 |
|
| 214 |
// send a watch request to test skipping long running request |
| 215 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/foos/foo/proxy", server.URL), http.StatusOK); err != nil { |
| 216 |
// request should not be rejected |
| 217 |
t.Error(err) |
| 218 |
} |
| 219 |
} |
| 220 |
|
| 221 |
func TestApfRejectRequest(t *testing.T) { |
| 222 |
server := newApfServerWithSingleRequest(t, decisionReject) |
| 223 |
defer server.Close() |
| 224 |
|
| 225 |
ctx, cancel := context.WithCancel(context.Background()) |
| 226 |
defer cancel() |
| 227 |
StartPriorityAndFairnessWatermarkMaintenance(ctx.Done()) |
| 228 |
|
| 229 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusTooManyRequests); err != nil { |
| 230 |
t.Error(err) |
| 231 |
} |
| 232 |
|
| 233 |
checkForExpectedMetrics(t, []string{ |
| 234 |
"apiserver_request_terminations_total", |
| 235 |
"apiserver_request_total", |
| 236 |
}) |
| 237 |
} |
| 238 |
|
| 239 |
func TestApfExemptRequest(t *testing.T) { |
| 240 |
server := newApfServerWithSingleRequest(t, decisionNoQueuingExecute) |
| 241 |
defer server.Close() |
| 242 |
|
| 243 |
ctx, cancel := context.WithCancel(context.Background()) |
| 244 |
defer cancel() |
| 245 |
StartPriorityAndFairnessWatermarkMaintenance(ctx.Done()) |
| 246 |
|
| 247 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil { |
| 248 |
t.Error(err) |
| 249 |
} |
| 250 |
|
| 251 |
checkForExpectedMetrics(t, []string{ |
| 252 |
"apiserver_current_inflight_requests", |
| 253 |
"apiserver_flowcontrol_read_vs_write_current_requests", |
| 254 |
}) |
| 255 |
} |
| 256 |
|
| 257 |
func TestApfExecuteRequest(t *testing.T) { |
| 258 |
server := newApfServerWithSingleRequest(t, decisionQueuingExecute) |
| 259 |
defer server.Close() |
| 260 |
|
| 261 |
ctx, cancel := context.WithCancel(context.Background()) |
| 262 |
defer cancel() |
| 263 |
StartPriorityAndFairnessWatermarkMaintenance(ctx.Done()) |
| 264 |
|
| 265 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil { |
| 266 |
t.Error(err) |
| 267 |
} |
| 268 |
|
| 269 |
checkForExpectedMetrics(t, []string{ |
| 270 |
"apiserver_current_inflight_requests", |
| 271 |
"apiserver_current_inqueue_requests", |
| 272 |
"apiserver_flowcontrol_read_vs_write_current_requests", |
| 273 |
}) |
| 274 |
} |
| 275 |
|
| 276 |
func TestApfExecuteMultipleRequests(t *testing.T) { |
| 277 |
concurrentRequests := 5 |
| 278 |
preStartExecute, postStartExecute := &sync.WaitGroup{}, &sync.WaitGroup{} |
| 279 |
preEnqueue, postEnqueue := &sync.WaitGroup{}, &sync.WaitGroup{} |
| 280 |
preDequeue, postDequeue := &sync.WaitGroup{}, &sync.WaitGroup{} |
| 281 |
finishExecute := &sync.WaitGroup{} |
| 282 |
for _, wg := range []*sync.WaitGroup{preStartExecute, postStartExecute, preEnqueue, postEnqueue, preDequeue, postDequeue, finishExecute} { |
| 283 |
wg.Add(concurrentRequests) |
| 284 |
} |
| 285 |
|
| 286 |
onExecuteFunc := func() { |
| 287 |
preStartExecute.Done() |
| 288 |
preStartExecute.Wait() |
| 289 |
currentValue := atomicReadOnlyExecuting.Load() |
| 290 |
if int(currentValue) != concurrentRequests { |
| 291 |
t.Errorf("Wanted %d requests executing, got %d", concurrentRequests, currentValue) |
| 292 |
} |
| 293 |
postStartExecute.Done() |
| 294 |
postStartExecute.Wait() |
| 295 |
} |
| 296 |
|
| 297 |
postEnqueueFunc := func() { |
| 298 |
preEnqueue.Done() |
| 299 |
preEnqueue.Wait() |
| 300 |
currentValue := atomicReadOnlyWaiting.Load() |
| 301 |
if int(currentValue) != concurrentRequests { |
| 302 |
t.Errorf("Wanted %d requests in queue, got %d", concurrentRequests, currentValue) |
| 303 |
} |
| 304 |
postEnqueue.Done() |
| 305 |
postEnqueue.Wait() |
| 306 |
} |
| 307 |
|
| 308 |
postDequeueFunc := func() { |
| 309 |
preDequeue.Done() |
| 310 |
preDequeue.Wait() |
| 311 |
currentValue := atomicReadOnlyWaiting.Load() |
| 312 |
if currentValue != 0 { |
| 313 |
t.Errorf("Wanted %d requests in queue, got %d", 0, currentValue) |
| 314 |
} |
| 315 |
postDequeue.Done() |
| 316 |
postDequeue.Wait() |
| 317 |
} |
| 318 |
|
| 319 |
postExecuteFunc := func() { |
| 320 |
finishExecute.Done() |
| 321 |
finishExecute.Wait() |
| 322 |
} |
| 323 |
|
| 324 |
server := newApfServerWithHooks(t, decisionQueuingExecute, onExecuteFunc, postExecuteFunc, postEnqueueFunc, postDequeueFunc) |
| 325 |
defer server.Close() |
| 326 |
|
| 327 |
ctx, cancel := context.WithCancel(context.Background()) |
| 328 |
defer cancel() |
| 329 |
StartPriorityAndFairnessWatermarkMaintenance(ctx.Done()) |
| 330 |
|
| 331 |
var wg sync.WaitGroup |
| 332 |
wg.Add(concurrentRequests) |
| 333 |
for i := 0; i < concurrentRequests; i++ { |
| 334 |
go func() { |
| 335 |
defer wg.Done() |
| 336 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil { |
| 337 |
t.Error(err) |
| 338 |
} |
| 339 |
}() |
| 340 |
} |
| 341 |
wg.Wait() |
| 342 |
|
| 343 |
checkForExpectedMetrics(t, []string{ |
| 344 |
"apiserver_current_inflight_requests", |
| 345 |
"apiserver_current_inqueue_requests", |
| 346 |
"apiserver_flowcontrol_read_vs_write_current_requests", |
| 347 |
}) |
| 348 |
} |
| 349 |
|
| 350 |
func TestApfCancelWaitRequest(t *testing.T) { |
| 351 |
server := newApfServerWithSingleRequest(t, decisionCancelWait) |
| 352 |
defer server.Close() |
| 353 |
|
| 354 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusTooManyRequests); err != nil { |
| 355 |
t.Error(err) |
| 356 |
} |
| 357 |
|
| 358 |
checkForExpectedMetrics(t, []string{ |
| 359 |
"apiserver_current_inflight_requests", |
| 360 |
"apiserver_request_terminations_total", |
| 361 |
"apiserver_request_total", |
| 362 |
}) |
| 363 |
} |
| 364 |
|
| 365 |
type fakeWatchApfFilter struct { |
| 366 |
lock sync.Mutex |
| 367 |
inflight int |
| 368 |
capacity int |
| 369 |
|
| 370 |
postExecutePanic bool |
| 371 |
preExecutePanic bool |
| 372 |
|
| 373 |
utilflowcontrol.WatchTracker |
| 374 |
utilflowcontrol.MaxSeatsTracker |
| 375 |
} |
| 376 |
|
| 377 |
func newFakeWatchApfFilter(capacity int) *fakeWatchApfFilter { |
| 378 |
return &fakeWatchApfFilter{ |
| 379 |
capacity: capacity, |
| 380 |
WatchTracker: utilflowcontrol.NewWatchTracker(), |
| 381 |
MaxSeatsTracker: utilflowcontrol.NewMaxSeatsTracker(), |
| 382 |
} |
| 383 |
} |
| 384 |
|
| 385 |
func (f *fakeWatchApfFilter) Handle(ctx context.Context, |
| 386 |
requestDigest utilflowcontrol.RequestDigest, |
| 387 |
noteFn func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string), |
| 388 |
_ func() fcrequest.WorkEstimate, |
| 389 |
_ fq.QueueNoteFn, |
| 390 |
execFn func(), |
| 391 |
) { |
| 392 |
noteFn(bootstrap.SuggestedFlowSchemaGlobalDefault, bootstrap.SuggestedPriorityLevelConfigurationGlobalDefault, requestDigest.User.GetName()) |
| 393 |
canExecute := false |
| 394 |
func() { |
| 395 |
f.lock.Lock() |
| 396 |
defer f.lock.Unlock() |
| 397 |
if f.inflight < f.capacity { |
| 398 |
f.inflight++ |
| 399 |
canExecute = true |
| 400 |
} |
| 401 |
}() |
| 402 |
if !canExecute { |
| 403 |
return |
| 404 |
} |
| 405 |
|
| 406 |
if f.preExecutePanic { |
| 407 |
panic("pre-exec-panic") |
| 408 |
} |
| 409 |
execFn() |
| 410 |
if f.postExecutePanic { |
| 411 |
panic("post-exec-panic") |
| 412 |
} |
| 413 |
|
| 414 |
f.lock.Lock() |
| 415 |
defer f.lock.Unlock() |
| 416 |
f.inflight-- |
| 417 |
} |
| 418 |
|
| 419 |
func (f *fakeWatchApfFilter) Run(stopCh <-chan struct{}) error { |
| 420 |
return nil |
| 421 |
} |
| 422 |
|
| 423 |
func (t *fakeWatchApfFilter) Install(c *mux.PathRecorderMux) { |
| 424 |
} |
| 425 |
|
| 426 |
func (f *fakeWatchApfFilter) wait() error { |
| 427 |
return wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { |
| 428 |
f.lock.Lock() |
| 429 |
defer f.lock.Unlock() |
| 430 |
return f.inflight == 0, nil |
| 431 |
}) |
| 432 |
} |
| 433 |
|
| 434 |
func TestApfExecuteWatchRequestsWithInitializationSignal(t *testing.T) { |
| 435 |
signalsLock := sync.Mutex{} |
| 436 |
signals := []utilflowcontrol.InitializationSignal{} |
| 437 |
sendSignals := func() { |
| 438 |
signalsLock.Lock() |
| 439 |
defer signalsLock.Unlock() |
| 440 |
for i := range signals { |
| 441 |
signals[i].Signal() |
| 442 |
} |
| 443 |
signals = signals[:0] |
| 444 |
} |
| 445 |
|
| 446 |
newInitializationSignal = func() utilflowcontrol.InitializationSignal { |
| 447 |
signalsLock.Lock() |
| 448 |
defer signalsLock.Unlock() |
| 449 |
signal := utilflowcontrol.NewInitializationSignal() |
| 450 |
signals = append(signals, signal) |
| 451 |
return signal |
| 452 |
} |
| 453 |
defer func() { |
| 454 |
newInitializationSignal = utilflowcontrol.NewInitializationSignal |
| 455 |
}() |
| 456 |
|
| 457 |
// We test if initialization after receiving initialization signal the |
| 458 |
// new requests will be allowed to run by: |
| 459 |
// - sending N requests that will occupy the whole capacity |
| 460 |
// - sending initialiation signals for them |
| 461 |
// - ensuring that number of inflight requests will get to zero |
| 462 |
concurrentRequests := 5 |
| 463 |
firstRunning := sync.WaitGroup{} |
| 464 |
firstRunning.Add(concurrentRequests) |
| 465 |
allRunning := sync.WaitGroup{} |
| 466 |
allRunning.Add(2 * concurrentRequests) |
| 467 |
|
| 468 |
fakeFilter := newFakeWatchApfFilter(concurrentRequests) |
| 469 |
|
| 470 |
onExecuteFunc := func() { |
| 471 |
firstRunning.Done() |
| 472 |
|
| 473 |
fakeFilter.wait() |
| 474 |
|
| 475 |
allRunning.Done() |
| 476 |
allRunning.Wait() |
| 477 |
} |
| 478 |
|
| 479 |
postExecuteFunc := func() {} |
| 480 |
|
| 481 |
server := newApfServerWithFilter(t, fakeFilter, time.Minute/4, onExecuteFunc, postExecuteFunc) |
| 482 |
defer server.Close() |
| 483 |
|
| 484 |
var wg sync.WaitGroup |
| 485 |
wg.Add(2 * concurrentRequests) |
| 486 |
for i := 0; i < concurrentRequests; i++ { |
| 487 |
go func() { |
| 488 |
defer wg.Done() |
| 489 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil { |
| 490 |
t.Error(err) |
| 491 |
} |
| 492 |
}() |
| 493 |
} |
| 494 |
|
| 495 |
firstRunning.Wait() |
| 496 |
sendSignals() |
| 497 |
fakeFilter.wait() |
| 498 |
firstRunning.Add(concurrentRequests) |
| 499 |
|
| 500 |
for i := 0; i < concurrentRequests; i++ { |
| 501 |
go func() { |
| 502 |
defer wg.Done() |
| 503 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil { |
| 504 |
t.Error(err) |
| 505 |
} |
| 506 |
}() |
| 507 |
} |
| 508 |
firstRunning.Wait() |
| 509 |
sendSignals() |
| 510 |
wg.Wait() |
| 511 |
} |
| 512 |
|
| 513 |
func TestApfRejectWatchRequestsWithInitializationSignal(t *testing.T) { |
| 514 |
fakeFilter := newFakeWatchApfFilter(0) |
| 515 |
|
| 516 |
onExecuteFunc := func() { |
| 517 |
t.Errorf("Request unexepectedly executing") |
| 518 |
} |
| 519 |
postExecuteFunc := func() {} |
| 520 |
|
| 521 |
server := newApfServerWithFilter(t, fakeFilter, time.Minute/4, onExecuteFunc, postExecuteFunc) |
| 522 |
defer server.Close() |
| 523 |
|
| 524 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusTooManyRequests); err != nil { |
| 525 |
t.Error(err) |
| 526 |
} |
| 527 |
} |
| 528 |
|
| 529 |
func TestApfWatchPanic(t *testing.T) { |
| 530 |
epmetrics.Register() |
| 531 |
fcmetrics.Register() |
| 532 |
|
| 533 |
fakeFilter := newFakeWatchApfFilter(1) |
| 534 |
|
| 535 |
onExecuteFunc := func() { |
| 536 |
panic("test panic") |
| 537 |
} |
| 538 |
postExecuteFunc := func() {} |
| 539 |
|
| 540 |
apfHandler := newApfHandlerWithFilter(t, fakeFilter, time.Minute/4, onExecuteFunc, postExecuteFunc) |
| 541 |
handler := func(w http.ResponseWriter, r *http.Request) { |
| 542 |
defer func() { |
| 543 |
if err := recover(); err == nil { |
| 544 |
t.Errorf("expected panic, got %v", err) |
| 545 |
} |
| 546 |
}() |
| 547 |
apfHandler.ServeHTTP(w, r) |
| 548 |
} |
| 549 |
server := httptest.NewServer(http.HandlerFunc(handler)) |
| 550 |
defer server.Close() |
| 551 |
|
| 552 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil { |
| 553 |
t.Errorf("unexpected error: %v", err) |
| 554 |
} |
| 555 |
} |
| 556 |
|
| 557 |
func TestApfWatchHandlePanic(t *testing.T) { |
| 558 |
epmetrics.Register() |
| 559 |
fcmetrics.Register() |
| 560 |
preExecutePanicingFilter := newFakeWatchApfFilter(1) |
| 561 |
preExecutePanicingFilter.preExecutePanic = true |
| 562 |
|
| 563 |
postExecutePanicingFilter := newFakeWatchApfFilter(1) |
| 564 |
postExecutePanicingFilter.postExecutePanic = true |
| 565 |
|
| 566 |
testCases := []struct { |
| 567 |
name string |
| 568 |
filter *fakeWatchApfFilter |
| 569 |
}{ |
| 570 |
{ |
| 571 |
name: "pre-execute panic", |
| 572 |
filter: preExecutePanicingFilter, |
| 573 |
}, |
| 574 |
{ |
| 575 |
name: "post-execute panic", |
| 576 |
filter: postExecutePanicingFilter, |
| 577 |
}, |
| 578 |
} |
| 579 |
|
| 580 |
onExecuteFunc := func() { |
| 581 |
time.Sleep(5 * time.Second) |
| 582 |
} |
| 583 |
postExecuteFunc := func() {} |
| 584 |
|
| 585 |
for _, test := range testCases { |
| 586 |
t.Run(test.name, func(t *testing.T) { |
| 587 |
apfHandler := newApfHandlerWithFilter(t, test.filter, time.Minute/4, onExecuteFunc, postExecuteFunc) |
| 588 |
handler := func(w http.ResponseWriter, r *http.Request) { |
| 589 |
defer func() { |
| 590 |
if err := recover(); err == nil { |
| 591 |
t.Errorf("expected panic, got %v", err) |
| 592 |
} |
| 593 |
}() |
| 594 |
apfHandler.ServeHTTP(w, r) |
| 595 |
} |
| 596 |
server := httptest.NewServer(http.HandlerFunc(handler)) |
| 597 |
defer server.Close() |
| 598 |
|
| 599 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil { |
| 600 |
t.Errorf("unexpected error: %v", err) |
| 601 |
} |
| 602 |
}) |
| 603 |
} |
| 604 |
} |
| 605 |
|
| 606 |
// TestContextClosesOnRequestProcessed ensures that the request context is cancelled |
| 607 |
// automatically even if the server doesn't cancel is explicitly. |
| 608 |
// This is required to ensure we won't be leaking goroutines that wait for context |
| 609 |
// cancelling (e.g. in queueset::StartRequest method). |
| 610 |
// Even though in production we are not using httptest.Server, this logic is shared |
| 611 |
// across these two. |
| 612 |
func TestContextClosesOnRequestProcessed(t *testing.T) { |
| 613 |
epmetrics.Register() |
| 614 |
fcmetrics.Register() |
| 615 |
wg := sync.WaitGroup{} |
| 616 |
wg.Add(1) |
| 617 |
handler := func(w http.ResponseWriter, r *http.Request) { |
| 618 |
ctx := r.Context() |
| 619 |
// asynchronously wait for context being closed |
| 620 |
go func() { |
| 621 |
<-ctx.Done() |
| 622 |
wg.Done() |
| 623 |
}() |
| 624 |
} |
| 625 |
server := httptest.NewServer(http.HandlerFunc(handler)) |
| 626 |
defer server.Close() |
| 627 |
|
| 628 |
if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default/pods?watch=true", server.URL), http.StatusOK); err != nil { |
| 629 |
t.Errorf("unexpected error: %v", err) |
| 630 |
} |
| 631 |
|
| 632 |
wg.Wait() |
| 633 |
} |
| 634 |
|
| 635 |
type fakeFilterRequestDigest struct { |
| 636 |
*fakeApfFilter |
| 637 |
requestDigestGot *utilflowcontrol.RequestDigest |
| 638 |
workEstimateGot fcrequest.WorkEstimate |
| 639 |
} |
| 640 |
|
| 641 |
func (f *fakeFilterRequestDigest) Handle(ctx context.Context, |
| 642 |
requestDigest utilflowcontrol.RequestDigest, |
| 643 |
noteFn func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string), |
| 644 |
workEstimator func() fcrequest.WorkEstimate, |
| 645 |
_ fq.QueueNoteFn, _ func(), |
| 646 |
) { |
| 647 |
f.requestDigestGot = &requestDigest |
| 648 |
noteFn(bootstrap.MandatoryFlowSchemaCatchAll, bootstrap.MandatoryPriorityLevelConfigurationCatchAll, "") |
| 649 |
f.workEstimateGot = workEstimator() |
| 650 |
} |
| 651 |
|
| 652 |
func TestApfWithRequestDigest(t *testing.T) { |
| 653 |
epmetrics.Register() |
| 654 |
fcmetrics.Register() |
| 655 |
longRunningFunc := func(_ *http.Request, _ *apirequest.RequestInfo) bool { return false } |
| 656 |
fakeFilter := &fakeFilterRequestDigest{} |
| 657 |
|
| 658 |
reqDigestExpected := &utilflowcontrol.RequestDigest{ |
| 659 |
RequestInfo: &apirequest.RequestInfo{Verb: "get"}, |
| 660 |
User: &user.DefaultInfo{Name: "foo"}, |
| 661 |
} |
| 662 |
workExpected := fcrequest.WorkEstimate{ |
| 663 |
InitialSeats: 5, |
| 664 |
FinalSeats: 7, |
| 665 |
AdditionalLatency: 3 * time.Second, |
| 666 |
} |
| 667 |
|
| 668 |
handler := WithPriorityAndFairness(http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) {}), |
| 669 |
longRunningFunc, |
| 670 |
fakeFilter, |
| 671 |
func(_ *http.Request, _, _ string) fcrequest.WorkEstimate { return workExpected }, |
| 672 |
time.Minute/4, |
| 673 |
) |
| 674 |
|
| 675 |
w := httptest.NewRecorder() |
| 676 |
req, err := http.NewRequest(http.MethodGet, "/bar", nil) |
| 677 |
if err != nil { |
| 678 |
t.Fatalf("Failed to create new http request - %v", err) |
| 679 |
} |
| 680 |
req = req.WithContext(apirequest.WithRequestInfo(req.Context(), reqDigestExpected.RequestInfo)) |
| 681 |
req = req.WithContext(apirequest.WithUser(req.Context(), reqDigestExpected.User)) |
| 682 |
|
| 683 |
handler.ServeHTTP(w, req) |
| 684 |
|
| 685 |
if !reflect.DeepEqual(reqDigestExpected, fakeFilter.requestDigestGot) { |
| 686 |
t.Errorf("Expected RequestDigest to match, diff: %s", cmp.Diff(reqDigestExpected, fakeFilter.requestDigestGot)) |
| 687 |
} |
| 688 |
if !reflect.DeepEqual(workExpected, fakeFilter.workEstimateGot) { |
| 689 |
t.Errorf("Expected WorkEstimate to match, diff: %s", cmp.Diff(workExpected, fakeFilter.workEstimateGot)) |
| 690 |
} |
| 691 |
} |
| 692 |
|
| 693 |
func TestPriorityAndFairnessWithPanicRecoveryAndTimeoutFilter(t *testing.T) { |
| 694 |
epmetrics.Register() |
| 695 |
fcmetrics.Register() |
| 696 |
timeFmt := "15:04:05.999" |
| 697 |
|
| 698 |
t.Run("priority level concurrency is set to 1, request handler panics, next request should not be rejected", func(t *testing.T) { |
| 699 |
const ( |
| 700 |
userName = "alice" |
| 701 |
fsName = "test-fs" |
| 702 |
plName = "test-pl" |
| 703 |
serverConcurrency, plConcurrencyShares, plConcurrency = 1, 1, 1 |
| 704 |
) |
| 705 |
|
| 706 |
apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, 0) |
| 707 |
stopCh := make(chan struct{}) |
| 708 |
controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency) |
| 709 |
|
| 710 |
headerMatcher := headerMatcher{} |
| 711 |
// we will raise a panic for the first request. |
| 712 |
firstRequestPathPanic, secondRequestPathShouldWork := "/request/panic-as-designed", "/request/should-succeed-as-expected" |
| 713 |
firstHandlerDoneCh, secondHandlerDoneCh := make(chan struct{}), make(chan struct{}) |
| 714 |
requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 715 |
headerMatcher.inspect(t, w, fsName, plName) |
| 716 |
switch { |
| 717 |
case r.URL.Path == firstRequestPathPanic: |
| 718 |
close(firstHandlerDoneCh) |
| 719 |
panic(fmt.Errorf("request handler panic'd as designed - %#v", r.RequestURI)) |
| 720 |
case r.URL.Path == secondRequestPathShouldWork: |
| 721 |
close(secondHandlerDoneCh) |
| 722 |
|
| 723 |
} |
| 724 |
}) |
| 725 |
|
| 726 |
// NOTE: the server will enforce a 1m timeout on every incoming |
| 727 |
// request, and the client enforces a timeout of 2m. |
| 728 |
handler := newHandlerChain(t, requestHandler, controller, userName, time.Minute) |
| 729 |
server, requestGetter := newHTTP2ServerWithClient(handler, 2*time.Minute) |
| 730 |
defer server.Close() |
| 731 |
|
| 732 |
// we send two requests synchronously, one at a time |
| 733 |
// - first request is expected to panic as designed |
| 734 |
// - second request is expected to succeed |
| 735 |
_, err := requestGetter(firstRequestPathPanic) |
| 736 |
|
| 737 |
// did the server handler panic, as expected? |
| 738 |
select { |
| 739 |
case <-firstHandlerDoneCh: |
| 740 |
case <-time.After(wait.ForeverTestTimeout): |
| 741 |
t.Errorf("Expected the server handler to panic for request: %q", firstRequestPathPanic) |
| 742 |
} |
| 743 |
if isClientTimeout(err) { |
| 744 |
t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", firstRequestPathPanic, err.Error()) |
| 745 |
} |
| 746 |
expectResetStreamError(t, err) |
| 747 |
|
| 748 |
// the second request should be served successfully. |
| 749 |
response, err := requestGetter(secondRequestPathShouldWork) |
| 750 |
if err != nil { |
| 751 |
t.Fatalf("Expected request: %q to get a response, but got error: %#v", secondRequestPathShouldWork, err) |
| 752 |
} |
| 753 |
if response.StatusCode != http.StatusOK { |
| 754 |
t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#v", http.StatusOK, secondRequestPathShouldWork, response) |
| 755 |
} |
| 756 |
select { |
| 757 |
case <-secondHandlerDoneCh: |
| 758 |
case <-time.After(wait.ForeverTestTimeout): |
| 759 |
t.Errorf("Expected the server handler to have completed: %q", secondRequestPathShouldWork) |
| 760 |
} |
| 761 |
|
| 762 |
close(stopCh) |
| 763 |
t.Log("Waiting for the controller to shutdown") |
| 764 |
|
| 765 |
controllerErr := <-controllerCompletedCh |
| 766 |
if controllerErr != nil { |
| 767 |
t.Errorf("Expected no error from the controller, but got: %#v", controllerErr) |
| 768 |
} |
| 769 |
}) |
| 770 |
|
| 771 |
t.Run("priority level concurrency is set to 1, request times out and inner handler hasn't written to the response yet", func(t *testing.T) { |
| 772 |
t.Parallel() |
| 773 |
const ( |
| 774 |
userName = "alice" |
| 775 |
fsName = "test-fs" |
| 776 |
plName = "test-pl" |
| 777 |
serverConcurrency, plConcurrencyShares, plConcurrency = 1, 1, 1 |
| 778 |
) |
| 779 |
|
| 780 |
apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, 0) |
| 781 |
stopCh := make(chan struct{}) |
| 782 |
controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency) |
| 783 |
|
| 784 |
headerMatcher := headerMatcher{} |
| 785 |
rquestTimesOutPath := "/request/time-out-as-designed" |
| 786 |
reqHandlerCompletedCh, callerRoundTripDoneCh := make(chan struct{}), make(chan struct{}) |
| 787 |
requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 788 |
headerMatcher.inspect(t, w, fsName, plName) |
| 789 |
|
| 790 |
if r.URL.Path == rquestTimesOutPath { |
| 791 |
defer close(reqHandlerCompletedCh) |
| 792 |
|
| 793 |
// this will force the request to time out. |
| 794 |
<-callerRoundTripDoneCh |
| 795 |
} |
| 796 |
}) |
| 797 |
|
| 798 |
// NOTE: the server will enforce a 5s timeout on every |
| 799 |
// incoming request, and the client enforces a timeout of 1m. |
| 800 |
handler := newHandlerChain(t, requestHandler, controller, userName, 5*time.Second) |
| 801 |
server, requestGetter := newHTTP2ServerWithClient(handler, time.Minute) |
| 802 |
defer server.Close() |
| 803 |
|
| 804 |
// send a request synchronously with a client timeout of 1m, this minimizes the |
| 805 |
// chance of a flake in ci, the cient waits long enough for the server to send a |
| 806 |
// timeout response to the client. |
| 807 |
var ( |
| 808 |
response *http.Response |
| 809 |
err error |
| 810 |
) |
| 811 |
func() { |
| 812 |
defer close(callerRoundTripDoneCh) |
| 813 |
|
| 814 |
t.Logf("Waiting for the request: %q to time out", rquestTimesOutPath) |
| 815 |
response, err = requestGetter(rquestTimesOutPath) |
| 816 |
if isClientTimeout(err) { |
| 817 |
t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", rquestTimesOutPath, err.Error()) |
| 818 |
} |
| 819 |
}() |
| 820 |
|
| 821 |
t.Logf("Waiting for the inner handler of the request: %q to complete", rquestTimesOutPath) |
| 822 |
select { |
| 823 |
case <-reqHandlerCompletedCh: |
| 824 |
case <-time.After(wait.ForeverTestTimeout): |
| 825 |
t.Errorf("Expected the server handler to have completed: %q", rquestTimesOutPath) |
| 826 |
} |
| 827 |
|
| 828 |
if err != nil { |
| 829 |
t.Fatalf("Expected request: %q to get a response, but got error: %#v", rquestTimesOutPath, err) |
| 830 |
} |
| 831 |
if response.StatusCode != http.StatusGatewayTimeout { |
| 832 |
t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#v", http.StatusGatewayTimeout, rquestTimesOutPath, response) |
| 833 |
} |
| 834 |
|
| 835 |
close(stopCh) |
| 836 |
t.Log("Waiting for the controller to shutdown") |
| 837 |
|
| 838 |
controllerErr := <-controllerCompletedCh |
| 839 |
if controllerErr != nil { |
| 840 |
t.Errorf("Expected no error from the controller, but got: %#v", controllerErr) |
| 841 |
} |
| 842 |
}) |
| 843 |
|
| 844 |
t.Run("priority level concurrency is set to 1, inner handler panics after the request times out", func(t *testing.T) { |
| 845 |
t.Parallel() |
| 846 |
const ( |
| 847 |
userName = "alice" |
| 848 |
fsName = "test-fs" |
| 849 |
plName = "test-pl" |
| 850 |
serverConcurrency, plConcurrencyShares, plConcurrency = 1, 1, 1 |
| 851 |
) |
| 852 |
|
| 853 |
apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, 0) |
| 854 |
stopCh := make(chan struct{}) |
| 855 |
controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency) |
| 856 |
|
| 857 |
headerMatcher := headerMatcher{} |
| 858 |
reqHandlerErrCh, callerRoundTripDoneCh := make(chan error, 1), make(chan struct{}) |
| 859 |
rquestTimesOutPath := "/request/time-out-as-designed" |
| 860 |
requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 861 |
headerMatcher.inspect(t, w, fsName, plName) |
| 862 |
|
| 863 |
if r.URL.Path == rquestTimesOutPath { |
| 864 |
<-callerRoundTripDoneCh |
| 865 |
|
| 866 |
// we expect the timeout handler to have timed out this request by now and any attempt |
| 867 |
// to write to the response should return a http.ErrHandlerTimeout error. |
| 868 |
_, innerHandlerWriteErr := w.Write([]byte("foo")) |
| 869 |
reqHandlerErrCh <- innerHandlerWriteErr |
| 870 |
|
| 871 |
panic(http.ErrAbortHandler) |
| 872 |
} |
| 873 |
}) |
| 874 |
|
| 875 |
// NOTE: the server will enforce a 5s timeout on every |
| 876 |
// incoming request, and the client enforces a timeout of 1m. |
| 877 |
handler := newHandlerChain(t, requestHandler, controller, userName, 5*time.Second) |
| 878 |
server, requestGetter := newHTTP2ServerWithClient(handler, time.Minute) |
| 879 |
defer server.Close() |
| 880 |
|
| 881 |
// send a request synchronously with a client timeout of 1m, this minimizes the |
| 882 |
// chance of a flake in ci, the cient waits long enough for the server to send a |
| 883 |
// timeout response to the client. |
| 884 |
var ( |
| 885 |
response *http.Response |
| 886 |
err error |
| 887 |
) |
| 888 |
func() { |
| 889 |
defer close(callerRoundTripDoneCh) |
| 890 |
t.Logf("Waiting for the request: %q to time out", rquestTimesOutPath) |
| 891 |
response, err = requestGetter(rquestTimesOutPath) |
| 892 |
if isClientTimeout(err) { |
| 893 |
t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", rquestTimesOutPath, err.Error()) |
| 894 |
} |
| 895 |
}() |
| 896 |
|
| 897 |
t.Logf("Waiting for the inner handler of the request: %q to complete", rquestTimesOutPath) |
| 898 |
select { |
| 899 |
case innerHandlerWriteErr := <-reqHandlerErrCh: |
| 900 |
if innerHandlerWriteErr != http.ErrHandlerTimeout { |
| 901 |
t.Fatalf("Expected error: %#v, but got: %#v", http.ErrHandlerTimeout, err) |
| 902 |
} |
| 903 |
case <-time.After(wait.ForeverTestTimeout): |
| 904 |
t.Errorf("Expected the server handler to have completed: %q", rquestTimesOutPath) |
| 905 |
} |
| 906 |
|
| 907 |
if err != nil { |
| 908 |
t.Fatalf("Expected request: %q to get a response, but got error: %#v", rquestTimesOutPath, err) |
| 909 |
} |
| 910 |
if response.StatusCode != http.StatusGatewayTimeout { |
| 911 |
t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#v", http.StatusGatewayTimeout, rquestTimesOutPath, response) |
| 912 |
} |
| 913 |
|
| 914 |
close(stopCh) |
| 915 |
t.Log("Waiting for the controller to shutdown") |
| 916 |
|
| 917 |
controllerErr := <-controllerCompletedCh |
| 918 |
if controllerErr != nil { |
| 919 |
t.Errorf("Expected no error from the controller, but got: %#v", controllerErr) |
| 920 |
} |
| 921 |
}) |
| 922 |
|
| 923 |
t.Run("priority level concurrency is set to 1, inner handler writes to the response before request times out", func(t *testing.T) { |
| 924 |
t.Parallel() |
| 925 |
const ( |
| 926 |
userName = "alice" |
| 927 |
fsName = "test-fs" |
| 928 |
plName = "test-pl" |
| 929 |
serverConcurrency, plConcurrencyShares, plConcurrency = 1, 1, 1 |
| 930 |
) |
| 931 |
|
| 932 |
apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, 0) |
| 933 |
stopCh := make(chan struct{}) |
| 934 |
controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency) |
| 935 |
|
| 936 |
headerMatcher := headerMatcher{} |
| 937 |
rquestTimesOutPath := "/request/time-out-as-designed" |
| 938 |
reqHandlerErrCh, callerRoundTripDoneCh := make(chan error, 1), make(chan struct{}) |
| 939 |
requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 940 |
headerMatcher.inspect(t, w, fsName, plName) |
| 941 |
|
| 942 |
if r.URL.Path == rquestTimesOutPath { |
| 943 |
|
| 944 |
// inner handler writes header and then let the request time out. |
| 945 |
w.WriteHeader(http.StatusBadRequest) |
| 946 |
<-callerRoundTripDoneCh |
| 947 |
|
| 948 |
// we expect the timeout handler to have timed out this request by now and any attempt |
| 949 |
// to write to the response should return a http.ErrHandlerTimeout error. |
| 950 |
_, innerHandlerWriteErr := w.Write([]byte("foo")) |
| 951 |
reqHandlerErrCh <- innerHandlerWriteErr |
| 952 |
} |
| 953 |
}) |
| 954 |
|
| 955 |
// NOTE: the server will enforce a 5s timeout on every |
| 956 |
// incoming request, and the client enforces a timeout of 1m. |
| 957 |
handler := newHandlerChain(t, requestHandler, controller, userName, 5*time.Second) |
| 958 |
server, requestGetter := newHTTP2ServerWithClient(handler, time.Minute) |
| 959 |
defer server.Close() |
| 960 |
|
| 961 |
// send a request synchronously with a client timeout of 1m, this minimizes the |
| 962 |
// chance of a flake in ci, the cient waits long enough for the server to send a |
| 963 |
// timeout response to the client. |
| 964 |
var err error |
| 965 |
func() { |
| 966 |
defer close(callerRoundTripDoneCh) |
| 967 |
t.Logf("Waiting for the request: %q to time out", rquestTimesOutPath) |
| 968 |
_, err = requestGetter(rquestTimesOutPath) |
| 969 |
if isClientTimeout(err) { |
| 970 |
t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", rquestTimesOutPath, err.Error()) |
| 971 |
} |
| 972 |
}() |
| 973 |
|
| 974 |
t.Logf("Waiting for the inner handler of the request: %q to complete", rquestTimesOutPath) |
| 975 |
select { |
| 976 |
case innerHandlerWriteErr := <-reqHandlerErrCh: |
| 977 |
if innerHandlerWriteErr != http.ErrHandlerTimeout { |
| 978 |
t.Fatalf("Expected error: %#v, but got: %#v", http.ErrHandlerTimeout, err) |
| 979 |
} |
| 980 |
case <-time.After(wait.ForeverTestTimeout): |
| 981 |
t.Errorf("Expected the server handler to have completed: %q", rquestTimesOutPath) |
| 982 |
} |
| 983 |
|
| 984 |
expectResetStreamError(t, err) |
| 985 |
|
| 986 |
close(stopCh) |
| 987 |
t.Log("Waiting for the controller to shutdown") |
| 988 |
|
| 989 |
controllerErr := <-controllerCompletedCh |
| 990 |
if controllerErr != nil { |
| 991 |
t.Errorf("Expected no error from the controller, but got: %#v", controllerErr) |
| 992 |
} |
| 993 |
}) |
| 994 |
|
| 995 |
t.Run("priority level concurrency is set to 1, queue length is 1, first request should time out and second (enqueued) request should time out as well", func(t *testing.T) { |
| 996 |
t.Parallel() |
| 997 |
|
| 998 |
type result struct { |
| 999 |
err error |
| 1000 |
response *http.Response |
| 1001 |
} |
| 1002 |
|
| 1003 |
const ( |
| 1004 |
userName = "alice" |
| 1005 |
fsName = "test-fs" |
| 1006 |
plName = "test-pl" |
| 1007 |
serverConcurrency, plConcurrencyShares, plConcurrency, queueLength = 1, 1, 1, 1 |
| 1008 |
) |
| 1009 |
|
| 1010 |
apfConfiguration := newConfiguration(fsName, plName, userName, plConcurrencyShares, queueLength) |
| 1011 |
stopCh := make(chan struct{}) |
| 1012 |
controller, controllerCompletedCh := startAPFController(t, stopCh, apfConfiguration, serverConcurrency, plName, plConcurrency) |
| 1013 |
|
| 1014 |
headerMatcher := headerMatcher{} |
| 1015 |
firstRequestTimesOutPath, secondRequestEnqueuedPath := "/request/first/time-out-as-designed", "/request/second/enqueued-as-designed" |
| 1016 |
firstReqHandlerErrCh, firstReqInProgressCh := make(chan error, 1), make(chan struct{}) |
| 1017 |
firstReqRoundTripDoneCh, secondReqRoundTripDoneCh := make(chan struct{}), make(chan struct{}) |
| 1018 |
requestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 1019 |
headerMatcher.inspect(t, w, fsName, plName) |
| 1020 |
switch { |
| 1021 |
case r.URL.Path == firstRequestTimesOutPath: |
| 1022 |
close(firstReqInProgressCh) |
| 1023 |
<-firstReqRoundTripDoneCh |
| 1024 |
|
| 1025 |
// make sure we wait until the caller of the second request returns, this is to |
| 1026 |
// ensure that second request never has a chance to be executed (to avoid flakes) |
| 1027 |
<-secondReqRoundTripDoneCh |
| 1028 |
|
| 1029 |
// we expect the timeout handler to have timed out this request by now and any attempt |
| 1030 |
// to write to the response should return a http.ErrHandlerTimeout error. |
| 1031 |
_, firstRequestInnerHandlerWriteErr := w.Write([]byte("foo")) |
| 1032 |
firstReqHandlerErrCh <- firstRequestInnerHandlerWriteErr |
| 1033 |
|
| 1034 |
case r.URL.Path == secondRequestEnqueuedPath: |
| 1035 |
// we expect the concurrency to be set to 1 and so this request should never be executed. |
| 1036 |
t.Errorf("Expected second request to be enqueued: %q", secondRequestEnqueuedPath) |
| 1037 |
} |
| 1038 |
}) |
| 1039 |
|
| 1040 |
// NOTE: the server will enforce a 5s timeout on every |
| 1041 |
// incoming request, and the client enforces a timeout of 1m. |
| 1042 |
handler := newHandlerChain(t, requestHandler, controller, userName, 5*time.Second) |
| 1043 |
server, requestGetter := newHTTP2ServerWithClient(handler, time.Minute) |
| 1044 |
defer server.Close() |
| 1045 |
|
| 1046 |
// This test involves two requests sent to the same priority level, which has 1 queue and |
| 1047 |
// a concurrency limit of 1. The handler chain include the timeout filter. |
| 1048 |
// Each request is sent from a separate goroutine, with a client-side timeout of 1m, on |
| 1049 |
// the other hand, the server enforces a timeout of 5s (via the timeout filter). |
| 1050 |
// The first request should get dispatched immediately; execution (a) starts with closing |
| 1051 |
// the channel that triggers the second client goroutine to send its request and then (b) |
| 1052 |
// waits for both client goroutines to have gotten a response (expected to be timeouts). |
| 1053 |
// The second request sits in the queue until the timeout filter does its thing, which |
| 1054 |
// it does concurrently to both requests. For the first request this should make the client |
| 1055 |
// get a timeout response without directly affecting execution. For the second request, the |
| 1056 |
// fact that the timeout filter closes the request's Context.Done() causes the request to be |
| 1057 |
// promptly ejected from its queue. The goroutine doing the APF handling writes an HTTP |
| 1058 |
// response message with status 429. |
| 1059 |
// The timeout handler invokes its inner handler in one goroutine while reacting to the |
| 1060 |
// passage of time in its original goroutine. That reaction to a time out consists of either |
| 1061 |
// (a) writing an HTTP response message with status 504 to indicate the timeout or (b) doing an |
| 1062 |
// HTTP/2 stream close; the latter is done if either the connection has been "hijacked" or some |
| 1063 |
// other goroutine (e.g., the one running the inner handler) has started to write a response. |
| 1064 |
// In the scenario tested here, there is thus a race between two goroutines to respond to |
| 1065 |
// the second request and any of their responses is allowed by the test. |
| 1066 |
firstReqResultCh, secondReqResultCh := make(chan result, 1), make(chan result, 1) |
| 1067 |
go func() { |
| 1068 |
defer close(firstReqRoundTripDoneCh) |
| 1069 |
t.Logf("At %s, Sending request: %q", time.Now().Format(timeFmt), firstRequestTimesOutPath) |
| 1070 |
resp, err := requestGetter(firstRequestTimesOutPath) |
| 1071 |
t.Logf("At %s, RoundTrip of request: %q has completed", time.Now().Format(timeFmt), firstRequestTimesOutPath) |
| 1072 |
firstReqResultCh <- result{err: err, response: resp} |
| 1073 |
}() |
| 1074 |
go func() { |
| 1075 |
// we must wait for the "first" request to start executing before |
| 1076 |
// we can initiate the "second". |
| 1077 |
defer close(secondReqRoundTripDoneCh) |
| 1078 |
|
| 1079 |
<-firstReqInProgressCh |
| 1080 |
t.Logf("At %s, Sending request: %q", time.Now().Format(timeFmt), secondRequestEnqueuedPath) |
| 1081 |
resp, err := requestGetter(secondRequestEnqueuedPath) |
| 1082 |
t.Logf("At %s, RoundTrip of request: %q has completed", time.Now().Format(timeFmt), secondRequestEnqueuedPath) |
| 1083 |
secondReqResultCh <- result{err: err, response: resp} |
| 1084 |
}() |
| 1085 |
|
| 1086 |
firstReqResult := <-firstReqResultCh |
| 1087 |
if isClientTimeout(firstReqResult.err) { |
| 1088 |
t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", firstRequestTimesOutPath, fmtError(firstReqResult.err)) |
| 1089 |
} |
| 1090 |
t.Logf("Waiting for the inner handler of the request: %q to complete", firstRequestTimesOutPath) |
| 1091 |
select { |
| 1092 |
case firstRequestInnerHandlerWriteErr := <-firstReqHandlerErrCh: |
| 1093 |
if firstRequestInnerHandlerWriteErr != http.ErrHandlerTimeout { |
| 1094 |
t.Fatalf("Expected error: %#v, but got: %s", http.ErrHandlerTimeout, fmtError(firstRequestInnerHandlerWriteErr)) |
| 1095 |
} |
| 1096 |
case <-time.After(wait.ForeverTestTimeout): |
| 1097 |
t.Errorf("Expected the server handler to have completed: %q", firstRequestTimesOutPath) |
| 1098 |
} |
| 1099 |
|
| 1100 |
// first request is expected to time out. |
| 1101 |
if isStreamReset(firstReqResult.err) || firstReqResult.response.StatusCode != http.StatusGatewayTimeout { |
| 1102 |
// got what was expected |
| 1103 |
} else if firstReqResult.err != nil { |
| 1104 |
t.Fatalf("Expected request: %q to get a response or stream reset, but got error: %s", firstRequestTimesOutPath, fmtError(firstReqResult.err)) |
| 1105 |
} else if firstReqResult.response.StatusCode != http.StatusGatewayTimeout { |
| 1106 |
t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#+v", http.StatusGatewayTimeout, firstRequestTimesOutPath, firstReqResult.response) |
| 1107 |
} |
| 1108 |
|
| 1109 |
// second request is expected to either be rejected (ideal behavior) or time out (current approximation of the ideal behavior) |
| 1110 |
secondReqResult := <-secondReqResultCh |
| 1111 |
if isClientTimeout(secondReqResult.err) { |
| 1112 |
t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", secondRequestEnqueuedPath, fmtError(secondReqResult.err)) |
| 1113 |
} |
| 1114 |
if isStreamReset(secondReqResult.err) || secondReqResult.response.StatusCode == http.StatusTooManyRequests || secondReqResult.response.StatusCode == http.StatusGatewayTimeout { |
| 1115 |
// got what was expected |
| 1116 |
} else if secondReqResult.err != nil { |
| 1117 |
t.Fatalf("Expected request: %q to get a response or stream reset, but got error: %s", secondRequestEnqueuedPath, fmtError(secondReqResult.err)) |
| 1118 |
} else if !(secondReqResult.response.StatusCode == http.StatusTooManyRequests || secondReqResult.response.StatusCode == http.StatusGatewayTimeout) { |
| 1119 |
t.Errorf("Expected HTTP status code: %d or %d for request: %q, but got: %#+v", http.StatusTooManyRequests, http.StatusGatewayTimeout, secondRequestEnqueuedPath, secondReqResult.response) |
| 1120 |
} |
| 1121 |
|
| 1122 |
close(stopCh) |
| 1123 |
t.Log("Waiting for the controller to shutdown") |
| 1124 |
|
| 1125 |
controllerErr := <-controllerCompletedCh |
| 1126 |
if controllerErr != nil { |
| 1127 |
t.Errorf("Expected no error from the controller, but got: %s", fmtError(controllerErr)) |
| 1128 |
} |
| 1129 |
}) |
| 1130 |
} |
| 1131 |
|
| 1132 |
func fmtError(err error) string { |
| 1133 |
return fmt.Sprintf("%#+v=%q", err, err.Error()) |
| 1134 |
} |
| 1135 |
|
| 1136 |
func startAPFController(t *testing.T, stopCh <-chan struct{}, apfConfiguration []runtime.Object, serverConcurrency int, |
| 1137 |
plName string, plConcurrency int) (utilflowcontrol.Interface, <-chan error) { |
| 1138 |
clientset := newClientset(t, apfConfiguration...) |
| 1139 |
// this test does not rely on resync, so resync period is set to zero |
| 1140 |
factory := informers.NewSharedInformerFactory(clientset, 0) |
| 1141 |
controller := utilflowcontrol.New(factory, clientset.FlowcontrolV1(), serverConcurrency) |
| 1142 |
|
| 1143 |
factory.Start(stopCh) |
| 1144 |
|
| 1145 |
// wait for the informer cache to sync. |
| 1146 |
timeout, cancel := context.WithTimeout(context.TODO(), 5*time.Second) |
| 1147 |
defer cancel() |
| 1148 |
cacheSyncDone := factory.WaitForCacheSync(timeout.Done()) |
| 1149 |
if names := unsyncedInformers(cacheSyncDone); len(names) > 0 { |
| 1150 |
t.Fatalf("WaitForCacheSync did not successfully complete, resources=%#v", names) |
| 1151 |
} |
| 1152 |
|
| 1153 |
controllerCompletedCh := make(chan error) |
| 1154 |
var controllerErr error |
| 1155 |
go func() { |
| 1156 |
controllerErr = controller.Run(stopCh) |
| 1157 |
controllerCompletedCh <- controllerErr |
| 1158 |
}() |
| 1159 |
|
| 1160 |
// make sure that apf controller syncs the priority level configuration object we are using in this test. |
| 1161 |
// read the metrics and ensure the concurrency limit for our priority level is set to the expected value. |
| 1162 |
pollErr := wait.PollImmediate(100*time.Millisecond, 5*time.Second, func() (done bool, err error) { |
| 1163 |
if err := gaugeValueMatch("apiserver_flowcontrol_nominal_limit_seats", map[string]string{"priority_level": plName}, plConcurrency); err != nil { |
| 1164 |
t.Logf("polling retry - error: %s", err) |
| 1165 |
return false, nil |
| 1166 |
} |
| 1167 |
return true, nil |
| 1168 |
}) |
| 1169 |
if pollErr != nil { |
| 1170 |
t.Fatalf("expected the apf controller to sync the priotity level configuration object: %s", plName) |
| 1171 |
} |
| 1172 |
|
| 1173 |
return controller, controllerCompletedCh |
| 1174 |
} |
| 1175 |
|
| 1176 |
// returns a started http2 server, with a client function to send request to the server. |
| 1177 |
func newHTTP2ServerWithClient(handler http.Handler, clientTimeout time.Duration) (*httptest.Server, func(path string) (*http.Response, error)) { |
| 1178 |
server := httptest.NewUnstartedServer(handler) |
| 1179 |
server.EnableHTTP2 = true |
| 1180 |
server.StartTLS() |
| 1181 |
cli := server.Client() |
| 1182 |
cli.Timeout = clientTimeout |
| 1183 |
return server, func(path string) (*http.Response, error) { |
| 1184 |
return cli.Get(server.URL + path) |
| 1185 |
} |
| 1186 |
} |
| 1187 |
|
| 1188 |
type headerMatcher struct{} |
| 1189 |
|
| 1190 |
// verifies that the expected flow schema and priority level UIDs are attached to the header. |
| 1191 |
func (m *headerMatcher) inspect(t *testing.T, w http.ResponseWriter, expectedFS, expectedPL string) { |
| 1192 |
t.Helper() |
| 1193 |
err := func() error { |
| 1194 |
if w == nil { |
| 1195 |
return fmt.Errorf("expected a non nil HTTP response") |
| 1196 |
} |
| 1197 |
|
| 1198 |
key := flowcontrol.ResponseHeaderMatchedFlowSchemaUID |
| 1199 |
if value := w.Header().Get(key); expectedFS != value { |
| 1200 |
return fmt.Errorf("expected HTTP header %s to have value %q, but got: %q", key, expectedFS, value) |
| 1201 |
} |
| 1202 |
|
| 1203 |
key = flowcontrol.ResponseHeaderMatchedPriorityLevelConfigurationUID |
| 1204 |
if value := w.Header().Get(key); expectedPL != value { |
| 1205 |
return fmt.Errorf("expected HTTP header %s to have value %q, but got %q", key, expectedPL, value) |
| 1206 |
} |
| 1207 |
return nil |
| 1208 |
}() |
| 1209 |
if err == nil { |
| 1210 |
return |
| 1211 |
} |
| 1212 |
t.Errorf("Expected APF headers to match, but got: %v", err) |
| 1213 |
} |
| 1214 |
|
| 1215 |
// when a request panics, http2 resets the stream with an INTERNAL_ERROR message |
| 1216 |
func expectResetStreamError(t *testing.T, err error) { |
| 1217 |
if err == nil { |
| 1218 |
t.Fatalf("expected the server to send an error, but got nil") |
| 1219 |
} |
| 1220 |
if isStreamReset(err) { |
| 1221 |
return |
| 1222 |
} |
| 1223 |
t.Fatalf("expected a stream reset error, but got %#+v=%s", err, err.Error()) |
| 1224 |
} |
| 1225 |
|
| 1226 |
func newClientset(t *testing.T, objects ...runtime.Object) clientset.Interface { |
| 1227 |
clientset := fake.NewSimpleClientset(objects...) |
| 1228 |
if clientset == nil { |
| 1229 |
t.Fatal("unable to create fake client set") |
| 1230 |
} |
| 1231 |
return clientset |
| 1232 |
} |
| 1233 |
|
| 1234 |
// builds a chain of handlers that include the panic recovery and timeout filter, so we can simulate the behavior of |
| 1235 |
// a real apiserver. |
| 1236 |
// the specified user is added as the authenticated user to the request context. |
| 1237 |
func newHandlerChain(t *testing.T, handler http.Handler, filter utilflowcontrol.Interface, userName string, requestTimeout time.Duration) http.Handler { |
| 1238 |
requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")} |
| 1239 |
longRunningRequestCheck := BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString("proxy")) |
| 1240 |
|
| 1241 |
apfHandler := WithPriorityAndFairness(handler, longRunningRequestCheck, filter, defaultRequestWorkEstimator, time.Minute/4) |
| 1242 |
|
| 1243 |
// add the handler in the chain that adds the specified user to the request context |
| 1244 |
handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 1245 |
r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{ |
| 1246 |
Name: userName, |
| 1247 |
Groups: []string{user.AllAuthenticated}, |
| 1248 |
})) |
| 1249 |
|
| 1250 |
apfHandler.ServeHTTP(w, r) |
| 1251 |
}) |
| 1252 |
|
| 1253 |
handler = WithTimeoutForNonLongRunningRequests(handler, longRunningRequestCheck) |
| 1254 |
// we don't have any request with invalid timeout, so leaving audit policy and sink nil. |
| 1255 |
handler = apifilters.WithRequestDeadline(handler, nil, nil, longRunningRequestCheck, nil, requestTimeout) |
| 1256 |
handler = apifilters.WithRequestInfo(handler, requestInfoFactory) |
| 1257 |
handler = WithPanicRecovery(handler, requestInfoFactory) |
| 1258 |
handler = apifilters.WithAuditInit(handler) |
| 1259 |
return handler |
| 1260 |
} |
| 1261 |
|
| 1262 |
func unsyncedInformers(status map[reflect.Type]bool) []string { |
| 1263 |
names := make([]string, 0) |
| 1264 |
|
| 1265 |
for objType, synced := range status { |
| 1266 |
if !synced { |
| 1267 |
names = append(names, objType.Name()) |
| 1268 |
} |
| 1269 |
} |
| 1270 |
|
| 1271 |
return names |
| 1272 |
} |
| 1273 |
|
| 1274 |
func newConfiguration(fsName, plName, user string, concurrency int32, queueLength int32) []runtime.Object { |
| 1275 |
fs := &flowcontrol.FlowSchema{ |
| 1276 |
ObjectMeta: metav1.ObjectMeta{ |
| 1277 |
Name: fsName, |
| 1278 |
UID: types.UID(fsName), |
| 1279 |
}, |
| 1280 |
Spec: flowcontrol.FlowSchemaSpec{ |
| 1281 |
MatchingPrecedence: 1, |
| 1282 |
PriorityLevelConfiguration: flowcontrol.PriorityLevelConfigurationReference{ |
| 1283 |
Name: plName, |
| 1284 |
}, |
| 1285 |
DistinguisherMethod: &flowcontrol.FlowDistinguisherMethod{ |
| 1286 |
Type: flowcontrol.FlowDistinguisherMethodByUserType, |
| 1287 |
}, |
| 1288 |
Rules: []flowcontrol.PolicyRulesWithSubjects{ |
| 1289 |
{ |
| 1290 |
Subjects: []flowcontrol.Subject{ |
| 1291 |
{ |
| 1292 |
Kind: flowcontrol.SubjectKindUser, |
| 1293 |
User: &flowcontrol.UserSubject{ |
| 1294 |
Name: user, |
| 1295 |
}, |
| 1296 |
}, |
| 1297 |
}, |
| 1298 |
NonResourceRules: []flowcontrol.NonResourcePolicyRule{ |
| 1299 |
{ |
| 1300 |
Verbs: []string{flowcontrol.VerbAll}, |
| 1301 |
NonResourceURLs: []string{flowcontrol.NonResourceAll}, |
| 1302 |
}, |
| 1303 |
}, |
| 1304 |
}, |
| 1305 |
}, |
| 1306 |
}, |
| 1307 |
} |
| 1308 |
|
| 1309 |
var ( |
| 1310 |
responseType flowcontrol.LimitResponseType = flowcontrol.LimitResponseTypeReject |
| 1311 |
qcfg *flowcontrol.QueuingConfiguration |
| 1312 |
) |
| 1313 |
if queueLength > 0 { |
| 1314 |
responseType = flowcontrol.LimitResponseTypeQueue |
| 1315 |
qcfg = &flowcontrol.QueuingConfiguration{ |
| 1316 |
Queues: 1, |
| 1317 |
QueueLengthLimit: queueLength, |
| 1318 |
HandSize: 1, |
| 1319 |
} |
| 1320 |
} |
| 1321 |
pl := &flowcontrol.PriorityLevelConfiguration{ |
| 1322 |
ObjectMeta: metav1.ObjectMeta{ |
| 1323 |
Name: plName, |
| 1324 |
UID: types.UID(plName), |
| 1325 |
}, |
| 1326 |
Spec: flowcontrol.PriorityLevelConfigurationSpec{ |
| 1327 |
Type: flowcontrol.PriorityLevelEnablementLimited, |
| 1328 |
Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ |
| 1329 |
NominalConcurrencyShares: ptr.To(concurrency), |
| 1330 |
LimitResponse: flowcontrol.LimitResponse{ |
| 1331 |
Type: responseType, |
| 1332 |
Queuing: qcfg, |
| 1333 |
}, |
| 1334 |
}, |
| 1335 |
}, |
| 1336 |
} |
| 1337 |
|
| 1338 |
return []runtime.Object{fs, pl} |
| 1339 |
} |
| 1340 |
|
| 1341 |
// gathers and checks the metrics. |
| 1342 |
func checkForExpectedMetrics(t *testing.T, expectedMetrics []string) { |
| 1343 |
metricsFamily, err := legacyregistry.DefaultGatherer.Gather() |
| 1344 |
if err != nil { |
| 1345 |
t.Fatalf("Failed to gather metrics %v", err) |
| 1346 |
} |
| 1347 |
|
| 1348 |
metrics := map[string]interface{}{} |
| 1349 |
for _, mf := range metricsFamily { |
| 1350 |
metrics[*mf.Name] = mf |
| 1351 |
} |
| 1352 |
|
| 1353 |
for _, metricName := range expectedMetrics { |
| 1354 |
if _, ok := metrics[metricName]; !ok { |
| 1355 |
if !ok { |
| 1356 |
t.Errorf("Scraped metrics did not include expected metric %s", metricName) |
| 1357 |
} |
| 1358 |
} |
| 1359 |
} |
| 1360 |
} |
| 1361 |
|
| 1362 |
// gaugeValueMatch ensures that the value of gauge metrics matching the labelFilter is as expected. |
| 1363 |
func gaugeValueMatch(name string, labelFilter map[string]string, wantValue int) error { |
| 1364 |
metrics, err := legacyregistry.DefaultGatherer.Gather() |
| 1365 |
if err != nil { |
| 1366 |
return fmt.Errorf("failed to gather metrics: %s", err) |
| 1367 |
} |
| 1368 |
|
| 1369 |
sum := 0 |
| 1370 |
familyMatch, labelMatch := false, false |
| 1371 |
for _, mf := range metrics { |
| 1372 |
if mf.GetName() != name { |
| 1373 |
continue |
| 1374 |
} |
| 1375 |
|
| 1376 |
familyMatch = true |
| 1377 |
for _, metric := range mf.GetMetric() { |
| 1378 |
if !testutil.LabelsMatch(metric, labelFilter) { |
| 1379 |
continue |
| 1380 |
} |
| 1381 |
|
| 1382 |
labelMatch = true |
| 1383 |
sum += int(metric.GetGauge().GetValue()) |
| 1384 |
} |
| 1385 |
} |
| 1386 |
if !familyMatch { |
| 1387 |
return fmt.Errorf("expected to find the metric family: %s in the gathered result", name) |
| 1388 |
} |
| 1389 |
if !labelMatch { |
| 1390 |
return fmt.Errorf("expected to find metrics with matching labels: %#+v", labelFilter) |
| 1391 |
} |
| 1392 |
if wantValue != sum { |
| 1393 |
return fmt.Errorf("expected the sum to be: %d, but got: %d for gauge metric: %s with labels %#+v", wantValue, sum, name, labelFilter) |
| 1394 |
} |
| 1395 |
|
| 1396 |
return nil |
| 1397 |
} |
| 1398 |
|
| 1399 |
func isClientTimeout(err error) bool { |
| 1400 |
if urlErr, ok := err.(*url.Error); ok { |
| 1401 |
return urlErr.Timeout() |
| 1402 |
} |
| 1403 |
return false |
| 1404 |
} |
| 1405 |
|
| 1406 |
func isStreamReset(err error) bool { |
| 1407 |
if err == nil { |
| 1408 |
return false |
| 1409 |
} |
| 1410 |
if urlErr, ok := err.(*url.Error); ok { |
| 1411 |
// Sadly, the client does not receive a more specific indication |
| 1412 |
// of stream reset. |
| 1413 |
return strings.Contains(urlErr.Err.Error(), "INTERNAL_ERROR") |
| 1414 |
} |
| 1415 |
return false |
| 1416 |
} |
| 1417 |
|
| 1418 |
func TestGetRequestWaitContext(t *testing.T) { |
| 1419 |
tests := []struct { |
| 1420 |
name string |
| 1421 |
defaultRequestWaitLimit time.Duration |
| 1422 |
parent func(t time.Time) (context.Context, context.CancelFunc) |
| 1423 |
newReqWaitCtxExpected bool |
| 1424 |
reqWaitLimitExpected time.Duration |
| 1425 |
}{ |
| 1426 |
{ |
| 1427 |
name: "context deadline has exceeded", |
| 1428 |
parent: func(time.Time) (context.Context, context.CancelFunc) { |
| 1429 |
ctx, cancel := context.WithCancel(context.Background()) |
| 1430 |
cancel() |
| 1431 |
return ctx, cancel |
| 1432 |
}, |
| 1433 |
}, |
| 1434 |
{ |
| 1435 |
name: "context has a deadline, 'received at' is not set, wait limit should be one fourth of the remaining deadline from now", |
| 1436 |
parent: func(now time.Time) (context.Context, context.CancelFunc) { |
| 1437 |
return context.WithDeadline(context.Background(), now.Add(60*time.Second)) |
| 1438 |
}, |
| 1439 |
newReqWaitCtxExpected: true, |
| 1440 |
reqWaitLimitExpected: 15 * time.Second, |
| 1441 |
}, |
| 1442 |
{ |
| 1443 |
name: "context has a deadline, 'received at' is set, wait limit should be one fourth of the deadline starting from the 'received at' time", |
| 1444 |
parent: func(now time.Time) (context.Context, context.CancelFunc) { |
| 1445 |
ctx := apirequest.WithReceivedTimestamp(context.Background(), now.Add(-10*time.Second)) |
| 1446 |
return context.WithDeadline(ctx, now.Add(50*time.Second)) |
| 1447 |
}, |
| 1448 |
newReqWaitCtxExpected: true, |
| 1449 |
reqWaitLimitExpected: 5 * time.Second, // from now |
| 1450 |
}, |
| 1451 |
{ |
| 1452 |
name: "context does not have any deadline, 'received at' is not set, default wait limit should be in effect from now", |
| 1453 |
defaultRequestWaitLimit: 15 * time.Second, |
| 1454 |
parent: func(time.Time) (context.Context, context.CancelFunc) { |
| 1455 |
return context.WithCancel(context.Background()) |
| 1456 |
}, |
| 1457 |
newReqWaitCtxExpected: true, |
| 1458 |
reqWaitLimitExpected: 15 * time.Second, |
| 1459 |
}, |
| 1460 |
{ |
| 1461 |
name: "context does not have any deadline, 'received at' is set, default wait limit should be in effect starting from the 'received at' time", |
| 1462 |
defaultRequestWaitLimit: 15 * time.Second, |
| 1463 |
parent: func(now time.Time) (context.Context, context.CancelFunc) { |
| 1464 |
ctx := apirequest.WithReceivedTimestamp(context.Background(), now.Add(-10*time.Second)) |
| 1465 |
return context.WithCancel(ctx) |
| 1466 |
}, |
| 1467 |
newReqWaitCtxExpected: true, |
| 1468 |
reqWaitLimitExpected: 5 * time.Second, // from now |
| 1469 |
}, |
| 1470 |
{ |
| 1471 |
name: "context has a deadline, wait limit should not exceed the hard limit of 1m", |
| 1472 |
parent: func(now time.Time) (context.Context, context.CancelFunc) { |
| 1473 |
// let 1/4th of the remaining deadline exceed the hard limit |
| 1474 |
return context.WithDeadline(context.Background(), now.Add(8*time.Minute)) |
| 1475 |
}, |
| 1476 |
newReqWaitCtxExpected: true, |
| 1477 |
reqWaitLimitExpected: time.Minute, |
| 1478 |
}, |
| 1479 |
{ |
| 1480 |
name: "context has no deadline, wait limit should not exceed the hard limit of 1m", |
| 1481 |
defaultRequestWaitLimit: 2 * time.Minute, // it exceeds the hard limit |
| 1482 |
parent: func(now time.Time) (context.Context, context.CancelFunc) { |
| 1483 |
return context.WithCancel(context.Background()) |
| 1484 |
}, |
| 1485 |
newReqWaitCtxExpected: true, |
| 1486 |
reqWaitLimitExpected: time.Minute, |
| 1487 |
}, |
| 1488 |
} |
| 1489 |
|
| 1490 |
for _, test := range tests { |
| 1491 |
t.Run(test.name, func(t *testing.T) { |
| 1492 |
now := time.Now() |
| 1493 |
parent, cancel := test.parent(now) |
| 1494 |
defer cancel() |
| 1495 |
|
| 1496 |
clock := clocktesting.NewFakePassiveClock(now) |
| 1497 |
newReqWaitCtxGot, cancelGot := getRequestWaitContext(parent, test.defaultRequestWaitLimit, clock) |
| 1498 |
if cancelGot == nil { |
| 1499 |
t.Errorf("Expected a non nil context.CancelFunc") |
| 1500 |
return |
| 1501 |
} |
| 1502 |
defer cancelGot() |
| 1503 |
|
| 1504 |
switch { |
| 1505 |
case test.newReqWaitCtxExpected: |
| 1506 |
deadlineGot, ok := newReqWaitCtxGot.Deadline() |
| 1507 |
if !ok { |
| 1508 |
t.Errorf("Expected the new wait limit context to have a deadline") |
| 1509 |
} |
| 1510 |
if waitLimitGot := deadlineGot.Sub(now); test.reqWaitLimitExpected != waitLimitGot { |
| 1511 |
t.Errorf("Expected request wait limit %s, but got: %s", test.reqWaitLimitExpected, waitLimitGot) |
| 1512 |
} |
| 1513 |
default: |
| 1514 |
if parent != newReqWaitCtxGot { |
| 1515 |
t.Errorf("Expected the parent context to be returned: want: %#v, got %#v", parent, newReqWaitCtxGot) |
| 1516 |
} |
| 1517 |
} |
| 1518 |
}) |
| 1519 |
} |
| 1520 |
} |