Debian Codegraph @

pkg/server/read_write_deadline_test.go

raw
1 /*
2 Copyright 2024 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package server
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "io"
24 "net/http"
25 "net/http/httptest"
26 "strings"
27 "testing"
28 "time"
29
30 "k8s.io/apimachinery/pkg/util/wait"
31 "k8s.io/apiserver/pkg/endpoints/request"
32 )
33
34 func TestRequestTimeoutBehavior(t *testing.T) {
35 type setup struct {
36 name string
37 clientTimeout time.Duration
38 serverReqTimeout time.Duration
39 handlerWritesBeforeTimeout bool
40 handlerFlushesBeforeTimeout bool
41 waiter waiter
42 }
43 type expectation struct {
44 clientErr verifier
45 clientStatusCodeExpected int
46 clientRespBodyReadErr verifier
47 handlerWriteErr verifier
48 }
49
50 tests := []struct {
51 setup setup
52 expectations map[string]expectation
53 }{
54 // scenario:
55 // a) timeout filter enabled: Yes
56 // b) client specifies timeout in the request URI: No
57 // c) the handler writes to the ResponseWriter object before request times out: No
58 // d) the handler flushes the ResponseWriter object before request times out: No
59 // observation:
60 // the timeout filter detects that the context of the request has exceeded its
61 // deadline, since the ResponseWriter object has not been written to yet,
62 // the following takes place:
63 // - it marks the ResponseWriter object as timeout=true, so any further
64 // attempt to write to it will yield an 'http: Handler timeout' error
65 // - it sends 504 status code to the client
66 // expectation (same behavior for both http/1x and http/2.0):
67 // client:
68 // - client receives a '504 GatewayTimeout' status code
69 // - reading the Body of the Response object yields an error
70 // server:
71 // - Write to the ResponseWriter yields an 'http: Handler timeout'
72 // error immediately.
73 {
74 setup: setup{
75 name: "timeout occurs before the handler writes to or flushes the ResponseWriter",
76 clientTimeout: 0, // b
77 handlerWritesBeforeTimeout: false, // c
78 handlerFlushesBeforeTimeout: false, // d
79 serverReqTimeout: time.Second,
80 },
81 expectations: map[string]expectation{
82 "HTTP/2.0": {
83 clientErr: wantNoError{},
84 clientStatusCodeExpected: http.StatusGatewayTimeout,
85 clientRespBodyReadErr: wantNoError{},
86 handlerWriteErr: wantError{http.ErrHandlerTimeout},
87 },
88 "HTTP/1.1": {
89 clientErr: wantNoError{},
90 clientStatusCodeExpected: http.StatusGatewayTimeout,
91 clientRespBodyReadErr: wantNoError{},
92 handlerWriteErr: wantError{http.ErrHandlerTimeout},
93 },
94 },
95 },
96
97 // scenario:
98 // a) timeout filter enabled: Yes
99 // b) client specifies timeout in the request URI: No
100 // c) the handler writes to the ResponseWriter object before request times out: Yes
101 // d) the handler flushes the ResponseWriter object before request times out: No
102 // observation:
103 // the timeout filter detects that the context of the request has exceeded its
104 // deadline, since the ResponseWriter object has already been written to,
105 // the following takes place:
106 // - it marks the ResponseWriter object as timeout=true, so any further attempt
107 // to write to it will yield an 'http: Handler timeout' error
108 // - it can't send '504 GatewayTimeout' to the client since the ResponseWriter
109 // object has already been written to, so it panics with 'net/http: abort Handler' error
110 {
111 setup: setup{
112 name: "timeout occurs after the handler writes to the ResponseWriter",
113 clientTimeout: 0, // b
114 handlerWritesBeforeTimeout: true, // c
115 handlerFlushesBeforeTimeout: false, // d
116 serverReqTimeout: time.Second,
117 },
118 expectations: map[string]expectation{
119 // expectation:
120 // - client: receives a stream reset error, no 'Response' from the server
121 // - server: Write to the ResponseWriter yields an 'http: Handler timeout' error
122 "HTTP/2.0": {
123 clientErr: wantContains{"stream error: stream ID 1; INTERNAL_ERROR; received from peer"},
124 clientStatusCodeExpected: 0,
125 clientRespBodyReadErr: wantNoError{},
126 handlerWriteErr: wantError{http.ErrHandlerTimeout},
127 },
128 // expectation:
129 // - client: receives an 'io.EOF' error, no 'Response' from the server
130 // - server: Write to the ResponseWriter yields an 'http: Handler timeout' error
131 "HTTP/1.1": {
132 clientErr: wantError{io.EOF},
133 clientStatusCodeExpected: 0,
134 clientRespBodyReadErr: wantNoError{},
135 handlerWriteErr: wantError{http.ErrHandlerTimeout},
136 },
137 },
138 },
139
140 // scenario:
141 // a) timeout filter enabled: Yes
142 // b) client specifies timeout in the request URI: No
143 // c) the handler writes to the ResponseWriter object before request times out: Yes
144 // d) the handler flushes the ResponseWriter object before request times out: Yes
145 // observation:
146 // the timeout filter detects that the context of the request has exceeded its
147 // deadline, since the ResponseWriter object has already been written to,
148 // the following takes place:
149 // - it marks the ResponseWriter object as timeout=true, so any further attempt
150 // to write to it will yield an 'http: Handler timeout' error
151 // - it can't send '504 GatewayTimeout' to the client since the ResponseWriter
152 // object has already been written to, so it panics with 'net/http: abort Handler' error
153 {
154 setup: setup{
155 name: "timeout occurs after the handler writes to and flushes the ResponseWriter",
156 clientTimeout: 0, // b
157 handlerWritesBeforeTimeout: true, // c
158 handlerFlushesBeforeTimeout: true, // d
159 serverReqTimeout: time.Second,
160 },
161 expectations: map[string]expectation{
162 // expectation:
163 // - client: since the ResponseWriter has been flushed the client
164 // receives a response from the server, but reading the response body
165 // is expected to yield a stream reset error.
166 // - server: Write to the ResponseWriter yields an 'http: Handler timeout' error
167 "HTTP/2.0": {
168 clientErr: wantNoError{},
169 clientStatusCodeExpected: 200,
170 clientRespBodyReadErr: wantContains{"stream error: stream ID 1; INTERNAL_ERROR; received from peer"},
171 handlerWriteErr: wantError{http.ErrHandlerTimeout},
172 },
173 // expectation:
174 // - client: since the ResponseWriter has been flushed the client
175 // receives a response from the server, but reading the response body
176 // will yield an 'unexpected EOF' error.
177 // - server: Write to the ResponseWriter yields an 'http: Handler timeout' error
178 "HTTP/1.1": {
179 clientErr: wantNoError{},
180 clientStatusCodeExpected: 200,
181 clientRespBodyReadErr: wantError{io.ErrUnexpectedEOF},
182 handlerWriteErr: wantError{http.ErrHandlerTimeout},
183 },
184 },
185 },
186
187 // scenario:
188 // a) timeout filter enabled: Yes
189 // b) client specifies timeout in the request URI: Yes
190 // c) the handler writes to the ResponseWriter object before request times out: Yes
191 // d) the handler flushes the ResponseWriter object before request times out: No
192 // observation:
193 // the timeout filter detects that the context of the request has exceeded its
194 // deadline, and the the following takes place:
195 // - it marks the ResponseWriter object as timeout=true, so any further attempt
196 // to write to it will yield an 'http: Handler timeout' error
197 // - it can't send '504 GatewayTimeout' to the client since the ResponseWriter
198 // object has already been written to, so it panics with 'net/http: abort Handler' error
199 // at the same time, the net/http client also detects that the context of the
200 // client-side request has exceeded its deadline, and so it aborts with a
201 // 'context deadline exceeded' error.
202 // NOTE: although the client is most likely to receive the context deadline error
203 // first due to the roundtrip time added to the arrival of the error from
204 // the server, nevertheless it could cause flakes in CI due to overload, so we
205 // need to check for either error to be flake free.
206 {
207 setup: setup{
208 name: "client specifies a timeout",
209 clientTimeout: time.Second, // b
210 handlerWritesBeforeTimeout: true, // c
211 handlerFlushesBeforeTimeout: false, // d
212 serverReqTimeout: wait.ForeverTestTimeout * 2, // this should not be in effect
213
214 // twice the request timeout so it can withstand flakes in CI
215 waiter: &waitWithDuration{after: 2 * time.Second},
216 },
217 expectations: map[string]expectation{
218 // expectation:
219 // - client: receives either a context.DeadlineExceeded error from its transport
220 // or it receives the error from the server
221 // - server: Write to the ResponseWriter will yields an 'http: Handler timeout'
222 "HTTP/2.0": {
223 clientErr: wantEitherOr{
224 err: context.DeadlineExceeded,
225 contains: "stream error: stream ID 1; INTERNAL_ERROR; received from peer",
226 },
227 clientStatusCodeExpected: 0,
228 clientRespBodyReadErr: wantNoError{},
229 handlerWriteErr: wantError{http.ErrHandlerTimeout},
230 },
231 // expectation:
232 // - client: receives either a context.DeadlineExceeded error from its
233 // transport, or it receives the error from the server
234 // - server: Write to the ResponseWriter will yields an 'http: Handler timeout'
235 "HTTP/1.1": {
236 clientErr: wantEitherOr{
237 err: context.DeadlineExceeded,
238 contains: "EOF",
239 },
240 clientStatusCodeExpected: 0,
241 clientRespBodyReadErr: wantNoError{},
242 handlerWriteErr: wantError{http.ErrHandlerTimeout},
243 },
244 },
245 },
246 }
247 for _, test := range tests {
248 for _, proto := range []string{"HTTP/1.1", "HTTP/2.0"} { // every test is run for both http/1x and http/2.0
249 t.Run(fmt.Sprintf("%s/%s", test.setup.name, proto), func(t *testing.T) {
250 setup := test.setup
251 want, ok := test.expectations[proto]
252 if !ok {
253 t.Fatalf("wrong test setup - no expectation for %s", proto)
254 }
255
256 fakeAudit := &fakeAudit{}
257 config, _ := setUp(t)
258 config.AuditPolicyRuleEvaluator = fakeAudit
259 config.AuditBackend = fakeAudit
260
261 // setup server run option --request-timeout
262 config.RequestTimeout = setup.serverReqTimeout
263
264 s, err := config.Complete(nil).New("test", NewEmptyDelegate())
265 if err != nil {
266 t.Fatalf("Error in setting up a GenericAPIServer object: %v", err)
267 }
268
269 // using this, the handler blocks until the timeout occurs
270 waiter := setup.waiter
271 if waiter == nil {
272 waiter = &waitWithChannelClose{after: make(chan time.Time)}
273 }
274
275 // this is the timeout we expect the context of a request
276 // on the server to have.
277 // - if the client does not specify a timeout parameter in
278 // the request URI then it should default to --request-timeout
279 // - otherwise, it should be the timeout specified by the client
280 reqCtxTimeoutWant := config.RequestTimeout
281 if setup.clientTimeout > 0 {
282 reqCtxTimeoutWant = setup.clientTimeout
283 }
284
285 handlerDoneCh := make(chan struct{})
286 handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
287 defer close(handlerDoneCh)
288
289 ctx := r.Context()
290 if r.Proto != proto {
291 t.Errorf("expected protocol: %q, but got: %q", proto, r.Proto)
292 return
293 }
294
295 // TODO: we don't support `FlushError` yet
296 flusher, ok := w.(http.Flusher)
297 if !ok {
298 t.Errorf("expected ResponseWriter object to implement FlushError")
299 return
300 }
301
302 // make sure that we have the right request
303 // - it must be a non long-running request
304 // - it must have a received-at timestamp so we
305 // can calculate the request deadline accurately.
306 // - the context of the request must have the
307 // expected deadline
308 reqInfo, ok := request.RequestInfoFrom(ctx)
309 if !ok {
310 t.Errorf("expected the request context to have a RequestInfo associated")
311 return
312 }
313 if config.LongRunningFunc(r, reqInfo) {
314 t.Errorf("wrong test setup, wanted a non long-running request, but got: %#v", reqInfo)
315 return
316 }
317 receivedAt, ok := request.ReceivedTimestampFrom(ctx)
318 if !ok {
319 t.Errorf("expected the request context to have a received-at timestamp")
320 return
321 }
322 deadline, ok := ctx.Deadline()
323 if !ok {
324 t.Errorf("expected the request context to have a deadline")
325 return
326 }
327 if want, got := reqCtxTimeoutWant, deadline.Sub(receivedAt); want != got {
328 t.Errorf("expected the request context to have a deadline of: %s, but got: %s", want, got)
329 return
330 }
331
332 // does the handler write to or flush the
333 // ResponseWriter object before timeout occurs?
334 if setup.handlerWritesBeforeTimeout {
335 if _, err := w.Write([]byte("hello")); err != nil {
336 t.Errorf("unexpected error from Write: %v", err)
337 return
338 }
339 }
340 if setup.handlerFlushesBeforeTimeout {
341 flusher.Flush()
342 }
343
344 // wait for the request context deadline to elapse
345 <-waiter.wait()
346
347 // write to the ResponseWriter object after timeout happens
348 _, err := w.Write([]byte("a"))
349 want.handlerWriteErr.verify(t, err)
350
351 // flush the ResponseWriter object after timeout happens
352 // http.Flusher does not return an error
353 flusher.Flush()
354 })
355 s.Handler.NonGoRestfulMux.Handle("/ping", handler)
356
357 server := httptest.NewUnstartedServer(s.Handler)
358 defer server.Close()
359 if proto == "HTTP/2.0" {
360 server.EnableHTTP2 = true
361 }
362 server.StartTLS()
363
364 func() {
365 defer waiter.close()
366
367 client := server.Client()
368
369 url := fmt.Sprintf("%s/ping", server.URL)
370 // if the user has specified a timeout then add
371 // it to the request URI
372 if setup.clientTimeout > 0 {
373 url = fmt.Sprintf("%s?timeout=%s", url, setup.clientTimeout)
374 }
375 // if the client has specified a timeout then we
376 // must wire the request context with the same
377 // deadline, this is how client-go behaves today.
378 ctx := context.Background()
379 if setup.clientTimeout > 0 {
380 var cancel context.CancelFunc
381 ctx, cancel = context.WithTimeout(ctx, setup.clientTimeout)
382 defer cancel()
383 }
384
385 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
386 if err != nil {
387 t.Errorf("failed to create a new http request - %v", err)
388 return
389 }
390
391 resp, err := client.Do(req)
392 want.clientErr.verify(t, err)
393
394 // do we expect a valid http status code?
395 switch {
396 case want.clientStatusCodeExpected > 0:
397 if resp == nil {
398 t.Errorf("expected a response from the server: %v", err)
399 return
400 }
401 if resp.StatusCode != want.clientStatusCodeExpected {
402 t.Errorf("expected a status code: %d, but got: %#v", want.clientStatusCodeExpected, resp)
403 }
404
405 // read off the body of the response, and verify what we expect
406 _, err = io.ReadAll(resp.Body)
407 want.clientRespBodyReadErr.verify(t, err)
408
409 if err := resp.Body.Close(); err != nil {
410 t.Errorf("unexpected error while closing the Body of the Response: %v", err)
411 }
412 default:
413 if resp != nil {
414 t.Errorf("did not expect a Response from the server, but got: %#v", resp)
415 }
416 return
417 }
418 }()
419
420 select {
421 case <-handlerDoneCh:
422 case <-time.After(wait.ForeverTestTimeout):
423 t.Errorf("expected the request handler to have terminated")
424 }
425 })
426 }
427 }
428 }
429
430 type verifier interface {
431 verify(t *testing.T, got error)
432 }
433
434 type wantNoError struct{}
435
436 func (v wantNoError) verify(t *testing.T, got error) {
437 t.Helper()
438 if got != nil {
439 t.Errorf("unexpected error: %v", got)
440 }
441 }
442
443 type wantContains struct {
444 contains string
445 }
446
447 func (v wantContains) verify(t *testing.T, got error) {
448 t.Helper()
449
450 switch {
451 case got != nil:
452 if !strings.Contains(got.Error(), v.contains) {
453 t.Errorf("expected the error to contain: %q, but got: %v", v.contains, got)
454 }
455 default:
456 t.Errorf("expected an error that contains %q, but got none", v.contains)
457 }
458 }
459
460 type wantError struct {
461 err error
462 }
463
464 func (v wantError) verify(t *testing.T, got error) {
465 t.Helper()
466
467 switch {
468 case got != nil:
469 if !errors.Is(got, v.err) {
470 t.Errorf("expected error: %v, but got: %v", v.err, got)
471 }
472 default:
473 t.Errorf("expected an error %v, but got none", v.err)
474 }
475 }
476
477 type wantEitherOr struct {
478 err error
479 contains string
480 }
481
482 func (v wantEitherOr) verify(t *testing.T, got error) {
483 t.Helper()
484
485 switch {
486 case got != nil:
487 if !(errors.Is(got, v.err) || strings.Contains(got.Error(), v.contains)) {
488 t.Errorf("expected the error to contain: %q or be: %v, but got: %v", v.contains, v.err, got)
489 }
490 default:
491 t.Errorf("expected an error to contain: %q or be: %v, but got none", v.contains, v.err)
492 }
493 }
494
495 type waiter interface {
496 wait() <-chan time.Time
497 close()
498 }
499
500 type waitWithDuration struct {
501 after time.Duration
502 }
503
504 func (w waitWithDuration) wait() <-chan time.Time { return time.After(w.after) }
505 func (w waitWithDuration) close() {}
506
507 type waitWithChannelClose struct {
508 after chan time.Time
509 }
510
511 func (w waitWithChannelClose) wait() <-chan time.Time {
512 // for http/2, we do the following:
513 // a) let the handler block indefinitely
514 // b) this forces the write timeout to occur on the server side
515 // c) the http2 client receives a stream reset error immediately
516 // after the write timeout occurs.
517 // d) the client then closes the channel by calling close
518 // e) the handler unblocks and terminates
519 return w.after
520 }
521
522 func (w waitWithChannelClose) close() { close(w.after) }