Debian Codegraph @

pkg/server/filters/timeout.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package filters
18
19 import (
20 "bufio"
21 "encoding/json"
22 "fmt"
23 "net"
24 "net/http"
25 "runtime"
26 "sync"
27 "time"
28
29 apierrors "k8s.io/apimachinery/pkg/api/errors"
30 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
31 "k8s.io/apiserver/pkg/endpoints/metrics"
32 apirequest "k8s.io/apiserver/pkg/endpoints/request"
33 "k8s.io/apiserver/pkg/endpoints/responsewriter"
34 "k8s.io/apiserver/pkg/server/httplog"
35 )
36
37 // WithTimeoutForNonLongRunningRequests times out non-long-running requests after the time given by timeout.
38 func WithTimeoutForNonLongRunningRequests(handler http.Handler, longRunning apirequest.LongRunningRequestCheck) http.Handler {
39 if longRunning == nil {
40 return handler
41 }
42 timeoutFunc := func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) {
43 // TODO unify this with apiserver.MaxInFlightLimit
44 ctx := req.Context()
45
46 requestInfo, ok := apirequest.RequestInfoFrom(ctx)
47 if !ok {
48 // if this happens, the handler chain isn't setup correctly because there is no request info
49 return req, false, func() {}, apierrors.NewInternalError(fmt.Errorf("no request info found for request during timeout"))
50 }
51
52 if longRunning(req, requestInfo) {
53 return req, true, nil, nil
54 }
55
56 postTimeoutFn := func() {
57 metrics.RecordRequestTermination(req, requestInfo, metrics.APIServerComponent, http.StatusGatewayTimeout)
58 }
59 return req, false, postTimeoutFn, apierrors.NewTimeoutError("request did not complete within the allotted timeout", 0)
60 }
61 return WithTimeout(handler, timeoutFunc)
62 }
63
64 type timeoutFunc = func(*http.Request) (req *http.Request, longRunning bool, postTimeoutFunc func(), err *apierrors.StatusError)
65
66 // WithTimeout returns an http.Handler that runs h with a timeout
67 // determined by timeoutFunc. The new http.Handler calls h.ServeHTTP to handle
68 // each request, but if a call runs for longer than its time limit, the
69 // handler responds with a 504 Gateway Timeout error and the message
70 // provided. (If msg is empty, a suitable default message will be sent.) After
71 // the handler times out, writes by h to its http.ResponseWriter will return
72 // http.ErrHandlerTimeout. If timeoutFunc returns a nil timeout channel, no
73 // timeout will be enforced. recordFn is a function that will be invoked whenever
74 // a timeout happens.
75 func WithTimeout(h http.Handler, timeoutFunc timeoutFunc) http.Handler {
76 return &timeoutHandler{h, timeoutFunc}
77 }
78
79 type timeoutHandler struct {
80 handler http.Handler
81 timeout timeoutFunc
82 }
83
84 func (t *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
85 r, longRunning, postTimeoutFn, err := t.timeout(r)
86 if longRunning {
87 t.handler.ServeHTTP(w, r)
88 return
89 }
90
91 timeoutCh := r.Context().Done()
92
93 // resultCh is used as both errCh and stopCh
94 resultCh := make(chan interface{})
95 var tw timeoutWriter
96 tw, w = newTimeoutWriter(w)
97
98 // Make a copy of request and work on it in new goroutine
99 // to avoid race condition when accessing/modifying request (e.g. headers)
100 rCopy := r.Clone(r.Context())
101 go func() {
102 defer func() {
103 err := recover()
104 // do not wrap the sentinel ErrAbortHandler panic value
105 if err != nil && err != http.ErrAbortHandler {
106 // Same as stdlib http server code. Manually allocate stack
107 // trace buffer size to prevent excessively large logs
108 const size = 64 << 10
109 buf := make([]byte, size)
110 buf = buf[:runtime.Stack(buf, false)]
111 err = fmt.Sprintf("%v\n%s", err, buf)
112 }
113 resultCh <- err
114 }()
115 t.handler.ServeHTTP(w, rCopy)
116 }()
117 select {
118 case err := <-resultCh:
119 // panic if error occurs; stop otherwise
120 if err != nil {
121 panic(err)
122 }
123 return
124 case <-timeoutCh:
125 defer func() {
126 // resultCh needs to have a reader, since the function doing
127 // the work needs to send to it. This is defer'd to ensure it runs
128 // ever if the post timeout work itself panics.
129 go func() {
130 timedOutAt := time.Now()
131 res := <-resultCh
132
133 status := metrics.PostTimeoutHandlerOK
134 if res != nil {
135 // a non nil res indicates that there was a panic.
136 status = metrics.PostTimeoutHandlerPanic
137 }
138
139 metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceTimeoutHandler, status)
140 utilruntime.HandleErrorWithContext(r.Context(), nil, "Post-timeout activity", "timeElapsed", time.Since(timedOutAt), "method", r.Method, "path", r.URL.Path, "result", res)
141 }()
142 }()
143 httplog.SetStacktracePredicate(r.Context(), func(status int) bool {
144 return false
145 })
146 defer postTimeoutFn()
147 tw.timeout(err)
148 }
149 }
150
151 type timeoutWriter interface {
152 http.ResponseWriter
153 timeout(*apierrors.StatusError)
154 }
155
156 func newTimeoutWriter(w http.ResponseWriter) (timeoutWriter, http.ResponseWriter) {
157 base := &baseTimeoutWriter{w: w, handlerHeaders: w.Header().Clone()}
158 wrapped := responsewriter.WrapForHTTP1Or2(base)
159
160 return base, wrapped
161 }
162
163 var _ http.ResponseWriter = &baseTimeoutWriter{}
164 var _ responsewriter.UserProvidedDecorator = &baseTimeoutWriter{}
165
166 type baseTimeoutWriter struct {
167 w http.ResponseWriter
168
169 // headers written by the normal handler
170 handlerHeaders http.Header
171
172 mu sync.Mutex
173 // if the timeout handler has timeout
174 timedOut bool
175 // if this timeout writer has wrote header
176 wroteHeader bool
177 // if this timeout writer has been hijacked
178 hijacked bool
179 }
180
181 func (tw *baseTimeoutWriter) Unwrap() http.ResponseWriter {
182 return tw.w
183 }
184
185 func (tw *baseTimeoutWriter) Header() http.Header {
186 tw.mu.Lock()
187 defer tw.mu.Unlock()
188
189 if tw.timedOut {
190 return http.Header{}
191 }
192
193 return tw.handlerHeaders
194 }
195
196 func (tw *baseTimeoutWriter) Write(p []byte) (int, error) {
197 tw.mu.Lock()
198 defer tw.mu.Unlock()
199
200 if tw.timedOut {
201 return 0, http.ErrHandlerTimeout
202 }
203 if tw.hijacked {
204 return 0, http.ErrHijacked
205 }
206
207 if !tw.wroteHeader {
208 copyHeaders(tw.w.Header(), tw.handlerHeaders)
209 tw.wroteHeader = true
210 }
211 return tw.w.Write(p)
212 }
213
214 func (tw *baseTimeoutWriter) Flush() {
215 tw.mu.Lock()
216 defer tw.mu.Unlock()
217
218 if tw.timedOut {
219 return
220 }
221
222 // the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
223 // http.Flusher if the inner object (tw.w) implements http.Flusher.
224 tw.w.(http.Flusher).Flush()
225 }
226
227 func (tw *baseTimeoutWriter) WriteHeader(code int) {
228 tw.mu.Lock()
229 defer tw.mu.Unlock()
230
231 if tw.timedOut || tw.wroteHeader || tw.hijacked {
232 return
233 }
234
235 copyHeaders(tw.w.Header(), tw.handlerHeaders)
236 tw.wroteHeader = true
237 tw.w.WriteHeader(code)
238 }
239
240 func copyHeaders(dst, src http.Header) {
241 for k, v := range src {
242 dst[k] = v
243 }
244 }
245
246 func (tw *baseTimeoutWriter) timeout(err *apierrors.StatusError) {
247 tw.mu.Lock()
248 defer tw.mu.Unlock()
249
250 tw.timedOut = true
251
252 // The timeout writer has not been used by the inner handler.
253 // We can safely timeout the HTTP request by sending by a timeout
254 // handler
255 if !tw.wroteHeader && !tw.hijacked {
256 tw.w.WriteHeader(http.StatusGatewayTimeout)
257 enc := json.NewEncoder(tw.w)
258 enc.Encode(&err.ErrStatus)
259 } else {
260 // The timeout writer has been used by the inner handler. There is
261 // no way to timeout the HTTP request at the point. We have to shutdown
262 // the connection for HTTP1 or reset stream for HTTP2.
263 //
264 // Note from the golang's docs:
265 // If ServeHTTP panics, the server (the caller of ServeHTTP) assumes
266 // that the effect of the panic was isolated to the active request.
267 // It recovers the panic, logs a stack trace to the server error log,
268 // and either closes the network connection or sends an HTTP/2
269 // RST_STREAM, depending on the HTTP protocol. To abort a handler so
270 // the client sees an interrupted response but the server doesn't log
271 // an error, panic with the value ErrAbortHandler.
272 //
273 // We are throwing http.ErrAbortHandler deliberately so that a client is notified and to suppress a not helpful stacktrace in the logs
274 panic(http.ErrAbortHandler)
275 }
276 }
277
278 func (tw *baseTimeoutWriter) CloseNotify() <-chan bool {
279 tw.mu.Lock()
280 defer tw.mu.Unlock()
281
282 if tw.timedOut {
283 done := make(chan bool)
284 close(done)
285 return done
286 }
287
288 // the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
289 // http.CloseNotifier if the inner object (tw.w) implements http.CloseNotifier.
290 return tw.w.(http.CloseNotifier).CloseNotify()
291 }
292
293 func (tw *baseTimeoutWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
294 tw.mu.Lock()
295 defer tw.mu.Unlock()
296
297 if tw.timedOut {
298 return nil, nil, http.ErrHandlerTimeout
299 }
300
301 // the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
302 // http.Hijacker if the inner object (tw.w) implements http.Hijacker.
303 conn, rw, err := tw.w.(http.Hijacker).Hijack()
304 if err == nil {
305 tw.hijacked = true
306 }
307 return conn, rw, err
308 }