Debian Codegraph @

pkg/endpoints/handlers/finisher/finisher.go

raw
1 /*
2 Copyright 2021 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package finisher
18
19 import (
20 "context"
21 "fmt"
22 "net/http"
23 goruntime "runtime"
24 "time"
25
26 "k8s.io/apimachinery/pkg/api/errors"
27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28 "k8s.io/apimachinery/pkg/runtime"
29 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
30 "k8s.io/apiserver/pkg/endpoints/metrics"
31 "k8s.io/klog/v2"
32 )
33
34 // ResultFunc is a function that returns a rest result and can be run in a goroutine
35 type ResultFunc func() (runtime.Object, error)
36
37 // result stores the return values or panic from a ResultFunc function
38 type result struct {
39 // object stores the response returned by the ResultFunc function
40 object runtime.Object
41 // err stores the error returned by the ResultFunc function
42 err error
43 // reason stores the reason from a panic thrown by the ResultFunc function
44 reason interface{}
45 }
46
47 // Return processes the result returned by a ResultFunc function
48 func (r *result) Return() (runtime.Object, error) {
49 switch {
50 case r.reason != nil:
51 // panic has higher precedence, the goroutine executing ResultFunc has panic'd,
52 // so propagate a panic to the caller.
53 panic(r.reason)
54 case r.err != nil:
55 return nil, r.err
56 default:
57 // if we are here, it means neither a panic, nor an error
58 if status, ok := r.object.(*metav1.Status); ok {
59 // An api.Status object with status != success is considered an "error",
60 // which interrupts the normal response flow.
61 if status.Status != metav1.StatusSuccess {
62 return nil, errors.FromObject(status)
63 }
64 }
65 return r.object, nil
66 }
67 }
68
69 // PostTimeoutLoggerFunc is a function that can be used to log the result returned
70 // by a ResultFunc after the request had timed out.
71 // timedOutAt is the time the request had been timed out.
72 // r is the result returned by the child goroutine.
73 type PostTimeoutLoggerFunc func(timedOutAt time.Time, r *result)
74
75 const (
76 // how much time the post-timeout receiver goroutine will wait for the sender
77 // (child goroutine executing ResultFunc) to send a result after the request.
78 // had timed out.
79 postTimeoutLoggerWait = 5 * time.Minute
80 )
81
82 // FinishRequest makes a given ResultFunc asynchronous and handles errors returned by the response.
83 func FinishRequest(ctx context.Context, fn ResultFunc) (runtime.Object, error) {
84 return finishRequest(ctx, fn, postTimeoutLoggerWait, logPostTimeoutResult)
85 }
86
87 func finishRequest(ctx context.Context, fn ResultFunc, postTimeoutWait time.Duration, postTimeoutLogger PostTimeoutLoggerFunc) (runtime.Object, error) {
88 // the channel needs to be buffered since the post-timeout receiver goroutine
89 // waits up to 5 minutes for the child goroutine to return.
90 resultCh := make(chan *result, 1)
91
92 go func() {
93 result := &result{}
94
95 // panics don't cross goroutine boundaries, so we have to handle ourselves
96 defer func() {
97 reason := recover()
98 if reason != nil {
99 // do not wrap the sentinel ErrAbortHandler panic value
100 if reason != http.ErrAbortHandler {
101 // Same as stdlib http server code. Manually allocate stack
102 // trace buffer size to prevent excessively large logs
103 const size = 64 << 10
104 buf := make([]byte, size)
105 buf = buf[:goruntime.Stack(buf, false)]
106 reason = fmt.Sprintf("%v\n%s", reason, buf)
107 }
108
109 // store the panic reason into the result.
110 result.reason = reason
111 }
112
113 // Propagate the result to the parent goroutine
114 resultCh <- result
115 }()
116
117 if object, err := fn(); err != nil {
118 result.err = err
119 } else {
120 result.object = object
121 }
122 }()
123
124 select {
125 case result := <-resultCh:
126 return result.Return()
127 case <-ctx.Done():
128 // we are going to send a timeout response to the caller, but the asynchronous goroutine
129 // (sender) is still executing the ResultFunc function.
130 // kick off a goroutine (receiver) here to wait for the sender (goroutine executing ResultFunc)
131 // to send the result and then log details of the result.
132 defer func() {
133 go func() {
134 timedOutAt := time.Now()
135
136 var result *result
137 select {
138 case result = <-resultCh:
139 case <-time.After(postTimeoutWait):
140 // we will not wait forever, if we are here then we know that some sender
141 // goroutines are taking longer than postTimeoutWait.
142 }
143 postTimeoutLogger(timedOutAt, result)
144 }()
145 }()
146 return nil, errors.NewTimeoutError(fmt.Sprintf("request did not complete within requested timeout - %s", ctx.Err()), 0)
147 }
148 }
149
150 // logPostTimeoutResult logs a panic or an error from the result that the sender (goroutine that is
151 // executing the ResultFunc function) has sent to the receiver after the request had timed out.
152 // timedOutAt is the time the request had been timed out
153 func logPostTimeoutResult(timedOutAt time.Time, r *result) {
154 if r == nil {
155 // we are using r == nil to indicate that the child goroutine never returned a result.
156 metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceRestHandler, metrics.PostTimeoutHandlerPending)
157 klog.Errorf("FinishRequest: post-timeout activity, waited for %s, child goroutine has not returned yet", time.Since(timedOutAt))
158 return
159 }
160
161 var status string
162 switch {
163 case r.reason != nil:
164 // a non empty reason inside a result object indicates that there was a panic.
165 status = metrics.PostTimeoutHandlerPanic
166 case r.err != nil:
167 status = metrics.PostTimeoutHandlerError
168 default:
169 status = metrics.PostTimeoutHandlerOK
170 }
171
172 metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceRestHandler, status)
173 err := fmt.Errorf("FinishRequest: post-timeout activity - time-elapsed: %s, panicked: %t, err: %v, panic-reason: %v",
174 time.Since(timedOutAt), r.reason != nil, r.err, r.reason)
175 utilruntime.HandleError(err)
176 }