| 1 |
/* |
| 2 |
Copyright 2021 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package finisher |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"net/http" |
| 23 |
goruntime "runtime" |
| 24 |
"time" |
| 25 |
|
| 26 |
"k8s.io/apimachinery/pkg/api/errors" |
| 27 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 28 |
"k8s.io/apimachinery/pkg/runtime" |
| 29 |
utilruntime "k8s.io/apimachinery/pkg/util/runtime" |
| 30 |
"k8s.io/apiserver/pkg/endpoints/metrics" |
| 31 |
"k8s.io/klog/v2" |
| 32 |
) |
| 33 |
|
| 34 |
// ResultFunc is a function that returns a rest result and can be run in a goroutine |
| 35 |
type ResultFunc func() (runtime.Object, error) |
| 36 |
|
| 37 |
// result stores the return values or panic from a ResultFunc function |
| 38 |
type result struct { |
| 39 |
// object stores the response returned by the ResultFunc function |
| 40 |
object runtime.Object |
| 41 |
// err stores the error returned by the ResultFunc function |
| 42 |
err error |
| 43 |
// reason stores the reason from a panic thrown by the ResultFunc function |
| 44 |
reason interface{} |
| 45 |
} |
| 46 |
|
| 47 |
// Return processes the result returned by a ResultFunc function |
| 48 |
func (r *result) Return() (runtime.Object, error) { |
| 49 |
switch { |
| 50 |
case r.reason != nil: |
| 51 |
// panic has higher precedence, the goroutine executing ResultFunc has panic'd, |
| 52 |
// so propagate a panic to the caller. |
| 53 |
panic(r.reason) |
| 54 |
case r.err != nil: |
| 55 |
return nil, r.err |
| 56 |
default: |
| 57 |
// if we are here, it means neither a panic, nor an error |
| 58 |
if status, ok := r.object.(*metav1.Status); ok { |
| 59 |
// An api.Status object with status != success is considered an "error", |
| 60 |
// which interrupts the normal response flow. |
| 61 |
if status.Status != metav1.StatusSuccess { |
| 62 |
return nil, errors.FromObject(status) |
| 63 |
} |
| 64 |
} |
| 65 |
return r.object, nil |
| 66 |
} |
| 67 |
} |
| 68 |
|
| 69 |
// PostTimeoutLoggerFunc is a function that can be used to log the result returned |
| 70 |
// by a ResultFunc after the request had timed out. |
| 71 |
// timedOutAt is the time the request had been timed out. |
| 72 |
// r is the result returned by the child goroutine. |
| 73 |
type PostTimeoutLoggerFunc func(timedOutAt time.Time, r *result) |
| 74 |
|
| 75 |
const ( |
| 76 |
// how much time the post-timeout receiver goroutine will wait for the sender |
| 77 |
// (child goroutine executing ResultFunc) to send a result after the request. |
| 78 |
// had timed out. |
| 79 |
postTimeoutLoggerWait = 5 * time.Minute |
| 80 |
) |
| 81 |
|
| 82 |
// FinishRequest makes a given ResultFunc asynchronous and handles errors returned by the response. |
| 83 |
func FinishRequest(ctx context.Context, fn ResultFunc) (runtime.Object, error) { |
| 84 |
return finishRequest(ctx, fn, postTimeoutLoggerWait, logPostTimeoutResult) |
| 85 |
} |
| 86 |
|
| 87 |
func finishRequest(ctx context.Context, fn ResultFunc, postTimeoutWait time.Duration, postTimeoutLogger PostTimeoutLoggerFunc) (runtime.Object, error) { |
| 88 |
// the channel needs to be buffered since the post-timeout receiver goroutine |
| 89 |
// waits up to 5 minutes for the child goroutine to return. |
| 90 |
resultCh := make(chan *result, 1) |
| 91 |
|
| 92 |
go func() { |
| 93 |
result := &result{} |
| 94 |
|
| 95 |
// panics don't cross goroutine boundaries, so we have to handle ourselves |
| 96 |
defer func() { |
| 97 |
reason := recover() |
| 98 |
if reason != nil { |
| 99 |
// do not wrap the sentinel ErrAbortHandler panic value |
| 100 |
if reason != http.ErrAbortHandler { |
| 101 |
// Same as stdlib http server code. Manually allocate stack |
| 102 |
// trace buffer size to prevent excessively large logs |
| 103 |
const size = 64 << 10 |
| 104 |
buf := make([]byte, size) |
| 105 |
buf = buf[:goruntime.Stack(buf, false)] |
| 106 |
reason = fmt.Sprintf("%v\n%s", reason, buf) |
| 107 |
} |
| 108 |
|
| 109 |
// store the panic reason into the result. |
| 110 |
result.reason = reason |
| 111 |
} |
| 112 |
|
| 113 |
// Propagate the result to the parent goroutine |
| 114 |
resultCh <- result |
| 115 |
}() |
| 116 |
|
| 117 |
if object, err := fn(); err != nil { |
| 118 |
result.err = err |
| 119 |
} else { |
| 120 |
result.object = object |
| 121 |
} |
| 122 |
}() |
| 123 |
|
| 124 |
select { |
| 125 |
case result := <-resultCh: |
| 126 |
return result.Return() |
| 127 |
case <-ctx.Done(): |
| 128 |
// we are going to send a timeout response to the caller, but the asynchronous goroutine |
| 129 |
// (sender) is still executing the ResultFunc function. |
| 130 |
// kick off a goroutine (receiver) here to wait for the sender (goroutine executing ResultFunc) |
| 131 |
// to send the result and then log details of the result. |
| 132 |
defer func() { |
| 133 |
go func() { |
| 134 |
timedOutAt := time.Now() |
| 135 |
|
| 136 |
var result *result |
| 137 |
select { |
| 138 |
case result = <-resultCh: |
| 139 |
case <-time.After(postTimeoutWait): |
| 140 |
// we will not wait forever, if we are here then we know that some sender |
| 141 |
// goroutines are taking longer than postTimeoutWait. |
| 142 |
} |
| 143 |
postTimeoutLogger(timedOutAt, result) |
| 144 |
}() |
| 145 |
}() |
| 146 |
return nil, errors.NewTimeoutError(fmt.Sprintf("request did not complete within requested timeout - %s", ctx.Err()), 0) |
| 147 |
} |
| 148 |
} |
| 149 |
|
| 150 |
// logPostTimeoutResult logs a panic or an error from the result that the sender (goroutine that is |
| 151 |
// executing the ResultFunc function) has sent to the receiver after the request had timed out. |
| 152 |
// timedOutAt is the time the request had been timed out |
| 153 |
func logPostTimeoutResult(timedOutAt time.Time, r *result) { |
| 154 |
if r == nil { |
| 155 |
// we are using r == nil to indicate that the child goroutine never returned a result. |
| 156 |
metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceRestHandler, metrics.PostTimeoutHandlerPending) |
| 157 |
klog.Errorf("FinishRequest: post-timeout activity, waited for %s, child goroutine has not returned yet", time.Since(timedOutAt)) |
| 158 |
return |
| 159 |
} |
| 160 |
|
| 161 |
var status string |
| 162 |
switch { |
| 163 |
case r.reason != nil: |
| 164 |
// a non empty reason inside a result object indicates that there was a panic. |
| 165 |
status = metrics.PostTimeoutHandlerPanic |
| 166 |
case r.err != nil: |
| 167 |
status = metrics.PostTimeoutHandlerError |
| 168 |
default: |
| 169 |
status = metrics.PostTimeoutHandlerOK |
| 170 |
} |
| 171 |
|
| 172 |
metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceRestHandler, status) |
| 173 |
err := fmt.Errorf("FinishRequest: post-timeout activity - time-elapsed: %s, panicked: %t, err: %v, panic-reason: %v", |
| 174 |
time.Since(timedOutAt), r.reason != nil, r.err, r.reason) |
| 175 |
utilruntime.HandleError(err) |
| 176 |
} |