| 1 |
/* |
| 2 |
Copyright 2021 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package server |
| 18 |
|
| 19 |
import ( |
| 20 |
"sync" |
| 21 |
) |
| 22 |
|
| 23 |
/* |
| 24 |
We make an attempt here to identify the events that take place during |
| 25 |
lifecycle of the apiserver. |
| 26 |
|
| 27 |
We also identify each event with a name so we can refer to it. |
| 28 |
|
| 29 |
Events: |
| 30 |
- ShutdownInitiated: KILL signal received |
| 31 |
- AfterShutdownDelayDuration: shutdown delay duration has passed |
| 32 |
- InFlightRequestsDrained: all in flight request(s) have been drained |
| 33 |
- HasBeenReady is signaled when the readyz endpoint succeeds for the first time |
| 34 |
|
| 35 |
The following is a sequence of shutdown events that we expect to see with |
| 36 |
'ShutdownSendRetryAfter' = false: |
| 37 |
|
| 38 |
T0: ShutdownInitiated: KILL signal received |
| 39 |
- /readyz starts returning red |
| 40 |
- run pre shutdown hooks |
| 41 |
|
| 42 |
T0+70s: AfterShutdownDelayDuration: shutdown delay duration has passed |
| 43 |
- the default value of 'ShutdownDelayDuration' is '70s' |
| 44 |
- it's time to initiate shutdown of the HTTP Server, server.Shutdown is invoked |
| 45 |
- as a consequene, the Close function has is called for all listeners |
| 46 |
- the HTTP Server stops listening immediately |
| 47 |
- any new request arriving on a new TCP socket is denied with |
| 48 |
a network error similar to 'connection refused' |
| 49 |
- the HTTP Server waits gracefully for existing requests to complete |
| 50 |
up to '60s' (dictated by ShutdownTimeout) |
| 51 |
- active long running requests will receive a GOAWAY. |
| 52 |
|
| 53 |
T0+70s: HTTPServerStoppedListening: |
| 54 |
- this event is signaled when the HTTP Server has stopped listening |
| 55 |
which is immediately after server.Shutdown has been invoked |
| 56 |
|
| 57 |
T0 + 70s + up-to 60s: InFlightRequestsDrained: existing in flight requests have been drained |
| 58 |
- long running requests are outside of this scope |
| 59 |
- up-to 60s: the default value of 'ShutdownTimeout' is 60s, this means that |
| 60 |
any request in flight has a hard timeout of 60s. |
| 61 |
- it's time to call 'Shutdown' on the audit events since all |
| 62 |
in flight request(s) have drained. |
| 63 |
|
| 64 |
|
| 65 |
The following is a sequence of shutdown events that we expect to see with |
| 66 |
'ShutdownSendRetryAfter' = true: |
| 67 |
|
| 68 |
T0: ShutdownInitiated: KILL signal received |
| 69 |
- /readyz starts returning red |
| 70 |
- run pre shutdown hooks |
| 71 |
|
| 72 |
T0+70s: AfterShutdownDelayDuration: shutdown delay duration has passed |
| 73 |
- the default value of 'ShutdownDelayDuration' is '70s' |
| 74 |
- the HTTP Server will continue to listen |
| 75 |
- the apiserver is not accepting new request(s) |
| 76 |
- it includes new request(s) on a new or an existing TCP connection |
| 77 |
- new request(s) arriving after this point are replied with a 429 |
| 78 |
and the response headers: 'Retry-After: 1` and 'Connection: close' |
| 79 |
- note: these new request(s) will not show up in audit logs |
| 80 |
|
| 81 |
T0 + 70s + up to 60s: InFlightRequestsDrained: existing in flight requests have been drained |
| 82 |
- long running requests are outside of this scope |
| 83 |
- up to 60s: the default value of 'ShutdownTimeout' is 60s, this means that |
| 84 |
any request in flight has a hard timeout of 60s. |
| 85 |
- server.Shutdown is called, the HTTP Server stops listening immediately |
| 86 |
- the HTTP Server waits gracefully for existing requests to complete |
| 87 |
up to '2s' (it's hard coded right now) |
| 88 |
*/ |
| 89 |
|
| 90 |
// lifecycleSignal encapsulates a named apiserver event |
| 91 |
type lifecycleSignal interface { |
| 92 |
// Signal signals the event, indicating that the event has occurred. |
| 93 |
// Signal is idempotent, once signaled the event stays signaled and |
| 94 |
// it immediately unblocks any goroutine waiting for this event. |
| 95 |
Signal() |
| 96 |
|
| 97 |
// Signaled returns a channel that is closed when the underlying event |
| 98 |
// has been signaled. Successive calls to Signaled return the same value. |
| 99 |
Signaled() <-chan struct{} |
| 100 |
|
| 101 |
// Name returns the name of the signal, useful for logging. |
| 102 |
Name() string |
| 103 |
} |
| 104 |
|
| 105 |
// lifecycleSignals provides an abstraction of the events that |
| 106 |
// transpire during the lifecycle of the apiserver. This abstraction makes it easy |
| 107 |
// for us to write unit tests that can verify expected graceful termination behavior. |
| 108 |
// |
| 109 |
// GenericAPIServer can use these to either: |
| 110 |
// - signal that a particular termination event has transpired |
| 111 |
// - wait for a designated termination event to transpire and do some action. |
| 112 |
type lifecycleSignals struct { |
| 113 |
// ShutdownInitiated event is signaled when an apiserver shutdown has been initiated. |
| 114 |
// It is signaled when the `stopCh` provided by the main goroutine |
| 115 |
// receives a KILL signal and is closed as a consequence. |
| 116 |
ShutdownInitiated lifecycleSignal |
| 117 |
|
| 118 |
// AfterShutdownDelayDuration event is signaled as soon as ShutdownDelayDuration |
| 119 |
// has elapsed since the ShutdownInitiated event. |
| 120 |
// ShutdownDelayDuration allows the apiserver to delay shutdown for some time. |
| 121 |
AfterShutdownDelayDuration lifecycleSignal |
| 122 |
|
| 123 |
// PreShutdownHooksStopped event is signaled when all registered |
| 124 |
// preshutdown hook(s) have finished running. |
| 125 |
PreShutdownHooksStopped lifecycleSignal |
| 126 |
|
| 127 |
// NotAcceptingNewRequest event is signaled when the server is no |
| 128 |
// longer accepting any new request, from this point on any new |
| 129 |
// request will receive an error. |
| 130 |
NotAcceptingNewRequest lifecycleSignal |
| 131 |
|
| 132 |
// InFlightRequestsDrained event is signaled when the existing requests |
| 133 |
// in flight have completed. This is used as signal to shut down the audit backends |
| 134 |
InFlightRequestsDrained lifecycleSignal |
| 135 |
|
| 136 |
// HTTPServerStoppedListening termination event is signaled when the |
| 137 |
// HTTP Server has stopped listening to the underlying socket. |
| 138 |
HTTPServerStoppedListening lifecycleSignal |
| 139 |
|
| 140 |
// HasBeenReady is signaled when the readyz endpoint succeeds for the first time. |
| 141 |
HasBeenReady lifecycleSignal |
| 142 |
|
| 143 |
// MuxAndDiscoveryComplete is signaled when all known HTTP paths have been installed. |
| 144 |
// It exists primarily to avoid returning a 404 response when a resource actually exists but we haven't installed the path to a handler. |
| 145 |
// The actual logic is implemented by an APIServer using the generic server library. |
| 146 |
MuxAndDiscoveryComplete lifecycleSignal |
| 147 |
} |
| 148 |
|
| 149 |
// ShuttingDown returns the lifecycle signal that is signaled when |
| 150 |
// the server is not accepting any new requests. |
| 151 |
// this is the lifecycle event that is exported to the request handler |
| 152 |
// logic to indicate that the server is shutting down. |
| 153 |
func (s lifecycleSignals) ShuttingDown() <-chan struct{} { |
| 154 |
return s.NotAcceptingNewRequest.Signaled() |
| 155 |
} |
| 156 |
|
| 157 |
// newLifecycleSignals returns an instance of lifecycleSignals interface to be used |
| 158 |
// to coordinate lifecycle of the apiserver |
| 159 |
func newLifecycleSignals() lifecycleSignals { |
| 160 |
return lifecycleSignals{ |
| 161 |
ShutdownInitiated: newNamedChannelWrapper("ShutdownInitiated"), |
| 162 |
AfterShutdownDelayDuration: newNamedChannelWrapper("AfterShutdownDelayDuration"), |
| 163 |
PreShutdownHooksStopped: newNamedChannelWrapper("PreShutdownHooksStopped"), |
| 164 |
NotAcceptingNewRequest: newNamedChannelWrapper("NotAcceptingNewRequest"), |
| 165 |
InFlightRequestsDrained: newNamedChannelWrapper("InFlightRequestsDrained"), |
| 166 |
HTTPServerStoppedListening: newNamedChannelWrapper("HTTPServerStoppedListening"), |
| 167 |
HasBeenReady: newNamedChannelWrapper("HasBeenReady"), |
| 168 |
MuxAndDiscoveryComplete: newNamedChannelWrapper("MuxAndDiscoveryComplete"), |
| 169 |
} |
| 170 |
} |
| 171 |
|
| 172 |
func newNamedChannelWrapper(name string) lifecycleSignal { |
| 173 |
return &namedChannelWrapper{ |
| 174 |
name: name, |
| 175 |
once: sync.Once{}, |
| 176 |
ch: make(chan struct{}), |
| 177 |
} |
| 178 |
} |
| 179 |
|
| 180 |
type namedChannelWrapper struct { |
| 181 |
name string |
| 182 |
once sync.Once |
| 183 |
ch chan struct{} |
| 184 |
} |
| 185 |
|
| 186 |
func (e *namedChannelWrapper) Signal() { |
| 187 |
e.once.Do(func() { |
| 188 |
close(e.ch) |
| 189 |
}) |
| 190 |
} |
| 191 |
|
| 192 |
func (e *namedChannelWrapper) Signaled() <-chan struct{} { |
| 193 |
return e.ch |
| 194 |
} |
| 195 |
|
| 196 |
func (e *namedChannelWrapper) Name() string { |
| 197 |
return e.name |
| 198 |
} |