Debian Codegraph @

pkg/server/lifecycle_signals.go

raw
1 /*
2 Copyright 2021 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package server
18
19 import (
20 "sync"
21 )
22
23 /*
24 We make an attempt here to identify the events that take place during
25 lifecycle of the apiserver.
26
27 We also identify each event with a name so we can refer to it.
28
29 Events:
30 - ShutdownInitiated: KILL signal received
31 - AfterShutdownDelayDuration: shutdown delay duration has passed
32 - InFlightRequestsDrained: all in flight request(s) have been drained
33 - HasBeenReady is signaled when the readyz endpoint succeeds for the first time
34
35 The following is a sequence of shutdown events that we expect to see with
36 'ShutdownSendRetryAfter' = false:
37
38 T0: ShutdownInitiated: KILL signal received
39 - /readyz starts returning red
40 - run pre shutdown hooks
41
42 T0+70s: AfterShutdownDelayDuration: shutdown delay duration has passed
43 - the default value of 'ShutdownDelayDuration' is '70s'
44 - it's time to initiate shutdown of the HTTP Server, server.Shutdown is invoked
45 - as a consequene, the Close function has is called for all listeners
46 - the HTTP Server stops listening immediately
47 - any new request arriving on a new TCP socket is denied with
48 a network error similar to 'connection refused'
49 - the HTTP Server waits gracefully for existing requests to complete
50 up to '60s' (dictated by ShutdownTimeout)
51 - active long running requests will receive a GOAWAY.
52
53 T0+70s: HTTPServerStoppedListening:
54 - this event is signaled when the HTTP Server has stopped listening
55 which is immediately after server.Shutdown has been invoked
56
57 T0 + 70s + up-to 60s: InFlightRequestsDrained: existing in flight requests have been drained
58 - long running requests are outside of this scope
59 - up-to 60s: the default value of 'ShutdownTimeout' is 60s, this means that
60 any request in flight has a hard timeout of 60s.
61 - it's time to call 'Shutdown' on the audit events since all
62 in flight request(s) have drained.
63
64
65 The following is a sequence of shutdown events that we expect to see with
66 'ShutdownSendRetryAfter' = true:
67
68 T0: ShutdownInitiated: KILL signal received
69 - /readyz starts returning red
70 - run pre shutdown hooks
71
72 T0+70s: AfterShutdownDelayDuration: shutdown delay duration has passed
73 - the default value of 'ShutdownDelayDuration' is '70s'
74 - the HTTP Server will continue to listen
75 - the apiserver is not accepting new request(s)
76 - it includes new request(s) on a new or an existing TCP connection
77 - new request(s) arriving after this point are replied with a 429
78 and the response headers: 'Retry-After: 1` and 'Connection: close'
79 - note: these new request(s) will not show up in audit logs
80
81 T0 + 70s + up to 60s: InFlightRequestsDrained: existing in flight requests have been drained
82 - long running requests are outside of this scope
83 - up to 60s: the default value of 'ShutdownTimeout' is 60s, this means that
84 any request in flight has a hard timeout of 60s.
85 - server.Shutdown is called, the HTTP Server stops listening immediately
86 - the HTTP Server waits gracefully for existing requests to complete
87 up to '2s' (it's hard coded right now)
88 */
89
90 // lifecycleSignal encapsulates a named apiserver event
91 type lifecycleSignal interface {
92 // Signal signals the event, indicating that the event has occurred.
93 // Signal is idempotent, once signaled the event stays signaled and
94 // it immediately unblocks any goroutine waiting for this event.
95 Signal()
96
97 // Signaled returns a channel that is closed when the underlying event
98 // has been signaled. Successive calls to Signaled return the same value.
99 Signaled() <-chan struct{}
100
101 // Name returns the name of the signal, useful for logging.
102 Name() string
103 }
104
105 // lifecycleSignals provides an abstraction of the events that
106 // transpire during the lifecycle of the apiserver. This abstraction makes it easy
107 // for us to write unit tests that can verify expected graceful termination behavior.
108 //
109 // GenericAPIServer can use these to either:
110 // - signal that a particular termination event has transpired
111 // - wait for a designated termination event to transpire and do some action.
112 type lifecycleSignals struct {
113 // ShutdownInitiated event is signaled when an apiserver shutdown has been initiated.
114 // It is signaled when the `stopCh` provided by the main goroutine
115 // receives a KILL signal and is closed as a consequence.
116 ShutdownInitiated lifecycleSignal
117
118 // AfterShutdownDelayDuration event is signaled as soon as ShutdownDelayDuration
119 // has elapsed since the ShutdownInitiated event.
120 // ShutdownDelayDuration allows the apiserver to delay shutdown for some time.
121 AfterShutdownDelayDuration lifecycleSignal
122
123 // PreShutdownHooksStopped event is signaled when all registered
124 // preshutdown hook(s) have finished running.
125 PreShutdownHooksStopped lifecycleSignal
126
127 // NotAcceptingNewRequest event is signaled when the server is no
128 // longer accepting any new request, from this point on any new
129 // request will receive an error.
130 NotAcceptingNewRequest lifecycleSignal
131
132 // InFlightRequestsDrained event is signaled when the existing requests
133 // in flight have completed. This is used as signal to shut down the audit backends
134 InFlightRequestsDrained lifecycleSignal
135
136 // HTTPServerStoppedListening termination event is signaled when the
137 // HTTP Server has stopped listening to the underlying socket.
138 HTTPServerStoppedListening lifecycleSignal
139
140 // HasBeenReady is signaled when the readyz endpoint succeeds for the first time.
141 HasBeenReady lifecycleSignal
142
143 // MuxAndDiscoveryComplete is signaled when all known HTTP paths have been installed.
144 // It exists primarily to avoid returning a 404 response when a resource actually exists but we haven't installed the path to a handler.
145 // The actual logic is implemented by an APIServer using the generic server library.
146 MuxAndDiscoveryComplete lifecycleSignal
147 }
148
149 // ShuttingDown returns the lifecycle signal that is signaled when
150 // the server is not accepting any new requests.
151 // this is the lifecycle event that is exported to the request handler
152 // logic to indicate that the server is shutting down.
153 func (s lifecycleSignals) ShuttingDown() <-chan struct{} {
154 return s.NotAcceptingNewRequest.Signaled()
155 }
156
157 // newLifecycleSignals returns an instance of lifecycleSignals interface to be used
158 // to coordinate lifecycle of the apiserver
159 func newLifecycleSignals() lifecycleSignals {
160 return lifecycleSignals{
161 ShutdownInitiated: newNamedChannelWrapper("ShutdownInitiated"),
162 AfterShutdownDelayDuration: newNamedChannelWrapper("AfterShutdownDelayDuration"),
163 PreShutdownHooksStopped: newNamedChannelWrapper("PreShutdownHooksStopped"),
164 NotAcceptingNewRequest: newNamedChannelWrapper("NotAcceptingNewRequest"),
165 InFlightRequestsDrained: newNamedChannelWrapper("InFlightRequestsDrained"),
166 HTTPServerStoppedListening: newNamedChannelWrapper("HTTPServerStoppedListening"),
167 HasBeenReady: newNamedChannelWrapper("HasBeenReady"),
168 MuxAndDiscoveryComplete: newNamedChannelWrapper("MuxAndDiscoveryComplete"),
169 }
170 }
171
172 func newNamedChannelWrapper(name string) lifecycleSignal {
173 return &namedChannelWrapper{
174 name: name,
175 once: sync.Once{},
176 ch: make(chan struct{}),
177 }
178 }
179
180 type namedChannelWrapper struct {
181 name string
182 once sync.Once
183 ch chan struct{}
184 }
185
186 func (e *namedChannelWrapper) Signal() {
187 e.once.Do(func() {
188 close(e.ch)
189 })
190 }
191
192 func (e *namedChannelWrapper) Signaled() <-chan struct{} {
193 return e.ch
194 }
195
196 func (e *namedChannelWrapper) Name() string {
197 return e.name
198 }