Debian Codegraph @

pkg/server/healthz/healthz.go

raw
1 /*
2 Copyright 2014 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package healthz
18
19 import (
20 "bytes"
21 "context"
22 "fmt"
23 "net/http"
24 "reflect"
25 "strings"
26 "sync"
27 "sync/atomic"
28 "time"
29
30 "k8s.io/apimachinery/pkg/util/sets"
31 "k8s.io/apimachinery/pkg/util/wait"
32 "k8s.io/apiserver/pkg/endpoints/metrics"
33 "k8s.io/apiserver/pkg/server/httplog"
34 "k8s.io/component-base/metrics/prometheus/slis"
35 "k8s.io/klog/v2"
36 )
37
38 const DefaultHealthzPath = "/healthz"
39
40 // HealthChecker is a named healthz checker.
41 type HealthChecker interface {
42 Name() string
43 Check(req *http.Request) error
44 }
45
46 // PingHealthz returns true automatically when checked
47 var PingHealthz HealthChecker = ping{}
48
49 // ping implements the simplest possible healthz checker.
50 type ping struct{}
51
52 func (ping) Name() string {
53 return "ping"
54 }
55
56 // PingHealthz is a health check that returns true.
57 func (ping) Check(_ *http.Request) error {
58 return nil
59 }
60
61 // LogHealthz returns true if logging is not blocked
62 var LogHealthz HealthChecker = &log{}
63
64 type log struct {
65 startOnce sync.Once
66 lastVerified atomic.Value
67 }
68
69 func (l *log) Name() string {
70 return "log"
71 }
72
73 func (l *log) Check(_ *http.Request) error {
74 l.startOnce.Do(func() {
75 l.lastVerified.Store(time.Now())
76 go wait.Forever(func() {
77 klog.Flush()
78 l.lastVerified.Store(time.Now())
79 }, time.Minute)
80 })
81
82 lastVerified := l.lastVerified.Load().(time.Time)
83 if time.Since(lastVerified) < (2 * time.Minute) {
84 return nil
85 }
86 return fmt.Errorf("logging blocked")
87 }
88
89 type cacheSyncWaiter interface {
90 WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool
91 }
92
93 type informerSync struct {
94 cacheSyncWaiter cacheSyncWaiter
95 }
96
97 var _ HealthChecker = &informerSync{}
98
99 // NewInformerSyncHealthz returns a new HealthChecker that will pass only if all informers in the given cacheSyncWaiter sync.
100 func NewInformerSyncHealthz(cacheSyncWaiter cacheSyncWaiter) HealthChecker {
101 return &informerSync{
102 cacheSyncWaiter: cacheSyncWaiter,
103 }
104 }
105
106 func (i *informerSync) Name() string {
107 return "informer-sync"
108 }
109
110 type shutdown struct {
111 stopCh <-chan struct{}
112 }
113
114 // NewShutdownHealthz returns a new HealthChecker that will fail if the embedded channel is closed.
115 // This is intended to allow for graceful shutdown sequences.
116 func NewShutdownHealthz(stopCh <-chan struct{}) HealthChecker {
117 return &shutdown{stopCh}
118 }
119
120 func (s *shutdown) Name() string {
121 return "shutdown"
122 }
123
124 func (s *shutdown) Check(req *http.Request) error {
125 select {
126 case <-s.stopCh:
127 return fmt.Errorf("process is shutting down")
128 default:
129 }
130 return nil
131 }
132
133 func (i *informerSync) Check(_ *http.Request) error {
134 stopCh := make(chan struct{})
135 // Close stopCh to force checking if informers are synced now.
136 close(stopCh)
137
138 informersByStarted := make(map[bool][]string)
139 for informerType, started := range i.cacheSyncWaiter.WaitForCacheSync(stopCh) {
140 informersByStarted[started] = append(informersByStarted[started], informerType.String())
141 }
142
143 if notStarted := informersByStarted[false]; len(notStarted) > 0 {
144 return fmt.Errorf("%d informers not started yet: %v", len(notStarted), notStarted)
145 }
146 return nil
147 }
148
149 // NamedCheck returns a healthz checker for the given name and function.
150 func NamedCheck(name string, check func(r *http.Request) error) HealthChecker {
151 return &healthzCheck{name, check}
152 }
153
154 // InstallHandler registers handlers for health checking on the path
155 // "/healthz" to mux. *All handlers* for mux must be specified in
156 // exactly one call to InstallHandler. Calling InstallHandler more
157 // than once for the same mux will result in a panic.
158 func InstallHandler(mux mux, checks ...HealthChecker) {
159 InstallPathHandler(mux, DefaultHealthzPath, checks...)
160 }
161
162 // InstallReadyzHandler registers handlers for health checking on the path
163 // "/readyz" to mux. *All handlers* for mux must be specified in
164 // exactly one call to InstallHandler. Calling InstallHandler more
165 // than once for the same mux will result in a panic.
166 func InstallReadyzHandler(mux mux, checks ...HealthChecker) {
167 InstallPathHandler(mux, "/readyz", checks...)
168 }
169
170 // InstallLivezHandler registers handlers for liveness checking on the path
171 // "/livez" to mux. *All handlers* for mux must be specified in
172 // exactly one call to InstallHandler. Calling InstallHandler more
173 // than once for the same mux will result in a panic.
174 func InstallLivezHandler(mux mux, checks ...HealthChecker) {
175 InstallPathHandler(mux, "/livez", checks...)
176 }
177
178 // InstallPathHandler registers handlers for health checking on
179 // a specific path to mux. *All handlers* for the path must be
180 // specified in exactly one call to InstallPathHandler. Calling
181 // InstallPathHandler more than once for the same path and mux will
182 // result in a panic.
183 func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
184 InstallPathHandlerWithHealthyFunc(mux, path, nil, checks...)
185 }
186
187 // InstallPathHandlerWithHealthyFunc is like InstallPathHandler, but calls firstTimeHealthy exactly once
188 // when the handler succeeds for the first time.
189 func InstallPathHandlerWithHealthyFunc(mux mux, path string, firstTimeHealthy func(), checks ...HealthChecker) {
190 if len(checks) == 0 {
191 klog.V(5).Info("No default health checks specified. Installing the ping handler.")
192 checks = []HealthChecker{PingHealthz}
193 }
194
195 klog.V(5).Infof("Installing health checkers for (%v): %v", path, formatQuoted(checkerNames(checks...)...))
196
197 name := strings.Split(strings.TrimPrefix(path, "/"), "/")[0]
198 mux.Handle(path,
199 metrics.InstrumentHandlerFunc("GET",
200 /* group = */ "",
201 /* version = */ "",
202 /* resource = */ "",
203 /* subresource = */ path,
204 /* scope = */ "",
205 /* component = */ "",
206 /* deprecated */ false,
207 /* removedRelease */ "",
208 handleRootHealth(name, firstTimeHealthy, checks...)))
209 for _, check := range checks {
210 mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check))
211 }
212 }
213
214 // mux is an interface describing the methods InstallHandler requires.
215 type mux interface {
216 Handle(pattern string, handler http.Handler)
217 }
218
219 // healthzCheck implements HealthChecker on an arbitrary name and check function.
220 type healthzCheck struct {
221 name string
222 check func(r *http.Request) error
223 }
224
225 var _ HealthChecker = &healthzCheck{}
226
227 func (c *healthzCheck) Name() string {
228 return c.name
229 }
230
231 func (c *healthzCheck) Check(r *http.Request) error {
232 return c.check(r)
233 }
234
235 // getExcludedChecks extracts the health check names to be excluded from the query param
236 func getExcludedChecks(r *http.Request) sets.String {
237 checks, found := r.URL.Query()["exclude"]
238 if found {
239 return sets.NewString(checks...)
240 }
241 return sets.NewString()
242 }
243
244 // handleRootHealth returns an http.HandlerFunc that serves the provided checks.
245 func handleRootHealth(name string, firstTimeHealthy func(), checks ...HealthChecker) http.HandlerFunc {
246 var notifyOnce sync.Once
247 return func(w http.ResponseWriter, r *http.Request) {
248 excluded := getExcludedChecks(r)
249 // failedVerboseLogOutput is for output to the log. It indicates detailed failed output information for the log.
250 var failedVerboseLogOutput bytes.Buffer
251 var failedChecks []string
252 var individualCheckOutput bytes.Buffer
253 for _, check := range checks {
254 // no-op the check if we've specified we want to exclude the check
255 if excluded.Has(check.Name()) {
256 excluded.Delete(check.Name())
257 fmt.Fprintf(&individualCheckOutput, "[+]%s excluded: ok\n", check.Name())
258 continue
259 }
260 if err := check.Check(r); err != nil {
261 slis.ObserveHealthcheck(context.Background(), check.Name(), name, slis.Error)
262 // don't include the error since this endpoint is public. If someone wants more detail
263 // they should have explicit permission to the detailed checks.
264 fmt.Fprintf(&individualCheckOutput, "[-]%s failed: reason withheld\n", check.Name())
265 // but we do want detailed information for our log
266 fmt.Fprintf(&failedVerboseLogOutput, "[-]%s failed: %v\n", check.Name(), err)
267 failedChecks = append(failedChecks, check.Name())
268 } else {
269 slis.ObserveHealthcheck(context.Background(), check.Name(), name, slis.Success)
270 fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", check.Name())
271 }
272 }
273 if excluded.Len() > 0 {
274 fmt.Fprintf(&individualCheckOutput, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
275 klog.V(6).Infof("cannot exclude some health checks, no health checks are installed matching %s",
276 formatQuoted(excluded.List()...))
277 }
278 // always be verbose on failure
279 if len(failedChecks) > 0 {
280 klog.V(2).Infof("%s check failed: %s\n%v", strings.Join(failedChecks, ","), name, failedVerboseLogOutput.String())
281 httplog.SetStacktracePredicate(r.Context(), func(int) bool { return false })
282 http.Error(w, fmt.Sprintf("%s%s check failed", individualCheckOutput.String(), name), http.StatusInternalServerError)
283 return
284 }
285
286 // signal first time this is healthy
287 if firstTimeHealthy != nil {
288 notifyOnce.Do(firstTimeHealthy)
289 }
290
291 w.Header().Set("Content-Type", "text/plain; charset=utf-8")
292 w.Header().Set("X-Content-Type-Options", "nosniff")
293 if _, found := r.URL.Query()["verbose"]; !found {
294 fmt.Fprint(w, "ok")
295 return
296 }
297
298 individualCheckOutput.WriteTo(w)
299 fmt.Fprintf(w, "%s check passed\n", name)
300 }
301 }
302
303 // adaptCheckToHandler returns an http.HandlerFunc that serves the provided checks.
304 func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc {
305 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
306 err := c(r)
307 if err != nil {
308 http.Error(w, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
309 } else {
310 fmt.Fprint(w, "ok")
311 }
312 })
313 }
314
315 // checkerNames returns the names of the checks in the same order as passed in.
316 func checkerNames(checks ...HealthChecker) []string {
317 // accumulate the names of checks for printing them out.
318 checkerNames := make([]string, 0, len(checks))
319 for _, check := range checks {
320 checkerNames = append(checkerNames, check.Name())
321 }
322 return checkerNames
323 }
324
325 // formatQuoted returns a formatted string of the health check names,
326 // preserving the order passed in.
327 func formatQuoted(names ...string) string {
328 quoted := make([]string, 0, len(names))
329 for _, name := range names {
330 quoted = append(quoted, fmt.Sprintf("%q", name))
331 }
332 return strings.Join(quoted, ",")
333 }