Debian Codegraph @

pkg/endpoints/metrics/metrics.go

raw
1 /*
2 Copyright 2015 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package metrics
18
19 import (
20 "context"
21 "fmt"
22 "net/http"
23 "net/url"
24 "strconv"
25 "strings"
26 "sync"
27 "time"
28
29 restful "github.com/emicklei/go-restful/v3"
30
31 metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
32 "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
33 "k8s.io/apimachinery/pkg/runtime/schema"
34 "k8s.io/apimachinery/pkg/types"
35 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
36 utilsets "k8s.io/apimachinery/pkg/util/sets"
37 "k8s.io/apiserver/pkg/audit"
38 "k8s.io/apiserver/pkg/authentication/user"
39 "k8s.io/apiserver/pkg/endpoints/request"
40 "k8s.io/apiserver/pkg/endpoints/responsewriter"
41 compbasemetrics "k8s.io/component-base/metrics"
42 "k8s.io/component-base/metrics/legacyregistry"
43 )
44
45 // resettableCollector is the interface implemented by prometheus.MetricVec
46 // that can be used by Prometheus to collect metrics and reset their values.
47 type resettableCollector interface {
48 compbasemetrics.Registerable
49 Reset()
50 }
51
52 const (
53 APIServerComponent string = "apiserver"
54 OtherRequestMethod string = "other"
55 )
56
57 /*
58 * By default, all the following metrics are defined as falling under
59 * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
60 *
61 * Promoting the stability level of the metric is a responsibility of the component owner, since it
62 * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
63 * the metric stability policy.
64 */
65 var (
66 deprecatedRequestGauge = compbasemetrics.NewGaugeVec(
67 &compbasemetrics.GaugeOpts{
68 Subsystem: APIServerComponent,
69 Name: "requested_deprecated_apis",
70 Help: "Gauge of deprecated APIs that have been requested, broken out by API group, version, resource, subresource, and removed_release.",
71 StabilityLevel: compbasemetrics.STABLE,
72 },
73 []string{"group", "version", "resource", "subresource", "removed_release"},
74 )
75
76 // TODO(a-robinson): Add unit tests for the handling of these metrics once
77 // the upstream library supports it.
78 requestCounter = compbasemetrics.NewCounterVec(
79 &compbasemetrics.CounterOpts{
80 Subsystem: APIServerComponent,
81 Name: "request_total",
82 Help: "Counter of apiserver requests broken out for each verb, dry run value, group, version, resource, scope, component, and HTTP response code.",
83 StabilityLevel: compbasemetrics.STABLE,
84 },
85 []string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component", "code"},
86 )
87 longRunningRequestsGauge = compbasemetrics.NewGaugeVec(
88 &compbasemetrics.GaugeOpts{
89 Subsystem: APIServerComponent,
90 Name: "longrunning_requests",
91 Help: "Gauge of all active long-running apiserver requests broken out by verb, group, version, resource, scope and component. Not all requests are tracked this way.",
92 StabilityLevel: compbasemetrics.STABLE,
93 },
94 []string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
95 )
96 requestLatencies = compbasemetrics.NewHistogramVec(
97 &compbasemetrics.HistogramOpts{
98 Subsystem: APIServerComponent,
99 Name: "request_duration_seconds",
100 Help: "Response latency distribution in seconds for each verb, dry run value, group, version, resource, subresource, scope and component.",
101 // This metric is used for verifying api call latencies SLO,
102 // as well as tracking regressions in this aspects.
103 // Thus we customize buckets significantly, to empower both usecases.
104 Buckets: []float64{0.005, 0.025, 0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3,
105 4, 5, 6, 8, 10, 15, 20, 30, 45, 60},
106 StabilityLevel: compbasemetrics.STABLE,
107 },
108 []string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component"},
109 )
110 requestSloLatencies = compbasemetrics.NewHistogramVec(
111 &compbasemetrics.HistogramOpts{
112 Subsystem: APIServerComponent,
113 Name: "request_slo_duration_seconds",
114 Help: "Response latency distribution (not counting webhook duration and priority & fairness queue wait times) in seconds for each verb, group, version, resource, subresource, scope and component.",
115 // This metric is supplementary to the requestLatencies metric.
116 // It measures request duration excluding webhooks as they are mostly
117 // dependant on user configuration.
118 Buckets: []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3,
119 4, 5, 6, 8, 10, 15, 20, 30, 45, 60},
120 StabilityLevel: compbasemetrics.ALPHA,
121 DeprecatedVersion: "1.27.0",
122 },
123 []string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
124 )
125 requestSliLatencies = compbasemetrics.NewHistogramVec(
126 &compbasemetrics.HistogramOpts{
127 Subsystem: APIServerComponent,
128 Name: "request_sli_duration_seconds",
129 Help: "Response latency distribution (not counting webhook duration and priority & fairness queue wait times) in seconds for each verb, group, version, resource, subresource, scope and component.",
130 // This metric is supplementary to the requestLatencies metric.
131 // It measures request duration excluding webhooks as they are mostly
132 // dependant on user configuration.
133 Buckets: []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3,
134 4, 5, 6, 8, 10, 15, 20, 30, 45, 60},
135 StabilityLevel: compbasemetrics.ALPHA,
136 },
137 []string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
138 )
139 fieldValidationRequestLatencies = compbasemetrics.NewHistogramVec(
140 &compbasemetrics.HistogramOpts{
141 Name: "field_validation_request_duration_seconds",
142 Help: "Response latency distribution in seconds for each field validation value",
143 // This metric is supplementary to the requestLatencies metric.
144 // It measures request durations for the various field validation
145 // values.
146 Buckets: []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3,
147 4, 5, 6, 8, 10, 15, 20, 30, 45, 60},
148 StabilityLevel: compbasemetrics.ALPHA,
149 },
150 []string{"field_validation"},
151 )
152 responseSizes = compbasemetrics.NewHistogramVec(
153 &compbasemetrics.HistogramOpts{
154 Subsystem: APIServerComponent,
155 Name: "response_sizes",
156 Help: "Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component.",
157 // Use buckets ranging from 1000 bytes (1KB) to 10^9 bytes (1GB).
158 Buckets: compbasemetrics.ExponentialBuckets(1000, 10.0, 7),
159 StabilityLevel: compbasemetrics.STABLE,
160 },
161 []string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
162 )
163 // TLSHandshakeErrors is a number of requests dropped with 'TLS handshake error from' error
164 TLSHandshakeErrors = compbasemetrics.NewCounter(
165 &compbasemetrics.CounterOpts{
166 Subsystem: APIServerComponent,
167 Name: "tls_handshake_errors_total",
168 Help: "Number of requests dropped with 'TLS handshake error from' error",
169 StabilityLevel: compbasemetrics.ALPHA,
170 },
171 )
172 WatchEvents = compbasemetrics.NewCounterVec(
173 &compbasemetrics.CounterOpts{
174 Subsystem: APIServerComponent,
175 Name: "watch_events_total",
176 Help: "Number of events sent in watch clients",
177 StabilityLevel: compbasemetrics.ALPHA,
178 },
179 []string{"group", "version", "kind"},
180 )
181 WatchEventsSizes = compbasemetrics.NewHistogramVec(
182 &compbasemetrics.HistogramOpts{
183 Subsystem: APIServerComponent,
184 Name: "watch_events_sizes",
185 Help: "Watch event size distribution in bytes",
186 Buckets: compbasemetrics.ExponentialBuckets(1024, 2.0, 8), // 1K, 2K, 4K, 8K, ..., 128K.
187 StabilityLevel: compbasemetrics.ALPHA,
188 },
189 []string{"group", "version", "kind"},
190 )
191 // Because of volatility of the base metric this is pre-aggregated one. Instead of reporting current usage all the time
192 // it reports maximal usage during the last second.
193 currentInflightRequests = compbasemetrics.NewGaugeVec(
194 &compbasemetrics.GaugeOpts{
195 Subsystem: APIServerComponent,
196 Name: "current_inflight_requests",
197 Help: "Maximal number of currently used inflight request limit of this apiserver per request kind in last second.",
198 StabilityLevel: compbasemetrics.STABLE,
199 },
200 []string{"request_kind"},
201 )
202 currentInqueueRequests = compbasemetrics.NewGaugeVec(
203 &compbasemetrics.GaugeOpts{
204 Subsystem: APIServerComponent,
205 Name: "current_inqueue_requests",
206 Help: "Maximal number of queued requests in this apiserver per request kind in last second.",
207 StabilityLevel: compbasemetrics.ALPHA,
208 },
209 []string{"request_kind"},
210 )
211
212 requestTerminationsTotal = compbasemetrics.NewCounterVec(
213 &compbasemetrics.CounterOpts{
214 Subsystem: APIServerComponent,
215 Name: "request_terminations_total",
216 Help: "Number of requests which apiserver terminated in self-defense.",
217 StabilityLevel: compbasemetrics.ALPHA,
218 },
219 []string{"verb", "group", "version", "resource", "subresource", "scope", "component", "code"},
220 )
221
222 apiSelfRequestCounter = compbasemetrics.NewCounterVec(
223 &compbasemetrics.CounterOpts{
224 Subsystem: APIServerComponent,
225 Name: "selfrequest_total",
226 Help: "Counter of apiserver self-requests broken out for each verb, API resource and subresource.",
227 StabilityLevel: compbasemetrics.ALPHA,
228 },
229 []string{"verb", "resource", "subresource"},
230 )
231
232 requestFilterDuration = compbasemetrics.NewHistogramVec(
233 &compbasemetrics.HistogramOpts{
234 Subsystem: APIServerComponent,
235 Name: "request_filter_duration_seconds",
236 Help: "Request filter latency distribution in seconds, for each filter type",
237 Buckets: []float64{0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 5.0, 10.0, 15.0, 30.0},
238 StabilityLevel: compbasemetrics.ALPHA,
239 },
240 []string{"filter"},
241 )
242
243 // requestAbortsTotal is a number of aborted requests with http.ErrAbortHandler
244 requestAbortsTotal = compbasemetrics.NewCounterVec(
245 &compbasemetrics.CounterOpts{
246 Subsystem: APIServerComponent,
247 Name: "request_aborts_total",
248 Help: "Number of requests which apiserver aborted possibly due to a timeout, for each group, version, verb, resource, subresource and scope",
249 StabilityLevel: compbasemetrics.ALPHA,
250 },
251 []string{"verb", "group", "version", "resource", "subresource", "scope"},
252 )
253
254 // requestPostTimeoutTotal tracks the activity of the executing request handler after the associated request
255 // has been timed out by the apiserver.
256 // source: the name of the handler that is recording this metric. Currently, we have two:
257 // - timeout-handler: the "executing" handler returns after the timeout filter times out the request.
258 // - rest-handler: the "executing" handler returns after the rest layer times out the request.
259 // status: whether the handler panicked or threw an error, possible values:
260 // - 'panic': the handler panicked
261 // - 'error': the handler return an error
262 // - 'ok': the handler returned a result (no error and no panic)
263 // - 'pending': the handler is still running in the background and it did not return
264 // within the wait threshold.
265 requestPostTimeoutTotal = compbasemetrics.NewCounterVec(
266 &compbasemetrics.CounterOpts{
267 Subsystem: APIServerComponent,
268 Name: "request_post_timeout_total",
269 Help: "Tracks the activity of the request handlers after the associated requests have been timed out by the apiserver",
270 StabilityLevel: compbasemetrics.ALPHA,
271 },
272 []string{"source", "status"},
273 )
274
275 requestTimestampComparisonDuration = compbasemetrics.NewHistogramVec(
276 &compbasemetrics.HistogramOpts{
277 Subsystem: APIServerComponent,
278 Name: "request_timestamp_comparison_time",
279 Help: "Time taken for comparison of old vs new objects in UPDATE or PATCH requests",
280 Buckets: []float64{0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 5.0},
281 StabilityLevel: compbasemetrics.ALPHA,
282 },
283 // Path the code takes to reach a conclusion:
284 // i.e. unequalObjectsFast, unequalObjectsSlow, equalObjectsSlow
285 []string{"code_path"},
286 )
287
288 watchListLatencies = compbasemetrics.NewHistogramVec(
289 &compbasemetrics.HistogramOpts{
290 Subsystem: APIServerComponent,
291 Name: "watch_list_duration_seconds",
292 Help: "Response latency distribution in seconds for watch list requests broken by group, version, resource and scope.",
293 Buckets: []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 2, 4, 6, 8, 10, 15, 20, 30, 45, 60},
294 StabilityLevel: compbasemetrics.ALPHA,
295 },
296 []string{"group", "version", "resource", "scope"},
297 )
298
299 metrics = []resettableCollector{
300 deprecatedRequestGauge,
301 requestCounter,
302 longRunningRequestsGauge,
303 requestLatencies,
304 requestSloLatencies,
305 requestSliLatencies,
306 fieldValidationRequestLatencies,
307 responseSizes,
308 TLSHandshakeErrors,
309 WatchEvents,
310 WatchEventsSizes,
311 currentInflightRequests,
312 currentInqueueRequests,
313 requestTerminationsTotal,
314 apiSelfRequestCounter,
315 requestFilterDuration,
316 requestAbortsTotal,
317 requestPostTimeoutTotal,
318 requestTimestampComparisonDuration,
319 watchListLatencies,
320 }
321
322 // these are the valid request methods which we report in our metrics. Any other request methods
323 // will be aggregated under 'unknown'
324 validRequestMethods = utilsets.NewString(
325 "APPLY",
326 "CONNECT",
327 "CREATE",
328 "DELETE",
329 "DELETECOLLECTION",
330 "GET",
331 "LIST",
332 "PATCH",
333 "POST",
334 "PROXY",
335 "PUT",
336 "UPDATE",
337 "WATCH",
338 "WATCHLIST")
339
340 // These are the valid connect requests which we report in our metrics.
341 validConnectRequests = utilsets.NewString(
342 "log",
343 "exec",
344 "portforward",
345 "attach",
346 "proxy")
347 )
348
349 const (
350 // ReadOnlyKind is a string identifying read only request kind
351 ReadOnlyKind = "readOnly"
352 // MutatingKind is a string identifying mutating request kind
353 MutatingKind = "mutating"
354
355 // WaitingPhase is the phase value for a request waiting in a queue
356 WaitingPhase = "waiting"
357 // ExecutingPhase is the phase value for an executing request
358 ExecutingPhase = "executing"
359 )
360
361 const (
362 // deprecatedAnnotationKey is a key for an audit annotation set to
363 // "true" on requests made to deprecated API versions
364 deprecatedAnnotationKey = "k8s.io/deprecated"
365 // removedReleaseAnnotationKey is a key for an audit annotation set to
366 // the target removal release, in "<major>.<minor>" format,
367 // on requests made to deprecated API versions with a target removal release
368 removedReleaseAnnotationKey = "k8s.io/removed-release"
369 )
370
371 const (
372 // The source that is recording the apiserver_request_post_timeout_total metric.
373 // The "executing" request handler returns after the timeout filter times out the request.
374 PostTimeoutSourceTimeoutHandler = "timeout-handler"
375
376 // The source that is recording the apiserver_request_post_timeout_total metric.
377 // The "executing" request handler returns after the rest layer times out the request.
378 PostTimeoutSourceRestHandler = "rest-handler"
379 )
380
381 const (
382 // The executing request handler panicked after the request had
383 // been timed out by the apiserver.
384 PostTimeoutHandlerPanic = "panic"
385
386 // The executing request handler has returned an error to the post-timeout
387 // receiver after the request had been timed out by the apiserver.
388 PostTimeoutHandlerError = "error"
389
390 // The executing request handler has returned a result to the post-timeout
391 // receiver after the request had been timed out by the apiserver.
392 PostTimeoutHandlerOK = "ok"
393
394 // The executing request handler has not panicked or returned any error/result to
395 // the post-timeout receiver yet after the request had been timed out by the apiserver.
396 // The post-timeout receiver gives up after waiting for certain threshold and if the
397 // executing request handler has not returned yet we use the following label.
398 PostTimeoutHandlerPending = "pending"
399 )
400
401 var registerMetrics sync.Once
402
403 // Register all metrics.
404 func Register() {
405 registerMetrics.Do(func() {
406 for _, metric := range metrics {
407 legacyregistry.MustRegister(metric)
408 }
409 })
410 }
411
412 // Reset all metrics.
413 func Reset() {
414 for _, metric := range metrics {
415 metric.Reset()
416 }
417 }
418
419 // ResetLabelAllowLists resets the label allow lists for all metrics.
420 // NOTE: This is only used for testing.
421 func ResetLabelAllowLists() {
422 for _, metric := range metrics {
423 if counterVec, ok := metric.(*compbasemetrics.CounterVec); ok {
424 counterVec.ResetLabelAllowLists()
425 continue
426 }
427 if gaugeVec, ok := metric.(*compbasemetrics.GaugeVec); ok {
428 gaugeVec.ResetLabelAllowLists()
429 continue
430 }
431 if histogramVec, ok := metric.(*compbasemetrics.HistogramVec); ok {
432 histogramVec.ResetLabelAllowLists()
433 continue
434 }
435 if summaryVec, ok := metric.(*compbasemetrics.SummaryVec); ok {
436 summaryVec.ResetLabelAllowLists()
437 continue
438 }
439 if timingHistogramVec, ok := metric.(*compbasemetrics.TimingHistogramVec); ok {
440 timingHistogramVec.ResetLabelAllowLists()
441 continue
442 }
443 }
444 }
445
446 // UpdateInflightRequestMetrics reports concurrency metrics classified by
447 // mutating vs Readonly.
448 func UpdateInflightRequestMetrics(phase string, nonmutating, mutating int) {
449 for _, kc := range []struct {
450 kind string
451 count int
452 }{{ReadOnlyKind, nonmutating}, {MutatingKind, mutating}} {
453 if phase == ExecutingPhase {
454 currentInflightRequests.WithLabelValues(kc.kind).Set(float64(kc.count))
455 } else {
456 currentInqueueRequests.WithLabelValues(kc.kind).Set(float64(kc.count))
457 }
458 }
459 }
460
461 func RecordFilterLatency(ctx context.Context, name string, elapsed time.Duration) {
462 requestFilterDuration.WithContext(ctx).WithLabelValues(name).Observe(elapsed.Seconds())
463 }
464
465 func RecordTimestampComparisonLatency(codePath string, elapsed time.Duration) {
466 requestTimestampComparisonDuration.WithLabelValues(codePath).Observe(elapsed.Seconds())
467 }
468
469 func RecordRequestPostTimeout(source string, status string) {
470 requestPostTimeoutTotal.WithLabelValues(source, status).Inc()
471 }
472
473 // RecordRequestAbort records that the request was aborted possibly due to a timeout.
474 func RecordRequestAbort(req *http.Request, requestInfo *request.RequestInfo) {
475 if requestInfo == nil {
476 requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
477 }
478
479 scope := CleanScope(requestInfo)
480 reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), "", req, requestInfo)
481 resource := requestInfo.Resource
482 subresource := requestInfo.Subresource
483 group := requestInfo.APIGroup
484 version := requestInfo.APIVersion
485
486 requestAbortsTotal.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Inc()
487 }
488
489 // RecordDroppedRequest records that the request was rejected via http.TooManyRequests.
490 func RecordDroppedRequest(req *http.Request, requestInfo *request.RequestInfo, component string, isMutatingRequest bool) {
491 if requestInfo == nil {
492 requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
493 }
494 scope := CleanScope(requestInfo)
495 dryRun := cleanDryRun(req.URL)
496
497 // We don't use verb from <requestInfo>, as this may be propagated from
498 // InstrumentRouteFunc which is registered in installer.go with predefined
499 // list of verbs (different than those translated to RequestInfo).
500 // However, we need to tweak it e.g. to differentiate GET from LIST.
501 reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), "", req, requestInfo)
502
503 if requestInfo.IsResourceRequest {
504 requestCounter.WithContext(req.Context()).WithLabelValues(reportedVerb, dryRun, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(http.StatusTooManyRequests)).Inc()
505 } else {
506 requestCounter.WithContext(req.Context()).WithLabelValues(reportedVerb, dryRun, "", "", "", requestInfo.Subresource, scope, component, codeToString(http.StatusTooManyRequests)).Inc()
507 }
508 }
509
510 // RecordRequestTermination records that the request was terminated early as part of a resource
511 // preservation or apiserver self-defense mechanism (e.g. timeouts, maxinflight throttling,
512 // proxyHandler errors). RecordRequestTermination should only be called zero or one times
513 // per request.
514 func RecordRequestTermination(req *http.Request, requestInfo *request.RequestInfo, component string, code int) {
515 if requestInfo == nil {
516 requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
517 }
518 scope := CleanScope(requestInfo)
519
520 // We don't use verb from <requestInfo>, as this may be propagated from
521 // InstrumentRouteFunc which is registered in installer.go with predefined
522 // list of verbs (different than those translated to RequestInfo).
523 // However, we need to tweak it e.g. to differentiate GET from LIST.
524 reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), "", req, requestInfo)
525
526 if requestInfo.IsResourceRequest {
527 requestTerminationsTotal.WithContext(req.Context()).WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(code)).Inc()
528 } else {
529 requestTerminationsTotal.WithContext(req.Context()).WithLabelValues(reportedVerb, "", "", "", requestInfo.Path, scope, component, codeToString(code)).Inc()
530 }
531 }
532
533 // RecordLongRunning tracks the execution of a long running request against the API server. It provides an accurate count
534 // of the total number of open long running requests. requestInfo may be nil if the caller is not in the normal request flow.
535 func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, component string, fn func()) {
536 if requestInfo == nil {
537 requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
538 }
539 var g compbasemetrics.GaugeMetric
540 scope := CleanScope(requestInfo)
541
542 // We don't use verb from <requestInfo>, as this may be propagated from
543 // InstrumentRouteFunc which is registered in installer.go with predefined
544 // list of verbs (different than those translated to RequestInfo).
545 // However, we need to tweak it e.g. to differentiate GET from LIST.
546 reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), "", req, requestInfo)
547
548 if requestInfo.IsResourceRequest {
549 g = longRunningRequestsGauge.WithContext(req.Context()).WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component)
550 } else {
551 g = longRunningRequestsGauge.WithContext(req.Context()).WithLabelValues(reportedVerb, "", "", "", requestInfo.Path, scope, component)
552 }
553 g.Inc()
554 defer g.Dec()
555 fn()
556 }
557
558 // RecordWatchListLatency simply records response latency for watch list requests.
559 func RecordWatchListLatency(ctx context.Context, gvr schema.GroupVersionResource, metricsScope string) {
560 requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(ctx)
561 if !ok {
562 utilruntime.HandleError(fmt.Errorf("unable to measure watchlist latency because no received ts found in the ctx, gvr: %s", gvr))
563 return
564 }
565 elapsedSeconds := time.Since(requestReceivedTimestamp).Seconds()
566
567 watchListLatencies.WithContext(ctx).WithLabelValues(gvr.Group, gvr.Version, gvr.Resource, metricsScope).Observe(elapsedSeconds)
568 }
569
570 // MonitorRequest handles standard transformations for client and the reported verb and then invokes Monitor to record
571 // a request. verb must be uppercase to be backwards compatible with existing monitoring tooling.
572 func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, httpCode, respSize int, elapsed time.Duration) {
573 requestInfo, ok := request.RequestInfoFrom(req.Context())
574 if !ok || requestInfo == nil {
575 requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
576 }
577 // We don't use verb from <requestInfo>, as this may be propagated from
578 // InstrumentRouteFunc which is registered in installer.go with predefined
579 // list of verbs (different than those translated to RequestInfo).
580 // However, we need to tweak it e.g. to differentiate GET from LIST.
581 reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), verb, req, requestInfo)
582
583 dryRun := cleanDryRun(req.URL)
584 elapsedSeconds := elapsed.Seconds()
585 requestCounter.WithContext(req.Context()).WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component, codeToString(httpCode)).Inc()
586 // MonitorRequest happens after authentication, so we can trust the username given by the request
587 info, ok := request.UserFrom(req.Context())
588 if ok && info.GetName() == user.APIServerUser {
589 apiSelfRequestCounter.WithContext(req.Context()).WithLabelValues(reportedVerb, resource, subresource).Inc()
590 }
591 if deprecated {
592 deprecatedRequestGauge.WithContext(req.Context()).WithLabelValues(group, version, resource, subresource, removedRelease).Set(1)
593 audit.AddAuditAnnotation(req.Context(), deprecatedAnnotationKey, "true")
594 if len(removedRelease) > 0 {
595 audit.AddAuditAnnotation(req.Context(), removedReleaseAnnotationKey, removedRelease)
596 }
597 }
598 requestLatencies.WithContext(req.Context()).WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component).Observe(elapsedSeconds)
599 fieldValidation := cleanFieldValidation(req.URL)
600 fieldValidationRequestLatencies.WithContext(req.Context()).WithLabelValues(fieldValidation)
601
602 if wd, ok := request.LatencyTrackersFrom(req.Context()); ok {
603 sliLatency := elapsedSeconds - (wd.MutatingWebhookTracker.GetLatency() + wd.ValidatingWebhookTracker.GetLatency() + wd.APFQueueWaitTracker.GetLatency()).Seconds()
604 requestSloLatencies.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope, component).Observe(sliLatency)
605 requestSliLatencies.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope, component).Observe(sliLatency)
606 }
607 // We are only interested in response sizes of read requests.
608 if verb == "GET" || verb == "LIST" {
609 responseSizes.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope, component).Observe(float64(respSize))
610 }
611 }
612
613 // InstrumentRouteFunc works like Prometheus' InstrumentHandlerFunc but wraps
614 // the go-restful RouteFunction instead of a HandlerFunc plus some Kubernetes endpoint specific information.
615 func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, routeFunc restful.RouteFunction) restful.RouteFunction {
616 return restful.RouteFunction(func(req *restful.Request, response *restful.Response) {
617 requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(req.Request.Context())
618 if !ok {
619 requestReceivedTimestamp = time.Now()
620 }
621
622 delegate := &ResponseWriterDelegator{ResponseWriter: response.ResponseWriter}
623
624 rw := responsewriter.WrapForHTTP1Or2(delegate)
625 response.ResponseWriter = rw
626
627 routeFunc(req, response)
628
629 MonitorRequest(req.Request, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Status(), delegate.ContentLength(), time.Since(requestReceivedTimestamp))
630 })
631 }
632
633 // InstrumentHandlerFunc works like Prometheus' InstrumentHandlerFunc but adds some Kubernetes endpoint specific information.
634 func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, handler http.HandlerFunc) http.HandlerFunc {
635 return func(w http.ResponseWriter, req *http.Request) {
636 requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(req.Context())
637 if !ok {
638 requestReceivedTimestamp = time.Now()
639 }
640
641 delegate := &ResponseWriterDelegator{ResponseWriter: w}
642 w = responsewriter.WrapForHTTP1Or2(delegate)
643
644 handler(w, req)
645
646 MonitorRequest(req, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Status(), delegate.ContentLength(), time.Since(requestReceivedTimestamp))
647 }
648 }
649
650 // NormalizedVerb returns normalized verb
651 func NormalizedVerb(req *http.Request) string {
652 verb := req.Method
653 requestInfo, ok := request.RequestInfoFrom(req.Context())
654 if ok {
655 // If we can find a requestInfo, we can get a scope, and then
656 // we can convert GETs to LISTs when needed.
657 scope := CleanScope(requestInfo)
658 verb = CanonicalVerb(strings.ToUpper(verb), scope)
659 }
660
661 // mark APPLY requests, WATCH requests and CONNECT requests correctly.
662 return CleanVerb(verb, req, requestInfo)
663 }
664
665 // CleanScope returns the scope of the request.
666 func CleanScope(requestInfo *request.RequestInfo) string {
667 if requestInfo.Name != "" || requestInfo.Verb == "create" {
668 return "resource"
669 }
670 if requestInfo.Namespace != "" {
671 return "namespace"
672 }
673 if requestInfo.IsResourceRequest {
674 return "cluster"
675 }
676 // this is the empty scope
677 return ""
678 }
679
680 // CleanListScope computes the request scope for metrics.
681 //
682 // Note that normally we would use CleanScope for computation.
683 // But due to the same reasons mentioned in determineRequestNamespaceAndName we cannot.
684 func CleanListScope(ctx context.Context, opts *metainternalversion.ListOptions) string {
685 namespace, name := determineRequestNamespaceAndName(ctx, opts)
686 if len(name) > 0 {
687 return "resource"
688 }
689 if len(namespace) > 0 {
690 return "namespace"
691 }
692 if requestInfo, ok := request.RequestInfoFrom(ctx); ok {
693 if requestInfo.IsResourceRequest {
694 return "cluster"
695 }
696 }
697 return ""
698 }
699
700 // CanonicalVerb distinguishes LISTs from GETs (and HEADs). It assumes verb is
701 // UPPERCASE.
702 func CanonicalVerb(verb string, scope string) string {
703 switch verb {
704 case "GET", "HEAD":
705 if scope != "resource" && scope != "" {
706 return "LIST"
707 }
708 return "GET"
709 default:
710 return verb
711 }
712 }
713
714 // CleanVerb returns a normalized verb, so that it is easy to tell WATCH from
715 // LIST, APPLY from PATCH and CONNECT from others.
716 func CleanVerb(verb string, request *http.Request, requestInfo *request.RequestInfo) string {
717 reportedVerb := verb
718 if suggestedVerb := getVerbIfWatch(request); suggestedVerb == "WATCH" {
719 reportedVerb = "WATCH"
720 }
721 // normalize the legacy WATCHLIST to WATCH to ensure users aren't surprised by metrics
722 if verb == "WATCHLIST" {
723 reportedVerb = "WATCH"
724 }
725 if verb == "PATCH" && request.Header.Get("Content-Type") == string(types.ApplyPatchType) {
726 reportedVerb = "APPLY"
727 }
728 if requestInfo != nil && requestInfo.IsResourceRequest && len(requestInfo.Subresource) > 0 && validConnectRequests.Has(requestInfo.Subresource) {
729 reportedVerb = "CONNECT"
730 }
731 return reportedVerb
732 }
733
734 // determineRequestNamespaceAndName computes name and namespace for the given requests
735 //
736 // note that the logic of this function was copy&pasted from cacher.go
737 // after an unsuccessful attempt of moving it to RequestInfo
738 //
739 // see: https://github.com/kubernetes/kubernetes/pull/120520
740 func determineRequestNamespaceAndName(ctx context.Context, opts *metainternalversion.ListOptions) (namespace, name string) {
741 if requestNamespace, ok := request.NamespaceFrom(ctx); ok && len(requestNamespace) > 0 {
742 namespace = requestNamespace
743 } else if opts != nil && opts.FieldSelector != nil {
744 if selectorNamespace, ok := opts.FieldSelector.RequiresExactMatch("metadata.namespace"); ok {
745 namespace = selectorNamespace
746 }
747 }
748 if requestInfo, ok := request.RequestInfoFrom(ctx); ok && requestInfo != nil && len(requestInfo.Name) > 0 {
749 name = requestInfo.Name
750 } else if opts != nil && opts.FieldSelector != nil {
751 if selectorName, ok := opts.FieldSelector.RequiresExactMatch("metadata.name"); ok {
752 name = selectorName
753 }
754 }
755 return
756 }
757
758 // cleanVerb additionally ensures that unknown verbs don't clog up the metrics.
759 func cleanVerb(verb, suggestedVerb string, request *http.Request, requestInfo *request.RequestInfo) string {
760 // CanonicalVerb (being an input for this function) doesn't handle correctly the
761 // deprecated path pattern for watch of:
762 // GET /api/{version}/watch/{resource}
763 // We correct it manually based on the pass verb from the installer.
764 if suggestedVerb == "WATCH" || suggestedVerb == "WATCHLIST" {
765 return "WATCH"
766 }
767 reportedVerb := CleanVerb(verb, request, requestInfo)
768 if validRequestMethods.Has(reportedVerb) {
769 return reportedVerb
770 }
771 return OtherRequestMethod
772 }
773
774 // getVerbIfWatch additionally ensures that GET or List would be transformed to WATCH
775 func getVerbIfWatch(req *http.Request) string {
776 if strings.ToUpper(req.Method) == "GET" || strings.ToUpper(req.Method) == "LIST" {
777 // see apimachinery/pkg/runtime/conversion.go Convert_Slice_string_To_bool
778 if values := req.URL.Query()["watch"]; len(values) > 0 {
779 if value := strings.ToLower(values[0]); value != "0" && value != "false" {
780 return "WATCH"
781 }
782 }
783 }
784 return ""
785 }
786
787 func cleanDryRun(u *url.URL) string {
788 // avoid allocating when we don't see dryRun in the query
789 if !strings.Contains(u.RawQuery, "dryRun") {
790 return ""
791 }
792 dryRun := u.Query()["dryRun"]
793 if errs := validation.ValidateDryRun(nil, dryRun); len(errs) > 0 {
794 return "invalid"
795 }
796 // Since dryRun could be valid with any arbitrarily long length
797 // we have to dedup and sort the elements before joining them together
798 // TODO: this is a fairly large allocation for what it does, consider
799 // a sort and dedup in a single pass
800 return strings.Join(utilsets.NewString(dryRun...).List(), ",")
801 }
802
803 func cleanFieldValidation(u *url.URL) string {
804 // avoid allocating when we don't see dryRun in the query
805 if !strings.Contains(u.RawQuery, "fieldValidation") {
806 return ""
807 }
808 fieldValidation := u.Query()["fieldValidation"]
809 if len(fieldValidation) != 1 {
810 return "invalid"
811 }
812 if errs := validation.ValidateFieldValidation(nil, fieldValidation[0]); len(errs) > 0 {
813 return "invalid"
814 }
815 return fieldValidation[0]
816 }
817
818 var _ http.ResponseWriter = (*ResponseWriterDelegator)(nil)
819 var _ responsewriter.UserProvidedDecorator = (*ResponseWriterDelegator)(nil)
820
821 // ResponseWriterDelegator interface wraps http.ResponseWriter to additionally record content-length, status-code, etc.
822 type ResponseWriterDelegator struct {
823 http.ResponseWriter
824
825 status int
826 written int64
827 wroteHeader bool
828 }
829
830 func (r *ResponseWriterDelegator) Unwrap() http.ResponseWriter {
831 return r.ResponseWriter
832 }
833
834 func (r *ResponseWriterDelegator) WriteHeader(code int) {
835 r.status = code
836 r.wroteHeader = true
837 r.ResponseWriter.WriteHeader(code)
838 }
839
840 func (r *ResponseWriterDelegator) Write(b []byte) (int, error) {
841 if !r.wroteHeader {
842 r.WriteHeader(http.StatusOK)
843 }
844 n, err := r.ResponseWriter.Write(b)
845 r.written += int64(n)
846 return n, err
847 }
848
849 func (r *ResponseWriterDelegator) Status() int {
850 return r.status
851 }
852
853 func (r *ResponseWriterDelegator) ContentLength() int {
854 return int(r.written)
855 }
856
857 // Small optimization over Itoa
858 func codeToString(s int) string {
859 switch s {
860 case 100:
861 return "100"
862 case 101:
863 return "101"
864
865 case 200:
866 return "200"
867 case 201:
868 return "201"
869 case 202:
870 return "202"
871 case 203:
872 return "203"
873 case 204:
874 return "204"
875 case 205:
876 return "205"
877 case 206:
878 return "206"
879
880 case 300:
881 return "300"
882 case 301:
883 return "301"
884 case 302:
885 return "302"
886 case 304:
887 return "304"
888 case 305:
889 return "305"
890 case 307:
891 return "307"
892
893 case 400:
894 return "400"
895 case 401:
896 return "401"
897 case 402:
898 return "402"
899 case 403:
900 return "403"
901 case 404:
902 return "404"
903 case 405:
904 return "405"
905 case 406:
906 return "406"
907 case 407:
908 return "407"
909 case 408:
910 return "408"
911 case 409:
912 return "409"
913 case 410:
914 return "410"
915 case 411:
916 return "411"
917 case 412:
918 return "412"
919 case 413:
920 return "413"
921 case 414:
922 return "414"
923 case 415:
924 return "415"
925 case 416:
926 return "416"
927 case 417:
928 return "417"
929 case 418:
930 return "418"
931
932 case 500:
933 return "500"
934 case 501:
935 return "501"
936 case 502:
937 return "502"
938 case 503:
939 return "503"
940 case 504:
941 return "504"
942 case 505:
943 return "505"
944
945 case 428:
946 return "428"
947 case 429:
948 return "429"
949 case 431:
950 return "431"
951 case 511:
952 return "511"
953
954 default:
955 return strconv.Itoa(s)
956 }
957 }