Debian Codegraph @

pkg/admission/metrics/metrics.go

raw
1 /*
2 Copyright 2017 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package metrics
18
19 import (
20 "context"
21 "strconv"
22 "time"
23
24 "k8s.io/apiserver/pkg/admission"
25 "k8s.io/component-base/metrics"
26 "k8s.io/component-base/metrics/legacyregistry"
27 )
28
29 // WebhookRejectionErrorType defines different error types that happen in a webhook rejection.
30 type WebhookRejectionErrorType string
31
32 const (
33 namespace = "apiserver"
34 subsystem = "admission"
35
36 // WebhookRejectionCallingWebhookError identifies a calling webhook error which causes
37 // a webhook admission to reject a request
38 WebhookRejectionCallingWebhookError WebhookRejectionErrorType = "calling_webhook_error"
39 // WebhookRejectionAPIServerInternalError identifies an apiserver internal error which
40 // causes a webhook admission to reject a request
41 WebhookRejectionAPIServerInternalError WebhookRejectionErrorType = "apiserver_internal_error"
42 // WebhookRejectionNoError identifies a webhook properly rejected a request
43 WebhookRejectionNoError WebhookRejectionErrorType = "no_error"
44 )
45
46 var (
47 latencySummaryMaxAge = 5 * time.Hour
48
49 // Metrics provides access to all admission metrics.
50 Metrics = newAdmissionMetrics()
51 )
52
53 // ObserverFunc is a func that emits metrics.
54 type ObserverFunc func(ctx context.Context, elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string)
55
56 const (
57 kindWebhook = "webhook"
58 kindPolicy = "policy"
59 stepValidate = "validate"
60 stepAdmit = "admit"
61 )
62
63 // WithControllerMetrics is a decorator for named admission handlers.
64 func WithControllerMetrics(i admission.Interface, name string) admission.Interface {
65 return WithMetrics(i, Metrics.ObserveAdmissionController, name)
66 }
67
68 // WithStepMetrics is a decorator for a whole admission phase, i.e. admit or validation.admission step.
69 func WithStepMetrics(i admission.Interface) admission.Interface {
70 return WithMetrics(i, Metrics.ObserveAdmissionStep)
71 }
72
73 // WithMetrics is a decorator for admission handlers with a generic observer func.
74 func WithMetrics(i admission.Interface, observer ObserverFunc, extraLabels ...string) admission.Interface {
75 return &pluginHandlerWithMetrics{
76 Interface: i,
77 observer: observer,
78 extraLabels: extraLabels,
79 }
80 }
81
82 // pluginHandlerWithMetrics decorates a admission handler with metrics.
83 type pluginHandlerWithMetrics struct {
84 admission.Interface
85 observer ObserverFunc
86 extraLabels []string
87 }
88
89 // Admit performs a mutating admission control check and emit metrics.
90 func (p pluginHandlerWithMetrics) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error {
91 mutatingHandler, ok := p.Interface.(admission.MutationInterface)
92 if !ok {
93 return nil
94 }
95
96 start := time.Now()
97 err := mutatingHandler.Admit(ctx, a, o)
98 p.observer(ctx, time.Since(start), err != nil, a, stepAdmit, p.extraLabels...)
99 return err
100 }
101
102 // Validate performs a non-mutating admission control check and emits metrics.
103 func (p pluginHandlerWithMetrics) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error {
104 validatingHandler, ok := p.Interface.(admission.ValidationInterface)
105 if !ok {
106 return nil
107 }
108
109 start := time.Now()
110 err := validatingHandler.Validate(ctx, a, o)
111 p.observer(ctx, time.Since(start), err != nil, a, stepValidate, p.extraLabels...)
112 return err
113 }
114
115 // AdmissionMetrics instruments admission with prometheus metrics.
116 type AdmissionMetrics struct {
117 step *metricSet
118 controller *metricSet
119 webhook *metricSet
120 webhookRejection *metrics.CounterVec
121 webhookFailOpen *metrics.CounterVec
122 webhookRequest *metrics.CounterVec
123 matchConditionEvalErrors *metrics.CounterVec
124 matchConditionExclusions *metrics.CounterVec
125 matchConditionEvaluationSeconds *metricSet
126 }
127
128 // newAdmissionMetrics create a new AdmissionMetrics, configured with default metric names.
129 func newAdmissionMetrics() *AdmissionMetrics {
130 // Admission metrics for a step of the admission flow. The entire admission flow is broken down into a series of steps
131 // Each step is identified by a distinct type label value.
132 // Use buckets ranging from 5 ms to 2.5 seconds.
133 step := &metricSet{
134 latencies: metrics.NewHistogramVec(
135 &metrics.HistogramOpts{
136 Namespace: namespace,
137 Subsystem: subsystem,
138 Name: "step_admission_duration_seconds",
139 Help: "Admission sub-step latency histogram in seconds, broken out for each operation and API resource and step type (validate or admit).",
140 Buckets: []float64{0.005, 0.025, 0.1, 0.5, 1.0, 2.5},
141 StabilityLevel: metrics.STABLE,
142 },
143 []string{"type", "operation", "rejected"},
144 ),
145
146 latenciesSummary: metrics.NewSummaryVec(
147 &metrics.SummaryOpts{
148 Namespace: namespace,
149 Subsystem: subsystem,
150 Name: "step_admission_duration_seconds_summary",
151 Help: "Admission sub-step latency summary in seconds, broken out for each operation and API resource and step type (validate or admit).",
152 MaxAge: latencySummaryMaxAge,
153 StabilityLevel: metrics.ALPHA,
154 },
155 []string{"type", "operation", "rejected"},
156 ),
157 }
158
159 // Built-in admission controller metrics. Each admission controller is identified by name.
160 // Use buckets ranging from 5 ms to 2.5 seconds.
161 controller := &metricSet{
162 latencies: metrics.NewHistogramVec(
163 &metrics.HistogramOpts{
164 Namespace: namespace,
165 Subsystem: subsystem,
166 Name: "controller_admission_duration_seconds",
167 Help: "Admission controller latency histogram in seconds, identified by name and broken out for each operation and API resource and type (validate or admit).",
168 Buckets: []float64{0.005, 0.025, 0.1, 0.5, 1.0, 2.5},
169 StabilityLevel: metrics.STABLE,
170 },
171 []string{"name", "type", "operation", "rejected"},
172 ),
173
174 latenciesSummary: nil,
175 }
176
177 // Admission webhook metrics. Each webhook is identified by name.
178 // Use buckets ranging from 5 ms to 2.5 seconds (admission webhooks timeout at 30 seconds by default).
179 webhook := &metricSet{
180 latencies: metrics.NewHistogramVec(
181 &metrics.HistogramOpts{
182 Namespace: namespace,
183 Subsystem: subsystem,
184 Name: "webhook_admission_duration_seconds",
185 Help: "Admission webhook latency histogram in seconds, identified by name and broken out for each operation and API resource and type (validate or admit).",
186 Buckets: []float64{0.005, 0.025, 0.1, 0.5, 1.0, 2.5, 10, 25},
187 StabilityLevel: metrics.STABLE,
188 },
189 []string{"name", "type", "operation", "rejected"},
190 ),
191
192 latenciesSummary: nil,
193 }
194
195 webhookRejection := metrics.NewCounterVec(
196 &metrics.CounterOpts{
197 Namespace: namespace,
198 Subsystem: subsystem,
199 Name: "webhook_rejection_count",
200 Help: "Admission webhook rejection count, identified by name and broken out for each admission type (validating or admit) and operation. Additional labels specify an error type (calling_webhook_error or apiserver_internal_error if an error occurred; no_error otherwise) and optionally a non-zero rejection code if the webhook rejects the request with an HTTP status code (honored by the apiserver when the code is greater or equal to 400). Codes greater than 600 are truncated to 600, to keep the metrics cardinality bounded.",
201 StabilityLevel: metrics.ALPHA,
202 },
203 []string{"name", "type", "operation", "error_type", "rejection_code"})
204
205 webhookFailOpen := metrics.NewCounterVec(
206 &metrics.CounterOpts{
207 Namespace: namespace,
208 Subsystem: subsystem,
209 Name: "webhook_fail_open_count",
210 Help: "Admission webhook fail open count, identified by name and broken out for each admission type (validating or admit).",
211 StabilityLevel: metrics.ALPHA,
212 },
213 []string{"name", "type"})
214
215 webhookRequest := metrics.NewCounterVec(
216 &metrics.CounterOpts{
217 Namespace: namespace,
218 Subsystem: subsystem,
219 Name: "webhook_request_total",
220 Help: "Admission webhook request total, identified by name and broken out for each admission type (validating or admit) and operation. Additional labels specify whether the request was rejected or not and an HTTP status code. Codes greater than 600 are truncated to 600, to keep the metrics cardinality bounded.",
221 StabilityLevel: metrics.ALPHA,
222 },
223 []string{"name", "type", "operation", "code", "rejected"})
224
225 matchConditionEvalError := metrics.NewCounterVec(
226 &metrics.CounterOpts{
227 Namespace: namespace,
228 Subsystem: subsystem,
229 Name: "match_condition_evaluation_errors_total",
230 Help: "Admission match condition evaluation errors count, identified by name of resource containing the match condition and broken out for each kind containing matchConditions (webhook or policy), operation and admission type (validate or admit).",
231 StabilityLevel: metrics.ALPHA,
232 },
233 []string{"name", "kind", "type", "operation"})
234
235 matchConditionExclusions := metrics.NewCounterVec(
236 &metrics.CounterOpts{
237 Namespace: namespace,
238 Subsystem: subsystem,
239 Name: "match_condition_exclusions_total",
240 Help: "Admission match condition evaluation exclusions count, identified by name of resource containing the match condition and broken out for each kind containing matchConditions (webhook or policy), operation and admission type (validate or admit).",
241 StabilityLevel: metrics.ALPHA,
242 },
243 []string{"name", "kind", "type", "operation"})
244
245 matchConditionEvaluationSeconds := &metricSet{
246 latencies: metrics.NewHistogramVec(
247 &metrics.HistogramOpts{
248 Namespace: namespace,
249 Subsystem: subsystem,
250 Name: "match_condition_evaluation_seconds",
251 Help: "Admission match condition evaluation time in seconds, identified by name and broken out for each kind containing matchConditions (webhook or policy), operation and type (validate or admit).",
252 Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.1, 0.2, 0.25},
253 StabilityLevel: metrics.ALPHA,
254 },
255 []string{"name", "kind", "type", "operation"},
256 ),
257 latenciesSummary: nil,
258 }
259
260 step.mustRegister()
261 controller.mustRegister()
262 webhook.mustRegister()
263 matchConditionEvaluationSeconds.mustRegister()
264 legacyregistry.MustRegister(webhookRejection)
265 legacyregistry.MustRegister(webhookFailOpen)
266 legacyregistry.MustRegister(webhookRequest)
267 legacyregistry.MustRegister(matchConditionEvalError)
268 legacyregistry.MustRegister(matchConditionExclusions)
269 return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest, matchConditionEvalErrors: matchConditionEvalError, matchConditionExclusions: matchConditionExclusions, matchConditionEvaluationSeconds: matchConditionEvaluationSeconds}
270 }
271
272 func (m *AdmissionMetrics) reset() {
273 m.step.reset()
274 m.controller.reset()
275 m.webhook.reset()
276 }
277
278 // ObserveAdmissionStep records admission related metrics for a admission step, identified by step type.
279 func (m *AdmissionMetrics) ObserveAdmissionStep(ctx context.Context, elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string) {
280 m.step.observe(ctx, elapsed, append(extraLabels, stepType, string(attr.GetOperation()), strconv.FormatBool(rejected))...)
281 }
282
283 // ObserveAdmissionController records admission related metrics for a built-in admission controller, identified by it's plugin handler name.
284 func (m *AdmissionMetrics) ObserveAdmissionController(ctx context.Context, elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string) {
285 m.controller.observe(ctx, elapsed, append(extraLabels, stepType, string(attr.GetOperation()), strconv.FormatBool(rejected))...)
286 }
287
288 // ObserveWebhook records admission related metrics for a admission webhook.
289 func (m *AdmissionMetrics) ObserveWebhook(ctx context.Context, name string, elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, code int) {
290 // We truncate codes greater than 600 to keep the cardinality bounded.
291 if code > 600 {
292 code = 600
293 }
294 m.webhookRequest.WithContext(ctx).WithLabelValues(name, stepType, string(attr.GetOperation()), strconv.Itoa(code), strconv.FormatBool(rejected)).Inc()
295 m.webhook.observe(ctx, elapsed, name, stepType, string(attr.GetOperation()), strconv.FormatBool(rejected))
296 }
297
298 // ObserveWebhookRejection records admission related metrics for an admission webhook rejection.
299 func (m *AdmissionMetrics) ObserveWebhookRejection(ctx context.Context, name, stepType, operation string, errorType WebhookRejectionErrorType, rejectionCode int) {
300 // We truncate codes greater than 600 to keep the cardinality bounded.
301 // This should be rarely done by a malfunctioning webhook server.
302 if rejectionCode > 600 {
303 rejectionCode = 600
304 }
305 m.webhookRejection.WithContext(ctx).WithLabelValues(name, stepType, operation, string(errorType), strconv.Itoa(rejectionCode)).Inc()
306 }
307
308 // ObserveWebhookFailOpen records validating or mutating webhook that fail open.
309 func (m *AdmissionMetrics) ObserveWebhookFailOpen(ctx context.Context, name, stepType string) {
310 m.webhookFailOpen.WithContext(ctx).WithLabelValues(name, stepType).Inc()
311 }
312
313 // ObserveMatchConditionEvalError records validating or mutating webhook that are not called due to match conditions
314 func (m *AdmissionMetrics) ObserveMatchConditionEvalError(ctx context.Context, name, kind, stepType, operation string) {
315 m.matchConditionEvalErrors.WithContext(ctx).WithLabelValues(name, kind, stepType, operation).Inc()
316 }
317
318 // ObserveMatchConditionExclusion records validating or mutating webhook that are not called due to match conditions
319 func (m *AdmissionMetrics) ObserveMatchConditionExclusion(ctx context.Context, name, kind, stepType, operation string) {
320 m.matchConditionExclusions.WithContext(ctx).WithLabelValues(name, kind, stepType, operation).Inc()
321 }
322
323 // ObserveMatchConditionEvaluationTime records duration of match condition evaluation process.
324 func (m *AdmissionMetrics) ObserveMatchConditionEvaluationTime(ctx context.Context, elapsed time.Duration, name, kind, stepType, operation string) {
325 m.matchConditionEvaluationSeconds.observe(ctx, elapsed, name, kind, stepType, operation)
326 }
327
328 type metricSet struct {
329 latencies *metrics.HistogramVec
330 latenciesSummary *metrics.SummaryVec
331 }
332
333 // MustRegister registers all the prometheus metrics in the metricSet.
334 func (m *metricSet) mustRegister() {
335 legacyregistry.MustRegister(m.latencies)
336 if m.latenciesSummary != nil {
337 legacyregistry.MustRegister(m.latenciesSummary)
338 }
339 }
340
341 // Reset resets all the prometheus metrics in the metricSet.
342 func (m *metricSet) reset() {
343 m.latencies.Reset()
344 if m.latenciesSummary != nil {
345 m.latenciesSummary.Reset()
346 }
347 }
348
349 // Observe records an observed admission event to all metrics in the metricSet.
350 func (m *metricSet) observe(ctx context.Context, elapsed time.Duration, labels ...string) {
351 elapsedSeconds := elapsed.Seconds()
352 m.latencies.WithContext(ctx).WithLabelValues(labels...).Observe(elapsedSeconds)
353 if m.latenciesSummary != nil {
354 m.latenciesSummary.WithContext(ctx).WithLabelValues(labels...).Observe(elapsedSeconds)
355 }
356 }