Debian Codegraph @

pkg/admission/plugin/webhook/mutating/dispatcher.go

raw
1 /*
2 Copyright 2018 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 // Package mutating delegates admission checks to dynamically configured
18 // mutating webhooks.
19 package mutating
20
21 import (
22 "context"
23 "errors"
24 "fmt"
25 "time"
26
27 "go.opentelemetry.io/otel/attribute"
28 jsonpatch "gopkg.in/evanphx/json-patch.v4"
29
30 admissionv1 "k8s.io/api/admission/v1"
31 admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
32 apiequality "k8s.io/apimachinery/pkg/api/equality"
33 apierrors "k8s.io/apimachinery/pkg/api/errors"
34 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
35 "k8s.io/apimachinery/pkg/runtime"
36 "k8s.io/apimachinery/pkg/runtime/schema"
37 "k8s.io/apimachinery/pkg/runtime/serializer/json"
38 utiljson "k8s.io/apimachinery/pkg/util/json"
39 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
40 "k8s.io/apiserver/pkg/admission"
41 admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
42 "k8s.io/apiserver/pkg/admission/plugin/webhook"
43 webhookerrors "k8s.io/apiserver/pkg/admission/plugin/webhook/errors"
44 "k8s.io/apiserver/pkg/admission/plugin/webhook/generic"
45 webhookrequest "k8s.io/apiserver/pkg/admission/plugin/webhook/request"
46 auditinternal "k8s.io/apiserver/pkg/apis/audit"
47 endpointsrequest "k8s.io/apiserver/pkg/endpoints/request"
48 webhookutil "k8s.io/apiserver/pkg/util/webhook"
49 "k8s.io/apiserver/pkg/warning"
50 "k8s.io/component-base/tracing"
51 "k8s.io/klog/v2"
52 )
53
54 const (
55 // PatchAuditAnnotationPrefix is a prefix for persisting webhook patch in audit annotation.
56 // Audit handler decides whether annotation with this prefix should be logged based on audit level.
57 // Since mutating webhook patches the request body, audit level must be greater or equal to Request
58 // for the annotation to be logged
59 PatchAuditAnnotationPrefix = "patch.webhook.admission.k8s.io/"
60 // MutationAuditAnnotationPrefix is a prefix for presisting webhook mutation existence in audit annotation.
61 MutationAuditAnnotationPrefix = "mutation.webhook.admission.k8s.io/"
62 // MutationAnnotationFailedOpenKeyPrefix in an annotation indicates
63 // the mutating webhook failed open when the webhook backend connection
64 // failed or returned an internal server error.
65 MutationAuditAnnotationFailedOpenKeyPrefix string = "failed-open." + MutationAuditAnnotationPrefix
66 )
67
68 type mutatingDispatcher struct {
69 cm *webhookutil.ClientManager
70 plugin *Plugin
71 }
72
73 func newMutatingDispatcher(p *Plugin) func(cm *webhookutil.ClientManager) generic.Dispatcher {
74 return func(cm *webhookutil.ClientManager) generic.Dispatcher {
75 return &mutatingDispatcher{cm, p}
76 }
77 }
78
79 var _ generic.VersionedAttributeAccessor = &versionedAttributeAccessor{}
80
81 type versionedAttributeAccessor struct {
82 versionedAttr *admission.VersionedAttributes
83 attr admission.Attributes
84 objectInterfaces admission.ObjectInterfaces
85 }
86
87 func (v *versionedAttributeAccessor) VersionedAttribute(gvk schema.GroupVersionKind) (*admission.VersionedAttributes, error) {
88 if v.versionedAttr == nil {
89 // First call, create versioned attributes
90 var err error
91 if v.versionedAttr, err = admission.NewVersionedAttributes(v.attr, gvk, v.objectInterfaces); err != nil {
92 return nil, apierrors.NewInternalError(err)
93 }
94 } else {
95 // Subsequent call, convert existing versioned attributes to the requested version
96 if err := admission.ConvertVersionedAttributes(v.versionedAttr, gvk, v.objectInterfaces); err != nil {
97 return nil, apierrors.NewInternalError(err)
98 }
99 }
100 return v.versionedAttr, nil
101 }
102
103 var _ generic.Dispatcher = &mutatingDispatcher{}
104
105 func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces, hooks []webhook.WebhookAccessor) error {
106 reinvokeCtx := attr.GetReinvocationContext()
107 var webhookReinvokeCtx *webhookReinvokeContext
108 if v := reinvokeCtx.Value(PluginName); v != nil {
109 webhookReinvokeCtx = v.(*webhookReinvokeContext)
110 } else {
111 webhookReinvokeCtx = &webhookReinvokeContext{}
112 reinvokeCtx.SetValue(PluginName, webhookReinvokeCtx)
113 }
114
115 if reinvokeCtx.IsReinvoke() && webhookReinvokeCtx.IsOutputChangedSinceLastWebhookInvocation(attr.GetObject()) {
116 // If the object has changed, we know the in-tree plugin re-invocations have mutated the object,
117 // and we need to reinvoke all eligible webhooks.
118 webhookReinvokeCtx.RequireReinvokingPreviouslyInvokedPlugins()
119 }
120 defer func() {
121 webhookReinvokeCtx.SetLastWebhookInvocationOutput(attr.GetObject())
122 }()
123 v := &versionedAttributeAccessor{
124 attr: attr,
125 objectInterfaces: o,
126 }
127 for i, hook := range hooks {
128 attrForCheck := attr
129 if v.versionedAttr != nil {
130 attrForCheck = v.versionedAttr
131 }
132
133 invocation, statusErr := a.plugin.ShouldCallHook(ctx, hook, attrForCheck, o, v)
134 if statusErr != nil {
135 return statusErr
136 }
137 if invocation == nil {
138 continue
139 }
140
141 hook, ok := invocation.Webhook.GetMutatingWebhook()
142 if !ok {
143 return fmt.Errorf("mutating webhook dispatch requires v1.MutatingWebhook, but got %T", hook)
144 }
145 // This means that during reinvocation, a webhook will not be
146 // called for the first time. For example, if the webhook is
147 // skipped in the first round because of mismatching labels,
148 // even if the labels become matching, the webhook does not
149 // get called during reinvocation.
150 if reinvokeCtx.IsReinvoke() && !webhookReinvokeCtx.ShouldReinvokeWebhook(invocation.Webhook.GetUID()) {
151 continue
152 }
153
154 versionedAttr, err := v.VersionedAttribute(invocation.Kind)
155 if err != nil {
156 return apierrors.NewInternalError(err)
157 }
158
159 t := time.Now()
160 round := 0
161 if reinvokeCtx.IsReinvoke() {
162 round = 1
163 }
164
165 annotator := newWebhookAnnotator(versionedAttr, round, i, hook.Name, invocation.Webhook.GetConfigurationName())
166 changed, err := a.callAttrMutatingHook(ctx, hook, invocation, versionedAttr, annotator, o, round, i)
167 ignoreClientCallFailures := hook.FailurePolicy != nil && *hook.FailurePolicy == admissionregistrationv1.Ignore
168 rejected := false
169 if err != nil {
170 switch err := err.(type) {
171 case *webhookutil.ErrCallingWebhook:
172 if !ignoreClientCallFailures {
173 rejected = true
174 // Ignore context cancelled from webhook metrics
175 if !errors.Is(err.Reason, context.Canceled) {
176 admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))
177 }
178 }
179 admissionmetrics.Metrics.ObserveWebhook(ctx, hook.Name, time.Since(t), rejected, versionedAttr.Attributes, "admit", int(err.Status.ErrStatus.Code))
180 case *webhookutil.ErrWebhookRejection:
181 rejected = true
182 admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionNoError, int(err.Status.ErrStatus.Code))
183 admissionmetrics.Metrics.ObserveWebhook(ctx, hook.Name, time.Since(t), rejected, versionedAttr.Attributes, "admit", int(err.Status.ErrStatus.Code))
184 default:
185 rejected = true
186 admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionAPIServerInternalError, 0)
187 admissionmetrics.Metrics.ObserveWebhook(ctx, hook.Name, time.Since(t), rejected, versionedAttr.Attributes, "admit", 0)
188 }
189 } else {
190 admissionmetrics.Metrics.ObserveWebhook(ctx, hook.Name, time.Since(t), rejected, versionedAttr.Attributes, "admit", 200)
191 }
192 if changed {
193 // Patch had changed the object. Prepare to reinvoke all previous mutations that are eligible for re-invocation.
194 webhookReinvokeCtx.RequireReinvokingPreviouslyInvokedPlugins()
195 reinvokeCtx.SetShouldReinvoke()
196 }
197 if hook.ReinvocationPolicy != nil && *hook.ReinvocationPolicy == admissionregistrationv1.IfNeededReinvocationPolicy {
198 webhookReinvokeCtx.AddReinvocableWebhookToPreviouslyInvoked(invocation.Webhook.GetUID())
199 }
200 if err == nil {
201 continue
202 }
203
204 if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
205 if ignoreClientCallFailures {
206 // Ignore context cancelled from webhook metrics
207 if errors.Is(callErr.Reason, context.Canceled) {
208 klog.Warningf("Context canceled when calling webhook %v", hook.Name)
209 } else {
210 klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
211 admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit")
212 annotator.addFailedOpenAnnotation()
213 }
214 utilruntime.HandleError(callErr)
215
216 select {
217 case <-ctx.Done():
218 // parent context is canceled or timed out, no point in continuing
219 return apierrors.NewTimeoutError("request did not complete within requested timeout", 0)
220 default:
221 // individual webhook timed out, but parent context did not, continue
222 continue
223 }
224 }
225 klog.Warningf("Failed calling webhook, failing closed %v: %v", hook.Name, err)
226 return apierrors.NewInternalError(err)
227 }
228 if rejectionErr, ok := err.(*webhookutil.ErrWebhookRejection); ok {
229 return rejectionErr.Status
230 }
231 return err
232 }
233
234 // convert versionedAttr.VersionedObject to the internal version in the underlying admission.Attributes
235 if v.versionedAttr != nil && v.versionedAttr.VersionedObject != nil && v.versionedAttr.Dirty {
236 return o.GetObjectConvertor().Convert(v.versionedAttr.VersionedObject, v.versionedAttr.Attributes.GetObject(), nil)
237 }
238
239 return nil
240 }
241
242 // note that callAttrMutatingHook updates attr
243
244 func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admissionregistrationv1.MutatingWebhook, invocation *generic.WebhookInvocation, attr *admission.VersionedAttributes, annotator *webhookAnnotator, o admission.ObjectInterfaces, round, idx int) (bool, error) {
245 configurationName := invocation.Webhook.GetConfigurationName()
246 changed := false
247 defer func() { annotator.addMutationAnnotation(changed) }()
248 if attr.Attributes.IsDryRun() {
249 if h.SideEffects == nil {
250 return false, &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("Webhook SideEffects is nil"), Status: apierrors.NewBadRequest("Webhook SideEffects is nil")}
251 }
252 if !(*h.SideEffects == admissionregistrationv1.SideEffectClassNone || *h.SideEffects == admissionregistrationv1.SideEffectClassNoneOnDryRun) {
253 return false, webhookerrors.NewDryRunUnsupportedErr(h.Name)
254 }
255 }
256
257 uid, request, response, err := webhookrequest.CreateAdmissionObjects(attr, invocation)
258 if err != nil {
259 return false, &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("could not create admission objects: %w", err), Status: apierrors.NewBadRequest("error creating admission objects")}
260 }
261 // Make the webhook request
262 client, err := invocation.Webhook.GetRESTClient(a.cm)
263 if err != nil {
264 return false, &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("could not get REST client: %w", err), Status: apierrors.NewBadRequest("error getting REST client")}
265 }
266 ctx, span := tracing.Start(ctx, "Call mutating webhook",
267 attribute.String("configuration", configurationName),
268 attribute.String("webhook", h.Name),
269 attribute.Stringer("resource", attr.GetResource()),
270 attribute.String("subresource", attr.GetSubresource()),
271 attribute.String("operation", string(attr.GetOperation())),
272 attribute.String("UID", string(uid)))
273 defer span.End(500 * time.Millisecond)
274
275 // if the webhook has a specific timeout, wrap the context to apply it
276 if h.TimeoutSeconds != nil {
277 var cancel context.CancelFunc
278 ctx, cancel = context.WithTimeout(ctx, time.Duration(*h.TimeoutSeconds)*time.Second)
279 defer cancel()
280 }
281
282 r := client.Post().Body(request)
283
284 // if the context has a deadline, set it as a parameter to inform the backend
285 if deadline, hasDeadline := ctx.Deadline(); hasDeadline {
286 // compute the timeout
287 if timeout := time.Until(deadline); timeout > 0 {
288 // if it's not an even number of seconds, round up to the nearest second
289 if truncated := timeout.Truncate(time.Second); truncated != timeout {
290 timeout = truncated + time.Second
291 }
292 // set the timeout
293 r.Timeout(timeout)
294 }
295 }
296
297 do := func() { err = r.Do(ctx).Into(response) }
298 if wd, ok := endpointsrequest.LatencyTrackersFrom(ctx); ok {
299 tmp := do
300 do = func() { wd.MutatingWebhookTracker.Track(tmp) }
301 }
302 do()
303 if err != nil {
304 var status *apierrors.StatusError
305 if se, ok := err.(*apierrors.StatusError); ok {
306 status = se
307 } else {
308 status = apierrors.NewBadRequest("error calling webhook")
309 }
310 return false, &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("failed to call webhook: %w", err), Status: status}
311 }
312 span.AddEvent("Request completed")
313
314 result, err := webhookrequest.VerifyAdmissionResponse(uid, true, response)
315 if err != nil {
316 return false, &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("received invalid webhook response: %w", err), Status: apierrors.NewServiceUnavailable("error validating webhook response")}
317 }
318
319 for k, v := range result.AuditAnnotations {
320 key := h.Name + "/" + k
321 if err := attr.Attributes.AddAnnotation(key, v); err != nil {
322 klog.Warningf("Failed to set admission audit annotation %s to %s for mutating webhook %s: %v", key, v, h.Name, err)
323 }
324 }
325 for _, w := range result.Warnings {
326 warning.AddWarning(ctx, "", w)
327 }
328
329 if !result.Allowed {
330 return false, &webhookutil.ErrWebhookRejection{Status: webhookerrors.ToStatusErr(h.Name, result.Result)}
331 }
332
333 if len(result.Patch) == 0 {
334 return false, nil
335 }
336 patchObj, err := jsonpatch.DecodePatch(result.Patch)
337 if err != nil {
338 return false, apierrors.NewInternalError(err)
339 }
340
341 if len(patchObj) == 0 {
342 return false, nil
343 }
344
345 // if a non-empty patch was provided, and we have no object we can apply it to (e.g. a DELETE admission operation), error
346 if attr.VersionedObject == nil {
347 return false, apierrors.NewInternalError(fmt.Errorf("admission webhook %q attempted to modify the object, which is not supported for this operation", h.Name))
348 }
349
350 var patchedJS []byte
351 jsonSerializer := json.NewSerializerWithOptions(json.DefaultMetaFactory, o.GetObjectCreater(), o.GetObjectTyper(), json.SerializerOptions{})
352 switch result.PatchType {
353 // VerifyAdmissionResponse normalizes to v1 patch types, regardless of the AdmissionReview version used
354 case admissionv1.PatchTypeJSONPatch:
355 objJS, err := runtime.Encode(jsonSerializer, attr.VersionedObject)
356 if err != nil {
357 return false, apierrors.NewInternalError(err)
358 }
359 patchedJS, err = patchObj.Apply(objJS)
360 if err != nil {
361 return false, apierrors.NewInternalError(err)
362 }
363 default:
364 return false, &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("unsupported patch type %q", result.PatchType), Status: webhookerrors.ToStatusErr(h.Name, result.Result)}
365 }
366
367 var newVersionedObject runtime.Object
368 if _, ok := attr.VersionedObject.(*unstructured.Unstructured); ok {
369 // Custom Resources don't have corresponding Go struct's.
370 // They are represented as Unstructured.
371 newVersionedObject = &unstructured.Unstructured{}
372 } else {
373 newVersionedObject, err = o.GetObjectCreater().New(attr.VersionedKind)
374 if err != nil {
375 return false, apierrors.NewInternalError(err)
376 }
377 }
378
379 // TODO: if we have multiple mutating webhooks, we can remember the json
380 // instead of encoding and decoding for each one.
381 if newVersionedObject, _, err = jsonSerializer.Decode(patchedJS, nil, newVersionedObject); err != nil {
382 return false, apierrors.NewInternalError(err)
383 }
384
385 changed = !apiequality.Semantic.DeepEqual(attr.VersionedObject, newVersionedObject)
386 span.AddEvent("Patch applied")
387 annotator.addPatchAnnotation(patchObj, result.PatchType)
388 attr.Dirty = true
389 attr.VersionedObject = newVersionedObject
390 o.GetObjectDefaulter().Default(attr.VersionedObject)
391 return changed, nil
392 }
393
394 type webhookAnnotator struct {
395 attr *admission.VersionedAttributes
396 failedOpenAnnotationKey string
397 patchAnnotationKey string
398 mutationAnnotationKey string
399 webhook string
400 configuration string
401 }
402
403 func newWebhookAnnotator(attr *admission.VersionedAttributes, round, idx int, webhook, configuration string) *webhookAnnotator {
404 return &webhookAnnotator{
405 attr: attr,
406 failedOpenAnnotationKey: fmt.Sprintf("%sround_%d_index_%d", MutationAuditAnnotationFailedOpenKeyPrefix, round, idx),
407 patchAnnotationKey: fmt.Sprintf("%sround_%d_index_%d", PatchAuditAnnotationPrefix, round, idx),
408 mutationAnnotationKey: fmt.Sprintf("%sround_%d_index_%d", MutationAuditAnnotationPrefix, round, idx),
409 webhook: webhook,
410 configuration: configuration,
411 }
412 }
413
414 func (w *webhookAnnotator) addFailedOpenAnnotation() {
415 if w.attr == nil || w.attr.Attributes == nil {
416 return
417 }
418 value := w.webhook
419 if err := w.attr.Attributes.AddAnnotation(w.failedOpenAnnotationKey, value); err != nil {
420 klog.Warningf("failed to set failed open annotation for mutating webhook key %s to %s: %v", w.failedOpenAnnotationKey, value, err)
421 }
422 }
423
424 func (w *webhookAnnotator) addMutationAnnotation(mutated bool) {
425 if w.attr == nil || w.attr.Attributes == nil {
426 return
427 }
428 value, err := mutationAnnotationValue(w.configuration, w.webhook, mutated)
429 if err != nil {
430 klog.Warningf("unexpected error composing mutating webhook annotation: %v", err)
431 return
432 }
433 if err := w.attr.Attributes.AddAnnotation(w.mutationAnnotationKey, value); err != nil {
434 klog.Warningf("failed to set mutation annotation for mutating webhook key %s to %s: %v", w.mutationAnnotationKey, value, err)
435 }
436 }
437
438 func (w *webhookAnnotator) addPatchAnnotation(patch interface{}, patchType admissionv1.PatchType) {
439 if w.attr == nil || w.attr.Attributes == nil {
440 return
441 }
442 var value string
443 var err error
444 switch patchType {
445 case admissionv1.PatchTypeJSONPatch:
446 value, err = jsonPatchAnnotationValue(w.configuration, w.webhook, patch)
447 if err != nil {
448 klog.Warningf("unexpected error composing mutating webhook JSON patch annotation: %v", err)
449 return
450 }
451 default:
452 klog.Warningf("unsupported patch type for mutating webhook annotation: %v", patchType)
453 return
454 }
455 if err := w.attr.Attributes.AddAnnotationWithLevel(w.patchAnnotationKey, value, auditinternal.LevelRequest); err != nil {
456 // NOTE: we don't log actual patch in kube-apiserver log to avoid potentially
457 // leaking information
458 klog.Warningf("failed to set patch annotation for mutating webhook key %s; confugiration name: %s, webhook name: %s", w.patchAnnotationKey, w.configuration, w.webhook)
459 }
460 }
461
462 // MutationAuditAnnotation logs if a webhook invocation mutated the request object
463 type MutationAuditAnnotation struct {
464 Configuration string `json:"configuration"`
465 Webhook string `json:"webhook"`
466 Mutated bool `json:"mutated"`
467 }
468
469 // PatchAuditAnnotation logs a patch from a mutating webhook
470 type PatchAuditAnnotation struct {
471 Configuration string `json:"configuration"`
472 Webhook string `json:"webhook"`
473 Patch interface{} `json:"patch,omitempty"`
474 PatchType string `json:"patchType,omitempty"`
475 }
476
477 func mutationAnnotationValue(configuration, webhook string, mutated bool) (string, error) {
478 m := MutationAuditAnnotation{
479 Configuration: configuration,
480 Webhook: webhook,
481 Mutated: mutated,
482 }
483 bytes, err := utiljson.Marshal(m)
484 return string(bytes), err
485 }
486
487 func jsonPatchAnnotationValue(configuration, webhook string, patch interface{}) (string, error) {
488 p := PatchAuditAnnotation{
489 Configuration: configuration,
490 Webhook: webhook,
491 Patch: patch,
492 PatchType: string(admissionv1.PatchTypeJSONPatch),
493 }
494 bytes, err := utiljson.Marshal(p)
495 return string(bytes), err
496 }