Debian Codegraph @

pkg/admission/plugin/policy/generic/policy_source.go

raw
1 /*
2 Copyright 2024 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package generic
18
19 import (
20 "context"
21 goerrors "errors"
22 "fmt"
23 "sync"
24 "sync/atomic"
25 "time"
26
27 corev1 "k8s.io/api/core/v1"
28 "k8s.io/apimachinery/pkg/api/errors"
29 "k8s.io/apimachinery/pkg/api/meta"
30 "k8s.io/apimachinery/pkg/labels"
31 "k8s.io/apimachinery/pkg/runtime"
32 "k8s.io/apimachinery/pkg/runtime/schema"
33 "k8s.io/apimachinery/pkg/types"
34 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
35 "k8s.io/apimachinery/pkg/util/wait"
36 "k8s.io/apiserver/pkg/admission/plugin/policy/internal/generic"
37 "k8s.io/client-go/dynamic"
38 "k8s.io/client-go/dynamic/dynamicinformer"
39 "k8s.io/client-go/informers"
40 "k8s.io/client-go/tools/cache"
41 "k8s.io/klog/v2"
42 )
43
44 // Interval for refreshing policies.
45 // TODO: Consider reducing this to a shorter duration or replacing this entirely
46 // with checks that detect when a policy change took effect.
47 const policyRefreshIntervalDefault = 1 * time.Second
48
49 var policyRefreshInterval = policyRefreshIntervalDefault
50
51 type policySource[P runtime.Object, B runtime.Object, E Evaluator] struct {
52 ctx context.Context
53 policyInformer generic.Informer[P]
54 bindingInformer generic.Informer[B]
55 restMapper meta.RESTMapper
56 newPolicyAccessor func(P) PolicyAccessor
57 newBindingAccessor func(B) BindingAccessor
58
59 informerFactory informers.SharedInformerFactory
60 dynamicClient dynamic.Interface
61
62 compiler func(P) E
63
64 // Currently compiled list of valid/active policy-binding pairs
65 policies atomic.Pointer[[]PolicyHook[P, B, E]]
66 // Whether the cache of policies is dirty and needs to be recompiled
67 policiesDirty atomic.Bool
68
69 lock sync.Mutex
70 compiledPolicies map[types.NamespacedName]compiledPolicyEntry[E]
71
72 // Temporary until we use the dynamic informer factory
73 paramsCRDControllers map[schema.GroupVersionKind]*paramInfo
74 }
75
76 type paramInfo struct {
77 mapping meta.RESTMapping
78
79 // When the param is changed, or the informer is done being used, the cancel
80 // func should be called to stop/cleanup the original informer
81 cancelFunc func()
82
83 // The lister for this param
84 informer informers.GenericInformer
85 }
86
87 type compiledPolicyEntry[E Evaluator] struct {
88 policyVersion string
89 evaluator E
90 }
91
92 type PolicyHook[P runtime.Object, B runtime.Object, E Evaluator] struct {
93 Policy P
94 Bindings []B
95
96 // ParamInformer is the informer for the param CRD for this policy, or nil if
97 // there is no param or if there was a configuration error
98 ParamInformer informers.GenericInformer
99 ParamScope meta.RESTScope
100
101 Evaluator E
102 ConfigurationError error
103 }
104
105 var _ Source[PolicyHook[runtime.Object, runtime.Object, Evaluator]] = &policySource[runtime.Object, runtime.Object, Evaluator]{}
106
107 func NewPolicySource[P runtime.Object, B runtime.Object, E Evaluator](
108 policyInformer cache.SharedIndexInformer,
109 bindingInformer cache.SharedIndexInformer,
110 newPolicyAccessor func(P) PolicyAccessor,
111 newBindingAccessor func(B) BindingAccessor,
112 compiler func(P) E,
113 paramInformerFactory informers.SharedInformerFactory,
114 dynamicClient dynamic.Interface,
115 restMapper meta.RESTMapper,
116 ) Source[PolicyHook[P, B, E]] {
117 res := &policySource[P, B, E]{
118 compiler: compiler,
119 policyInformer: generic.NewInformer[P](policyInformer),
120 bindingInformer: generic.NewInformer[B](bindingInformer),
121 compiledPolicies: map[types.NamespacedName]compiledPolicyEntry[E]{},
122 newPolicyAccessor: newPolicyAccessor,
123 newBindingAccessor: newBindingAccessor,
124 paramsCRDControllers: map[schema.GroupVersionKind]*paramInfo{},
125 informerFactory: paramInformerFactory,
126 dynamicClient: dynamicClient,
127 restMapper: restMapper,
128 }
129 return res
130 }
131
132 // SetPolicyRefreshIntervalForTests allows the refresh interval to be overridden during tests.
133 // This should only be called from tests.
134 func SetPolicyRefreshIntervalForTests(interval time.Duration) func() {
135 policyRefreshInterval = interval
136 return func() {
137 policyRefreshInterval = policyRefreshIntervalDefault
138 }
139 }
140
141 func (s *policySource[P, B, E]) Run(ctx context.Context) error {
142 if s.ctx != nil {
143 return fmt.Errorf("policy source already running")
144 }
145
146 // Wait for initial cache sync of policies and informers before reconciling
147 // any
148 if !cache.WaitForNamedCacheSync(fmt.Sprintf("%T", s), ctx.Done(), s.UpstreamHasSynced) {
149 err := ctx.Err()
150 if err == nil {
151 err = fmt.Errorf("initial cache sync for %T failed", s)
152 }
153 return err
154 }
155
156 s.ctx = ctx
157
158 // Perform initial policy compilation after initial list has finished
159 s.notify()
160 s.refreshPolicies()
161
162 notifyFuncs := cache.ResourceEventHandlerFuncs{
163 AddFunc: func(_ interface{}) {
164 s.notify()
165 },
166 UpdateFunc: func(_, _ interface{}) {
167 s.notify()
168 },
169 DeleteFunc: func(_ interface{}) {
170 s.notify()
171 },
172 }
173 handle, err := s.policyInformer.AddEventHandler(notifyFuncs)
174 if err != nil {
175 return err
176 }
177 defer func() {
178 if err := s.policyInformer.RemoveEventHandler(handle); err != nil {
179 utilruntime.HandleError(fmt.Errorf("failed to remove policy event handler: %w", err))
180 }
181 }()
182
183 bindingHandle, err := s.bindingInformer.AddEventHandler(notifyFuncs)
184 if err != nil {
185 return err
186 }
187 defer func() {
188 if err := s.bindingInformer.RemoveEventHandler(bindingHandle); err != nil {
189 utilruntime.HandleError(fmt.Errorf("failed to remove binding event handler: %w", err))
190 }
191 }()
192
193 // Start a worker that checks every second to see if policy data is dirty
194 // and needs to be recompiled
195 go func() {
196 // Loop every 1 second until context is cancelled, refreshing policies
197 wait.Until(s.refreshPolicies, policyRefreshInterval, ctx.Done())
198 }()
199
200 <-ctx.Done()
201 return nil
202 }
203
204 func (s *policySource[P, B, E]) UpstreamHasSynced() bool {
205 return s.policyInformer.HasSynced() && s.bindingInformer.HasSynced()
206 }
207
208 // HasSynced implements Source.
209 func (s *policySource[P, B, E]) HasSynced() bool {
210 // As an invariant we never store `nil` into the atomic list of processed
211 // policy hooks. If it is nil, then we haven't compiled all the policies
212 // and stored them yet.
213 return s.Hooks() != nil
214 }
215
216 // Hooks implements Source.
217 func (s *policySource[P, B, E]) Hooks() []PolicyHook[P, B, E] {
218 res := s.policies.Load()
219
220 // Error case should not happen since evaluation function never
221 // returns error
222 if res == nil {
223 // Not yet synced
224 return nil
225 }
226
227 return *res
228 }
229
230 func (s *policySource[P, B, E]) refreshPolicies() {
231 if !s.UpstreamHasSynced() {
232 return
233 } else if !s.policiesDirty.Swap(false) {
234 return
235 }
236
237 // It is ok the cache gets marked dirty again between us clearing the
238 // flag and us calculating the policies. The dirty flag would be marked again,
239 // and we'd have a no-op after comparing resource versions on the next sync.
240 klog.Infof("refreshing policies")
241 policies, err := s.calculatePolicyData()
242
243 // Intentionally store policy list regardless of error. There may be
244 // an error returned if there was a configuration error in one of the policies,
245 // but we would still want those policies evaluated
246 // (for instance to return error on failaction). Or if there was an error
247 // listing all policies at all, we would want to wipe the list.
248 s.policies.Store(&policies)
249
250 if err != nil {
251 // An error was generated while syncing policies. Mark it as dirty again
252 // so we can retry later
253 utilruntime.HandleError(fmt.Errorf("encountered error syncing policies: %w. Rescheduling policy sync", err))
254 s.notify()
255 }
256 }
257
258 func (s *policySource[P, B, E]) notify() {
259 s.policiesDirty.Store(true)
260 }
261
262 // calculatePolicyData calculates the list of policies and bindings for each
263 // policy. If there is an error in generation, it will return the error and
264 // the partial list of policies that were able to be generated. Policies that
265 // have an error will have a non-nil ConfigurationError field, but still be
266 // included in the result.
267 //
268 // This function caches the result of the intermediate compilations
269 func (s *policySource[P, B, E]) calculatePolicyData() ([]PolicyHook[P, B, E], error) {
270 if !s.UpstreamHasSynced() {
271 return nil, fmt.Errorf("cannot calculate policy data until upstream has synced")
272 }
273
274 // Fat-fingered lock that can be made more fine-tuned if required
275 s.lock.Lock()
276 defer s.lock.Unlock()
277
278 // Create a local copy of all policies and bindings
279 policiesToBindings := map[types.NamespacedName][]B{}
280 bindingList, err := s.bindingInformer.List(labels.Everything())
281 if err != nil {
282 // This should never happen unless types are misconfigured
283 // (can't use meta.accessor on them)
284 return nil, err
285 }
286
287 // Gather a list of all active policy bindings
288 for _, bindingSpec := range bindingList {
289 bindingAccessor := s.newBindingAccessor(bindingSpec)
290 policyKey := bindingAccessor.GetPolicyName()
291
292 // Add this binding to the list of bindings for this policy
293 policiesToBindings[policyKey] = append(policiesToBindings[policyKey], bindingSpec)
294 }
295
296 result := make([]PolicyHook[P, B, E], 0, len(bindingList))
297 usedParams := map[schema.GroupVersionKind]struct{}{}
298 var errs []error
299 for policyKey, bindingSpecs := range policiesToBindings {
300 var inf generic.NamespacedLister[P] = s.policyInformer
301 if len(policyKey.Namespace) > 0 {
302 inf = s.policyInformer.Namespaced(policyKey.Namespace)
303 }
304 policySpec, err := inf.Get(policyKey.Name)
305 if errors.IsNotFound(err) {
306 // Policy for bindings doesn't exist. This can happen if the policy
307 // was deleted before the binding, or the binding was created first.
308 //
309 // Just skip bindings that refer to non-existent policies
310 // If the policy is recreated, the cache will be marked dirty and
311 // this function will run again.
312 continue
313 } else if err != nil {
314 // This should never happen since fetching from a cache should never
315 // fail and this function checks that the cache was synced before
316 // even getting to this point.
317 errs = append(errs, err)
318 continue
319 }
320
321 var parsedParamKind *schema.GroupVersionKind
322 policyAccessor := s.newPolicyAccessor(policySpec)
323
324 if paramKind := policyAccessor.GetParamKind(); paramKind != nil {
325 groupVersion, err := schema.ParseGroupVersion(paramKind.APIVersion)
326 if err != nil {
327 errs = append(errs, fmt.Errorf("failed to parse paramKind APIVersion: %w", err))
328 continue
329 }
330 parsedParamKind = &schema.GroupVersionKind{
331 Group: groupVersion.Group,
332 Version: groupVersion.Version,
333 Kind: paramKind.Kind,
334 }
335
336 // TEMPORARY UNTIL WE HAVE SHARED PARAM INFORMERS
337 usedParams[*parsedParamKind] = struct{}{}
338 }
339
340 paramInformer, paramScope, configurationError := s.ensureParamsForPolicyLocked(parsedParamKind)
341 result = append(result, PolicyHook[P, B, E]{
342 Policy: policySpec,
343 Bindings: bindingSpecs,
344 Evaluator: s.compilePolicyLocked(policySpec),
345 ParamInformer: paramInformer,
346 ParamScope: paramScope,
347 ConfigurationError: configurationError,
348 })
349
350 // Should queue a re-sync for policy sync error. If our shared param
351 // informer can notify us when CRD discovery changes we can remove this
352 // and just rely on the informer to notify us when the CRDs change
353 if configurationError != nil {
354 errs = append(errs, configurationError)
355 }
356 }
357
358 // Clean up orphaned policies by replacing the old cache of compiled policies
359 // (the map of used policies is updated by `compilePolicy`)
360 for policyKey := range s.compiledPolicies {
361 if _, wasSeen := policiesToBindings[policyKey]; !wasSeen {
362 delete(s.compiledPolicies, policyKey)
363 }
364 }
365
366 // Clean up orphaned param informers
367 for paramKind, info := range s.paramsCRDControllers {
368 if _, wasSeen := usedParams[paramKind]; !wasSeen {
369 info.cancelFunc()
370 delete(s.paramsCRDControllers, paramKind)
371 }
372 }
373
374 err = nil
375 if len(errs) > 0 {
376 err = goerrors.Join(errs...)
377 }
378 return result, err
379 }
380
381 // ensureParamsForPolicyLocked ensures that the informer for the paramKind is
382 // started and returns the informer and the scope of the paramKind.
383 //
384 // Must be called under write lock
385 func (s *policySource[P, B, E]) ensureParamsForPolicyLocked(paramSource *schema.GroupVersionKind) (informers.GenericInformer, meta.RESTScope, error) {
386 if paramSource == nil {
387 return nil, nil, nil
388 } else if info, ok := s.paramsCRDControllers[*paramSource]; ok {
389 return info.informer, info.mapping.Scope, nil
390 }
391
392 mapping, err := s.restMapper.RESTMapping(schema.GroupKind{
393 Group: paramSource.Group,
394 Kind: paramSource.Kind,
395 }, paramSource.Version)
396
397 if err != nil {
398 // Failed to resolve. Return error so we retry again (rate limited)
399 // Save a record of this definition with an evaluator that unconditionally
400 return nil, nil, fmt.Errorf("failed to find resource referenced by paramKind: '%v'", *paramSource)
401 }
402
403 // We are not watching this param. Start an informer for it.
404 instanceContext, instanceCancel := context.WithCancel(s.ctx)
405
406 var informer informers.GenericInformer
407
408 // Try to see if our provided informer factory has an informer for this type.
409 // We assume the informer is already started, and starts all types associated
410 // with it.
411 if genericInformer, err := s.informerFactory.ForResource(mapping.Resource); err == nil {
412 informer = genericInformer
413
414 // Start the informer
415 s.informerFactory.Start(instanceContext.Done())
416
417 } else {
418 // Dynamic JSON informer fallback.
419 // Cannot use shared dynamic informer since it would be impossible
420 // to clean CRD informers properly with multiple dependents
421 // (cannot start ahead of time, and cannot track dependencies via stopCh)
422 informer = dynamicinformer.NewFilteredDynamicInformer(
423 s.dynamicClient,
424 mapping.Resource,
425 corev1.NamespaceAll,
426 // Use same interval as is used for k8s typed sharedInformerFactory
427 // https://github.com/kubernetes/kubernetes/blob/7e0923899fed622efbc8679cca6b000d43633e38/cmd/kube-apiserver/app/server.go#L430
428 10*time.Minute,
429 cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
430 nil,
431 )
432 go informer.Informer().Run(instanceContext.Done())
433 }
434
435 klog.Infof("informer started for %v", *paramSource)
436 ret := &paramInfo{
437 mapping: *mapping,
438 cancelFunc: instanceCancel,
439 informer: informer,
440 }
441 s.paramsCRDControllers[*paramSource] = ret
442 return ret.informer, mapping.Scope, nil
443 }
444
445 // For testing
446 func (s *policySource[P, B, E]) getParamInformer(param schema.GroupVersionKind) (informers.GenericInformer, meta.RESTScope) {
447 s.lock.Lock()
448 defer s.lock.Unlock()
449
450 if info, ok := s.paramsCRDControllers[param]; ok {
451 return info.informer, info.mapping.Scope
452 }
453
454 return nil, nil
455 }
456
457 // compilePolicyLocked compiles the policy and returns the evaluator for it.
458 // If the policy has not changed since the last compilation, it will return
459 // the cached evaluator.
460 //
461 // Must be called under write lock
462 func (s *policySource[P, B, E]) compilePolicyLocked(policySpec P) E {
463 policyMeta, err := meta.Accessor(policySpec)
464 if err != nil {
465 // This should not happen if P, and B have ObjectMeta, but
466 // unfortunately there is no way to express "able to call
467 // meta.Accessor" as a type constraint
468 utilruntime.HandleError(err)
469 var emptyEvaluator E
470 return emptyEvaluator
471 }
472 key := types.NamespacedName{
473 Namespace: policyMeta.GetNamespace(),
474 Name: policyMeta.GetName(),
475 }
476
477 compiledPolicy, wasCompiled := s.compiledPolicies[key]
478
479 // If the policy or binding has changed since it was last compiled,
480 // and if there is no configuration error (like a missing param CRD)
481 // then we recompile
482 if !wasCompiled ||
483 compiledPolicy.policyVersion != policyMeta.GetResourceVersion() {
484
485 compiledPolicy = compiledPolicyEntry[E]{
486 policyVersion: policyMeta.GetResourceVersion(),
487 evaluator: s.compiler(policySpec),
488 }
489 s.compiledPolicies[key] = compiledPolicy
490 }
491
492 return compiledPolicy.evaluator
493 }