Debian Codegraph @

pkg/admission/plugin/resourcequota/controller.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package resourcequota
18
19 import (
20 "fmt"
21 "sort"
22 "strings"
23 "sync"
24 "time"
25
26 "k8s.io/klog/v2"
27
28 corev1 "k8s.io/api/core/v1"
29 apierrors "k8s.io/apimachinery/pkg/api/errors"
30 "k8s.io/apimachinery/pkg/api/meta"
31 "k8s.io/apimachinery/pkg/runtime"
32 "k8s.io/apimachinery/pkg/runtime/schema"
33 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
34 "k8s.io/apimachinery/pkg/util/sets"
35 "k8s.io/apimachinery/pkg/util/wait"
36 "k8s.io/apiserver/pkg/admission"
37 resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota"
38 quota "k8s.io/apiserver/pkg/quota/v1"
39 "k8s.io/apiserver/pkg/quota/v1/generic"
40 "k8s.io/client-go/util/workqueue"
41 )
42
43 // Evaluator is used to see if quota constraints are satisfied.
44 type Evaluator interface {
45 // Evaluate takes an operation and checks to see if quota constraints are satisfied. It returns an error if they are not.
46 // The default implementation processes related operations in chunks when possible.
47 Evaluate(a admission.Attributes) error
48 }
49
50 type quotaEvaluator struct {
51 quotaAccessor QuotaAccessor
52 // lockAcquisitionFunc acquires any required locks and returns a cleanup method to defer
53 lockAcquisitionFunc func([]corev1.ResourceQuota) func()
54
55 ignoredResources map[schema.GroupResource]struct{}
56
57 // registry that knows how to measure usage for objects
58 registry quota.Registry
59
60 // TODO these are used together to bucket items by namespace and then batch them up for processing.
61 // The technique is valuable for rollup activities to avoid fanout and reduce resource contention.
62 // We could move this into a library if another component needed it.
63 // queue is indexed by namespace, so that we bundle up on a per-namespace basis
64 queue *workqueue.Typed[string]
65 workLock sync.Mutex
66 work map[string][]*admissionWaiter
67 dirtyWork map[string][]*admissionWaiter
68 inProgress sets.String
69
70 // controls the run method so that we can cleanly conform to the Evaluator interface
71 workers int
72 stopCh <-chan struct{}
73 init sync.Once
74
75 // lets us know what resources are limited by default
76 config *resourcequotaapi.Configuration
77 }
78
79 type admissionWaiter struct {
80 attributes admission.Attributes
81 finished chan struct{}
82 result error
83 }
84
85 type defaultDeny struct{}
86
87 func (defaultDeny) Error() string {
88 return "DEFAULT DENY"
89 }
90
91 // IsDefaultDeny returns true if the error is defaultDeny
92 func IsDefaultDeny(err error) bool {
93 if err == nil {
94 return false
95 }
96
97 _, ok := err.(defaultDeny)
98 return ok
99 }
100
101 func newAdmissionWaiter(a admission.Attributes) *admissionWaiter {
102 return &admissionWaiter{
103 attributes: a,
104 finished: make(chan struct{}),
105 result: defaultDeny{},
106 }
107 }
108
109 // NewQuotaEvaluator configures an admission controller that can enforce quota constraints
110 // using the provided registry. The registry must have the capability to handle group/kinds that
111 // are persisted by the server this admission controller is intercepting
112 func NewQuotaEvaluator(quotaAccessor QuotaAccessor, ignoredResources map[schema.GroupResource]struct{}, quotaRegistry quota.Registry, lockAcquisitionFunc func([]corev1.ResourceQuota) func(), config *resourcequotaapi.Configuration, workers int, stopCh <-chan struct{}) Evaluator {
113 // if we get a nil config, just create an empty default.
114 if config == nil {
115 config = &resourcequotaapi.Configuration{}
116 }
117
118 evaluator := &quotaEvaluator{
119 quotaAccessor: quotaAccessor,
120 lockAcquisitionFunc: lockAcquisitionFunc,
121
122 ignoredResources: ignoredResources,
123 registry: quotaRegistry,
124
125 queue: workqueue.NewTypedWithConfig(workqueue.TypedQueueConfig[string]{Name: "admission_quota_controller"}),
126 work: map[string][]*admissionWaiter{},
127 dirtyWork: map[string][]*admissionWaiter{},
128 inProgress: sets.String{},
129
130 workers: workers,
131 stopCh: stopCh,
132 config: config,
133 }
134
135 // The queue underneath is starting a goroutine for metrics
136 // exportint that is only stopped on calling ShutDown.
137 // Given that QuotaEvaluator is created for each layer of apiserver
138 // and often not started for some of those (e.g. aggregated apiserver)
139 // we explicitly shut it down on stopCh signal even if it wasn't
140 // effectively started.
141 go evaluator.shutdownOnStop()
142
143 return evaluator
144 }
145
146 // start begins watching and syncing.
147 func (e *quotaEvaluator) start() {
148 defer utilruntime.HandleCrash()
149
150 for i := 0; i < e.workers; i++ {
151 go wait.Until(e.doWork, time.Second, e.stopCh)
152 }
153 }
154
155 func (e *quotaEvaluator) shutdownOnStop() {
156 <-e.stopCh
157 klog.Infof("Shutting down quota evaluator")
158 e.queue.ShutDown()
159 }
160
161 func (e *quotaEvaluator) doWork() {
162 workFunc := func() bool {
163 ns, admissionAttributes, quit := e.getWork()
164 if quit {
165 return true
166 }
167 defer e.completeWork(ns)
168 if len(admissionAttributes) == 0 {
169 return false
170 }
171 e.checkAttributes(ns, admissionAttributes)
172 return false
173 }
174 for {
175 if quit := workFunc(); quit {
176 klog.Infof("quota evaluator worker shutdown")
177 return
178 }
179 }
180 }
181
182 // checkAttributes iterates evaluates all the waiting admissionAttributes. It will always notify all waiters
183 // before returning. The default is to deny.
184 func (e *quotaEvaluator) checkAttributes(ns string, admissionAttributes []*admissionWaiter) {
185 // notify all on exit
186 defer func() {
187 for _, admissionAttribute := range admissionAttributes {
188 close(admissionAttribute.finished)
189 }
190 }()
191
192 quotas, err := e.quotaAccessor.GetQuotas(ns)
193 if err != nil {
194 for _, admissionAttribute := range admissionAttributes {
195 admissionAttribute.result = err
196 }
197 return
198 }
199 // if limited resources are disabled, we can just return safely when there are no quotas.
200 limitedResourcesDisabled := len(e.config.LimitedResources) == 0
201 if len(quotas) == 0 && limitedResourcesDisabled {
202 for _, admissionAttribute := range admissionAttributes {
203 admissionAttribute.result = nil
204 }
205 return
206 }
207
208 if e.lockAcquisitionFunc != nil {
209 releaseLocks := e.lockAcquisitionFunc(quotas)
210 defer releaseLocks()
211 }
212
213 e.checkQuotas(quotas, admissionAttributes, 3)
214 }
215
216 // checkQuotas checks the admission attributes against the passed quotas. If a quota applies, it will attempt to update it
217 // AFTER it has checked all the admissionAttributes. The method breaks down into phase like this:
218 // 0. make a copy of the quotas to act as a "running" quota so we know what we need to update and can still compare against the
219 // originals
220 // 1. check each admission attribute to see if it fits within *all* the quotas. If it didn't fit, mark the waiter as failed
221 // and the running quota doesn't change. If it did fit, check to see if any quota was changed. If there was no quota change
222 // mark the waiter as succeeded. If some quota did change, update the running quotas
223 // 2. If no running quota was changed, return now since no updates are needed.
224 // 3. for each quota that has changed, attempt an update. If all updates succeeded, update all unset waiters to success status and return. If the some
225 // updates failed on conflict errors and we have retries left, re-get the failed quota from our cache for the latest version
226 // and recurse into this method with the subset. It's safe for us to evaluate ONLY the subset, because the other quota
227 // documents for these waiters have already been evaluated. Step 1, will mark all the ones that should already have succeeded.
228 func (e *quotaEvaluator) checkQuotas(quotas []corev1.ResourceQuota, admissionAttributes []*admissionWaiter, remainingRetries int) {
229 // yet another copy to compare against originals to see if we actually have deltas
230 originalQuotas, err := copyQuotas(quotas)
231 if err != nil {
232 utilruntime.HandleError(err)
233 return
234 }
235
236 atLeastOneChanged := false
237 for i := range admissionAttributes {
238 admissionAttribute := admissionAttributes[i]
239 newQuotas, err := e.checkRequest(quotas, admissionAttribute.attributes)
240 if err != nil {
241 admissionAttribute.result = err
242 continue
243 }
244
245 // Don't update quota for admissionAttributes that correspond to dry-run requests
246 if admissionAttribute.attributes.IsDryRun() {
247 admissionAttribute.result = nil
248 continue
249 }
250
251 // if the new quotas are the same as the old quotas, then this particular one doesn't issue any updates
252 // that means that no quota docs applied, so it can get a pass
253 atLeastOneChangeForThisWaiter := false
254 for j := range newQuotas {
255 if !quota.Equals(quotas[j].Status.Used, newQuotas[j].Status.Used) {
256 atLeastOneChanged = true
257 atLeastOneChangeForThisWaiter = true
258 break
259 }
260 }
261
262 if !atLeastOneChangeForThisWaiter {
263 admissionAttribute.result = nil
264 }
265
266 quotas = newQuotas
267 }
268
269 // if none of the requests changed anything, there's no reason to issue an update, just fail them all now
270 if !atLeastOneChanged {
271 return
272 }
273
274 // now go through and try to issue updates. Things get a little weird here:
275 // 1. check to see if the quota changed. If not, skip.
276 // 2. if the quota changed and the update passes, be happy
277 // 3. if the quota changed and the update fails, add the original to a retry list
278 var updatedFailedQuotas []corev1.ResourceQuota
279 var lastErr error
280 for i := range quotas {
281 newQuota := quotas[i]
282
283 // if this quota didn't have its status changed, skip it
284 if quota.Equals(originalQuotas[i].Status.Used, newQuota.Status.Used) {
285 continue
286 }
287
288 if err := e.quotaAccessor.UpdateQuotaStatus(&newQuota); err != nil {
289 updatedFailedQuotas = append(updatedFailedQuotas, newQuota)
290 lastErr = err
291 }
292 }
293
294 if len(updatedFailedQuotas) == 0 {
295 // all the updates succeeded. At this point, anything with the default deny error was just waiting to
296 // get a successful update, so we can mark and notify
297 for _, admissionAttribute := range admissionAttributes {
298 if IsDefaultDeny(admissionAttribute.result) {
299 admissionAttribute.result = nil
300 }
301 }
302 return
303 }
304
305 // at this point, errors are fatal. Update all waiters without status to failed and return
306 if remainingRetries <= 0 {
307 for _, admissionAttribute := range admissionAttributes {
308 if IsDefaultDeny(admissionAttribute.result) {
309 admissionAttribute.result = lastErr
310 }
311 }
312 return
313 }
314
315 // this retry logic has the same bug that its possible to be checking against quota in a state that never actually exists where
316 // you've added a new documented, then updated an old one, your resource matches both and you're only checking one
317 // updates for these quota names failed. Get the current quotas in the namespace, compare by name, check to see if the
318 // resource versions have changed. If not, we're going to fall through an fail everything. If they all have, then we can try again
319 newQuotas, err := e.quotaAccessor.GetQuotas(quotas[0].Namespace)
320 if err != nil {
321 // this means that updates failed. Anything with a default deny error has failed and we need to let them know
322 for _, admissionAttribute := range admissionAttributes {
323 if IsDefaultDeny(admissionAttribute.result) {
324 admissionAttribute.result = lastErr
325 }
326 }
327 return
328 }
329
330 // this logic goes through our cache to find the new version of all quotas that failed update. If something has been removed
331 // it is skipped on this retry. After all, you removed it.
332 quotasToCheck := []corev1.ResourceQuota{}
333 for _, newQuota := range newQuotas {
334 for _, oldQuota := range updatedFailedQuotas {
335 if newQuota.Name == oldQuota.Name {
336 quotasToCheck = append(quotasToCheck, newQuota)
337 break
338 }
339 }
340 }
341 e.checkQuotas(quotasToCheck, admissionAttributes, remainingRetries-1)
342 }
343
344 func copyQuotas(in []corev1.ResourceQuota) ([]corev1.ResourceQuota, error) {
345 out := make([]corev1.ResourceQuota, 0, len(in))
346 for _, quota := range in {
347 out = append(out, *quota.DeepCopy())
348 }
349
350 return out, nil
351 }
352
353 // filterLimitedResourcesByGroupResource filters the input that match the specified groupResource
354 func filterLimitedResourcesByGroupResource(input []resourcequotaapi.LimitedResource, groupResource schema.GroupResource) []resourcequotaapi.LimitedResource {
355 result := []resourcequotaapi.LimitedResource{}
356 for i := range input {
357 limitedResource := input[i]
358 limitedGroupResource := schema.GroupResource{Group: limitedResource.APIGroup, Resource: limitedResource.Resource}
359 if limitedGroupResource == groupResource {
360 result = append(result, limitedResource)
361 }
362 }
363 return result
364 }
365
366 // limitedByDefault determines from the specified usage and limitedResources the set of resources names
367 // that must be present in a covering quota. It returns empty set if it was unable to determine if
368 // a resource was not limited by default.
369 func limitedByDefault(usage corev1.ResourceList, limitedResources []resourcequotaapi.LimitedResource) []corev1.ResourceName {
370 result := []corev1.ResourceName{}
371 for _, limitedResource := range limitedResources {
372 for k, v := range usage {
373 // if a resource is consumed, we need to check if it matches on the limited resource list.
374 if v.Sign() == 1 {
375 // if we get a match, we add it to limited set
376 for _, matchContain := range limitedResource.MatchContains {
377 if strings.Contains(string(k), matchContain) {
378 result = append(result, k)
379 break
380 }
381 }
382 }
383 }
384 }
385 return result
386 }
387
388 func getMatchedLimitedScopes(evaluator quota.Evaluator, inputObject runtime.Object, limitedResources []resourcequotaapi.LimitedResource) ([]corev1.ScopedResourceSelectorRequirement, error) {
389 scopes := []corev1.ScopedResourceSelectorRequirement{}
390 for _, limitedResource := range limitedResources {
391 matched, err := evaluator.MatchingScopes(inputObject, limitedResource.MatchScopes)
392 if err != nil {
393 klog.ErrorS(err, "Error while matching limited Scopes")
394 return []corev1.ScopedResourceSelectorRequirement{}, err
395 }
396 scopes = append(scopes, matched...)
397 }
398 return scopes, nil
399 }
400
401 // checkRequest verifies that the request does not exceed any quota constraint. it returns a copy of quotas not yet persisted
402 // that capture what the usage would be if the request succeeded. It return an error if there is insufficient quota to satisfy the request
403 func (e *quotaEvaluator) checkRequest(quotas []corev1.ResourceQuota, a admission.Attributes) ([]corev1.ResourceQuota, error) {
404 evaluator := e.registry.Get(a.GetResource().GroupResource())
405 if evaluator == nil {
406 return quotas, nil
407 }
408 return CheckRequest(quotas, a, evaluator, e.config.LimitedResources)
409 }
410
411 // CheckRequest is a static version of quotaEvaluator.checkRequest, possible to be called from outside.
412 func CheckRequest(quotas []corev1.ResourceQuota, a admission.Attributes, evaluator quota.Evaluator,
413 limited []resourcequotaapi.LimitedResource) ([]corev1.ResourceQuota, error) {
414 if !evaluator.Handles(a) {
415 return quotas, nil
416 }
417
418 // if we have limited resources enabled for this resource, always calculate usage
419 inputObject := a.GetObject()
420
421 // Check if object matches AdmissionConfiguration matchScopes
422 limitedScopes, err := getMatchedLimitedScopes(evaluator, inputObject, limited)
423 if err != nil {
424 return quotas, nil
425 }
426
427 // determine the set of resource names that must exist in a covering quota
428 limitedResourceNames := []corev1.ResourceName{}
429 limitedResources := filterLimitedResourcesByGroupResource(limited, a.GetResource().GroupResource())
430 if len(limitedResources) > 0 {
431 deltaUsage, err := evaluator.Usage(inputObject)
432 if err != nil {
433 return quotas, err
434 }
435 limitedResourceNames = limitedByDefault(deltaUsage, limitedResources)
436 }
437 limitedResourceNamesSet := quota.ToSet(limitedResourceNames)
438
439 // find the set of quotas that are pertinent to this request
440 // reject if we match the quota, but usage is not calculated yet
441 // reject if the input object does not satisfy quota constraints
442 // if there are no pertinent quotas, we can just return
443 interestingQuotaIndexes := []int{}
444 // track the cumulative set of resources that were required across all quotas
445 // this is needed to know if we have satisfied any constraints where consumption
446 // was limited by default.
447 restrictedResourcesSet := sets.String{}
448 restrictedScopes := []corev1.ScopedResourceSelectorRequirement{}
449 for i := range quotas {
450 resourceQuota := quotas[i]
451 scopeSelectors := getScopeSelectorsFromQuota(resourceQuota)
452 localRestrictedScopes, err := evaluator.MatchingScopes(inputObject, scopeSelectors)
453 if err != nil {
454 return nil, fmt.Errorf("error matching scopes of quota %s, err: %v", resourceQuota.Name, err)
455 }
456 restrictedScopes = append(restrictedScopes, localRestrictedScopes...)
457
458 match, err := evaluator.Matches(&resourceQuota, inputObject)
459 if err != nil {
460 klog.ErrorS(err, "Error occurred while matching resource quota against input object",
461 "resourceQuota", resourceQuota)
462 return quotas, err
463 }
464 if !match {
465 continue
466 }
467
468 hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
469 restrictedResources := evaluator.MatchingResources(hardResources)
470 if err := evaluator.Constraints(restrictedResources, inputObject); err != nil {
471 return nil, admission.NewForbidden(a, fmt.Errorf("failed quota: %s: %v", resourceQuota.Name, err))
472 }
473 if !hasUsageStats(&resourceQuota, restrictedResources) {
474 return nil, admission.NewForbidden(a, fmt.Errorf("status unknown for quota: %s, resources: %s", resourceQuota.Name, prettyPrintResourceNames(restrictedResources)))
475 }
476 interestingQuotaIndexes = append(interestingQuotaIndexes, i)
477 localRestrictedResourcesSet := quota.ToSet(restrictedResources)
478 restrictedResourcesSet.Insert(localRestrictedResourcesSet.List()...)
479 }
480
481 // Usage of some resources cannot be counted in isolation. For example, when
482 // the resource represents a number of unique references to external
483 // resource. In such a case an evaluator needs to process other objects in
484 // the same namespace which needs to be known.
485 namespace := a.GetNamespace()
486 if accessor, err := meta.Accessor(inputObject); namespace != "" && err == nil {
487 if accessor.GetNamespace() == "" {
488 accessor.SetNamespace(namespace)
489 }
490 }
491 // there is at least one quota that definitely matches our object
492 // as a result, we need to measure the usage of this object for quota
493 // on updates, we need to subtract the previous measured usage
494 // if usage shows no change, just return since it has no impact on quota
495 inputUsage, err := evaluator.Usage(inputObject)
496 if err != nil {
497 return quotas, err
498 }
499
500 // ensure that usage for input object is never negative (this would mean a resource made a negative resource requirement)
501 if negativeUsage := quota.IsNegative(inputUsage); len(negativeUsage) > 0 {
502 return nil, admission.NewForbidden(a, fmt.Errorf("quota usage is negative for resource(s): %s", prettyPrintResourceNames(negativeUsage)))
503 }
504
505 // initialize a map of delta usage for each interesting quota index.
506 deltaUsageIndexMap := make(map[int]corev1.ResourceList, len(interestingQuotaIndexes))
507 for _, index := range interestingQuotaIndexes {
508 deltaUsageIndexMap[index] = inputUsage
509 }
510 var deltaUsageWhenNoInterestingQuota corev1.ResourceList
511 if admission.Create == a.GetOperation() && len(interestingQuotaIndexes) == 0 {
512 deltaUsageWhenNoInterestingQuota = inputUsage
513 }
514
515 if admission.Update == a.GetOperation() {
516 prevItem := a.GetOldObject()
517 if prevItem == nil {
518 return nil, admission.NewForbidden(a, fmt.Errorf("unable to get previous usage since prior version of object was not found"))
519 }
520
521 // if we can definitively determine that this is not a case of "create on update",
522 // then charge based on the delta. Otherwise, bill the maximum
523 metadata, err := meta.Accessor(prevItem)
524 if err == nil {
525 if len(metadata.GetResourceVersion()) > 0 {
526 prevUsage, innerErr := evaluator.Usage(prevItem)
527 if innerErr != nil {
528 return quotas, innerErr
529 }
530
531 deltaUsage := quota.SubtractWithNonNegativeResult(inputUsage, prevUsage)
532 if len(interestingQuotaIndexes) == 0 {
533 deltaUsageWhenNoInterestingQuota = deltaUsage
534 }
535
536 for _, index := range interestingQuotaIndexes {
537 resourceQuota := quotas[index]
538 match, err := evaluator.Matches(&resourceQuota, prevItem)
539 if err != nil {
540 klog.ErrorS(err, "Error occurred while matching resource quota against the existing object",
541 "resourceQuota", resourceQuota)
542 return quotas, err
543 }
544 if match {
545 deltaUsageIndexMap[index] = deltaUsage
546 }
547 }
548 } else if len(interestingQuotaIndexes) == 0 {
549 deltaUsageWhenNoInterestingQuota = inputUsage
550 }
551 }
552 }
553
554 // ignore items in deltaUsageIndexMap with zero usage,
555 // as they will not impact the quota.
556 for index := range deltaUsageIndexMap {
557 deltaUsageIndexMap[index] = quota.RemoveZeros(deltaUsageIndexMap[index])
558 if len(deltaUsageIndexMap[index]) == 0 {
559 delete(deltaUsageIndexMap, index)
560 }
561 }
562
563 // if there is no remaining non-zero usage, short-circuit and return
564 if len(interestingQuotaIndexes) != 0 {
565 if len(deltaUsageIndexMap) == 0 {
566 return quotas, nil
567 }
568 } else {
569 deltaUsage := quota.RemoveZeros(deltaUsageWhenNoInterestingQuota)
570 if len(deltaUsage) == 0 {
571 return quotas, nil
572 }
573 }
574
575 // verify that for every resource that had limited by default consumption
576 // enabled that there was a corresponding quota that covered its use.
577 // if not, we reject the request.
578 hasNoCoveringQuota := limitedResourceNamesSet.Difference(restrictedResourcesSet)
579 if len(hasNoCoveringQuota) > 0 {
580 return quotas, admission.NewForbidden(a, fmt.Errorf("insufficient quota to consume: %v", strings.Join(hasNoCoveringQuota.List(), ",")))
581 }
582
583 // verify that for every scope that had limited access enabled
584 // that there was a corresponding quota that covered it.
585 // if not, we reject the request.
586 scopesHasNoCoveringQuota, err := evaluator.UncoveredQuotaScopes(limitedScopes, restrictedScopes)
587 if err != nil {
588 return quotas, err
589 }
590 if len(scopesHasNoCoveringQuota) > 0 {
591 return quotas, fmt.Errorf("insufficient quota to match these scopes: %v", scopesHasNoCoveringQuota)
592 }
593
594 if len(interestingQuotaIndexes) == 0 {
595 return quotas, nil
596 }
597
598 outQuotas, err := copyQuotas(quotas)
599 if err != nil {
600 return nil, err
601 }
602
603 for _, index := range interestingQuotaIndexes {
604 resourceQuota := outQuotas[index]
605 deltaUsage, ok := deltaUsageIndexMap[index]
606 if !ok {
607 continue
608 }
609
610 hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
611 requestedUsage := quota.Mask(deltaUsage, hardResources)
612 newUsage := quota.Add(resourceQuota.Status.Used, requestedUsage)
613
614 if a.GetSubresource() != "status" {
615 maskedNewUsage := quota.Mask(newUsage, quota.ResourceNames(requestedUsage))
616
617 if allowed, exceeded := quota.LessThanOrEqual(maskedNewUsage, resourceQuota.Status.Hard); !allowed {
618 failedRequestedUsage := quota.Mask(requestedUsage, exceeded)
619 failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded)
620 failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded)
621 return nil, admission.NewForbidden(a,
622 fmt.Errorf("exceeded quota: %s, requested: %s, used: %s, limited: %s",
623 resourceQuota.Name,
624 prettyPrint(failedRequestedUsage),
625 prettyPrint(failedUsed),
626 prettyPrint(failedHard)))
627 }
628 }
629
630 // update to the new usage number
631 outQuotas[index].Status.Used = newUsage
632 }
633
634 return outQuotas, nil
635 }
636
637 func getScopeSelectorsFromQuota(quota corev1.ResourceQuota) []corev1.ScopedResourceSelectorRequirement {
638 selectors := []corev1.ScopedResourceSelectorRequirement{}
639 for _, scope := range quota.Spec.Scopes {
640 selectors = append(selectors, corev1.ScopedResourceSelectorRequirement{
641 ScopeName: scope,
642 Operator: corev1.ScopeSelectorOpExists})
643 }
644 if quota.Spec.ScopeSelector != nil {
645 selectors = append(selectors, quota.Spec.ScopeSelector.MatchExpressions...)
646 }
647 return selectors
648 }
649
650 func (e *quotaEvaluator) Evaluate(a admission.Attributes) error {
651 e.init.Do(e.start)
652
653 // is this resource ignored?
654 gvr := a.GetResource()
655 gr := gvr.GroupResource()
656 if _, ok := e.ignoredResources[gr]; ok {
657 return nil
658 }
659
660 // if we do not know how to evaluate use for this resource, create an evaluator
661 evaluator := e.registry.Get(gr)
662 if evaluator == nil {
663 // create an object count evaluator if no evaluator previously registered
664 // note, we do not need aggregate usage here, so we pass a nil informer func
665 evaluator = generic.NewObjectCountEvaluator(gr, nil, "")
666 e.registry.Add(evaluator)
667 klog.Infof("quota admission added evaluator for: %s", gr)
668 }
669 // for this kind, check if the operation could mutate any quota resources
670 // if no resources tracked by quota are impacted, then just return
671 if !evaluator.Handles(a) {
672 return nil
673 }
674 waiter := newAdmissionWaiter(a)
675
676 e.addWork(waiter)
677
678 // wait for completion or timeout
679 select {
680 case <-waiter.finished:
681 case <-time.After(10 * time.Second):
682 return apierrors.NewInternalError(fmt.Errorf("resource quota evaluation timed out"))
683 }
684
685 return waiter.result
686 }
687
688 func (e *quotaEvaluator) addWork(a *admissionWaiter) {
689 e.workLock.Lock()
690 defer e.workLock.Unlock()
691
692 ns := a.attributes.GetNamespace()
693 // this Add can trigger a Get BEFORE the work is added to a list, but this is ok because the getWork routine
694 // waits the worklock before retrieving the work to do, so the writes in this method will be observed
695 e.queue.Add(ns)
696
697 if e.inProgress.Has(ns) {
698 e.dirtyWork[ns] = append(e.dirtyWork[ns], a)
699 return
700 }
701
702 e.work[ns] = append(e.work[ns], a)
703 }
704
705 func (e *quotaEvaluator) completeWork(ns string) {
706 e.workLock.Lock()
707 defer e.workLock.Unlock()
708
709 e.queue.Done(ns)
710 e.work[ns] = e.dirtyWork[ns]
711 delete(e.dirtyWork, ns)
712 e.inProgress.Delete(ns)
713 }
714
715 // getWork returns a namespace, a list of work items in that
716 // namespace, and a shutdown boolean. If not shutdown then the return
717 // must eventually be followed by a call on completeWork for the
718 // returned namespace (regardless of whether the work item list is
719 // empty).
720 func (e *quotaEvaluator) getWork() (string, []*admissionWaiter, bool) {
721 ns, shutdown := e.queue.Get()
722 if shutdown {
723 return "", []*admissionWaiter{}, shutdown
724 }
725
726 e.workLock.Lock()
727 defer e.workLock.Unlock()
728 // at this point, we know we have a coherent view of e.work. It is entirely possible
729 // that our workqueue has another item requeued to it, but we'll pick it up early. This ok
730 // because the next time will go into our dirty list
731
732 work := e.work[ns]
733 delete(e.work, ns)
734 delete(e.dirtyWork, ns)
735 e.inProgress.Insert(ns)
736 return ns, work, false
737 }
738
739 // prettyPrint formats a resource list for usage in errors
740 // it outputs resources sorted in increasing order
741 func prettyPrint(item corev1.ResourceList) string {
742 parts := []string{}
743 keys := []string{}
744 for key := range item {
745 keys = append(keys, string(key))
746 }
747 sort.Strings(keys)
748 for _, key := range keys {
749 value := item[corev1.ResourceName(key)]
750 constraint := key + "=" + value.String()
751 parts = append(parts, constraint)
752 }
753 return strings.Join(parts, ",")
754 }
755
756 func prettyPrintResourceNames(a []corev1.ResourceName) string {
757 values := []string{}
758 for _, value := range a {
759 values = append(values, string(value))
760 }
761 sort.Strings(values)
762 return strings.Join(values, ",")
763 }
764
765 // hasUsageStats returns true if for each hard constraint in interestingResources there is a value for its current usage
766 func hasUsageStats(resourceQuota *corev1.ResourceQuota, interestingResources []corev1.ResourceName) bool {
767 interestingSet := quota.ToSet(interestingResources)
768 for resourceName := range resourceQuota.Status.Hard {
769 if !interestingSet.Has(string(resourceName)) {
770 continue
771 }
772 if _, found := resourceQuota.Status.Used[resourceName]; !found {
773 return false
774 }
775 }
776 return true
777 }