| 1 |
/* |
| 2 |
Copyright 2021 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package request |
| 18 |
|
| 19 |
import ( |
| 20 |
"math" |
| 21 |
"net/http" |
| 22 |
|
| 23 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 24 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 25 |
apirequest "k8s.io/apiserver/pkg/endpoints/request" |
| 26 |
"k8s.io/apiserver/pkg/storage/cacher/delegator" |
| 27 |
"k8s.io/klog/v2" |
| 28 |
) |
| 29 |
|
| 30 |
func newListWorkEstimator(countFn objectCountGetterFunc, config *WorkEstimatorConfig, maxSeatsFn maxSeatsFunc) WorkEstimatorFunc { |
| 31 |
estimator := &listWorkEstimator{ |
| 32 |
config: config, |
| 33 |
countGetterFn: countFn, |
| 34 |
maxSeatsFn: maxSeatsFn, |
| 35 |
} |
| 36 |
return estimator.estimate |
| 37 |
} |
| 38 |
|
| 39 |
type listWorkEstimator struct { |
| 40 |
config *WorkEstimatorConfig |
| 41 |
countGetterFn objectCountGetterFunc |
| 42 |
maxSeatsFn maxSeatsFunc |
| 43 |
} |
| 44 |
|
| 45 |
func (e *listWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLevelName string) WorkEstimate { |
| 46 |
minSeats := e.config.MinimumSeats |
| 47 |
maxSeats := e.maxSeatsFn(priorityLevelName) |
| 48 |
if maxSeats == 0 || maxSeats > e.config.MaximumSeatsLimit { |
| 49 |
maxSeats = e.config.MaximumSeatsLimit |
| 50 |
} |
| 51 |
|
| 52 |
requestInfo, ok := apirequest.RequestInfoFrom(r.Context()) |
| 53 |
if !ok { |
| 54 |
// no RequestInfo should never happen, but to be on the safe side |
| 55 |
// let's return maximumSeats |
| 56 |
return WorkEstimate{InitialSeats: maxSeats} |
| 57 |
} |
| 58 |
|
| 59 |
if requestInfo.Name != "" { |
| 60 |
// Requests with metadata.name specified are usually executed as get |
| 61 |
// requests in storage layer so their width should be 1. |
| 62 |
// Example of such list requests: |
| 63 |
// /apis/certificates.k8s.io/v1/certificatesigningrequests?fieldSelector=metadata.name%3Dcsr-xxs4m |
| 64 |
// /api/v1/namespaces/test/configmaps?fieldSelector=metadata.name%3Dbig-deployment-1&limit=500&resourceVersion=0 |
| 65 |
return WorkEstimate{InitialSeats: minSeats} |
| 66 |
} |
| 67 |
|
| 68 |
query := r.URL.Query() |
| 69 |
listOptions := metav1.ListOptions{} |
| 70 |
if err := metav1.Convert_url_Values_To_v1_ListOptions(&query, &listOptions, nil); err != nil { |
| 71 |
klog.ErrorS(err, "Failed to convert options while estimating work for the list request") |
| 72 |
|
| 73 |
// This request is destined to fail in the validation layer, |
| 74 |
// return maximumSeats for this request to be consistent. |
| 75 |
return WorkEstimate{InitialSeats: maxSeats} |
| 76 |
} |
| 77 |
|
| 78 |
// For watch requests, we want to adjust the cost only if they explicitly request |
| 79 |
// sending initial events. |
| 80 |
if requestInfo.Verb == "watch" { |
| 81 |
if listOptions.SendInitialEvents == nil || !*listOptions.SendInitialEvents { |
| 82 |
return WorkEstimate{InitialSeats: e.config.MinimumSeats} |
| 83 |
} |
| 84 |
} |
| 85 |
// TODO: Check whether watchcache is enabled. |
| 86 |
result, err := delegator.ShouldDelegateListMeta(&listOptions, delegator.CacheWithoutSnapshots{}) |
| 87 |
if err != nil { |
| 88 |
return WorkEstimate{InitialSeats: maxSeats} |
| 89 |
} |
| 90 |
listFromStorage := result.ShouldDelegate |
| 91 |
isListFromCache := requestInfo.Verb == "watch" || !listFromStorage |
| 92 |
|
| 93 |
numStored, err := e.countGetterFn(key(requestInfo)) |
| 94 |
switch { |
| 95 |
case err == ObjectCountStaleErr: |
| 96 |
// object count going stale is indicative of degradation, so we should |
| 97 |
// be conservative here and allocate maximum seats to this list request. |
| 98 |
// NOTE: if a CRD is removed, its count will go stale first and then the |
| 99 |
// pruner will eventually remove the CRD from the cache. |
| 100 |
return WorkEstimate{InitialSeats: maxSeats} |
| 101 |
case err == ObjectCountNotFoundErr: |
| 102 |
// there are multiple scenarios in which we can see this error: |
| 103 |
// a. the type is truly unknown, a typo on the caller's part. |
| 104 |
// b. the count has gone stale for too long and the pruner |
| 105 |
// has removed the type from the cache. |
| 106 |
// c. the type is an aggregated resource that is served by a |
| 107 |
// different apiserver (thus its object count is not updated) |
| 108 |
// we don't have a way to distinguish between those situations. |
| 109 |
// However, in case c, the request is delegated to a different apiserver, |
| 110 |
// and thus its cost for our server is minimal. To avoid the situation |
| 111 |
// when aggregated API calls are overestimated, we allocate the minimum |
| 112 |
// possible seats (see #109106 as an example when being more conservative |
| 113 |
// led to problems). |
| 114 |
return WorkEstimate{InitialSeats: minSeats} |
| 115 |
case err != nil: |
| 116 |
// we should never be here since Get returns either ObjectCountStaleErr or |
| 117 |
// ObjectCountNotFoundErr, return maximumSeats to be on the safe side. |
| 118 |
klog.ErrorS(err, "Unexpected error from object count tracker") |
| 119 |
return WorkEstimate{InitialSeats: maxSeats} |
| 120 |
} |
| 121 |
|
| 122 |
limit := numStored |
| 123 |
if listOptions.Limit > 0 && listOptions.Limit < numStored { |
| 124 |
limit = listOptions.Limit |
| 125 |
} |
| 126 |
|
| 127 |
var estimatedObjectsToBeProcessed int64 |
| 128 |
|
| 129 |
switch { |
| 130 |
case isListFromCache: |
| 131 |
// TODO: For resources that implement indexes at the watchcache level, |
| 132 |
// we need to adjust the cost accordingly |
| 133 |
estimatedObjectsToBeProcessed = numStored |
| 134 |
case listOptions.FieldSelector != "" || listOptions.LabelSelector != "": |
| 135 |
estimatedObjectsToBeProcessed = numStored + limit |
| 136 |
default: |
| 137 |
estimatedObjectsToBeProcessed = 2 * limit |
| 138 |
} |
| 139 |
|
| 140 |
// for now, our rough estimate is to allocate one seat to each 100 obejcts that |
| 141 |
// will be processed by the list request. |
| 142 |
// we will come up with a different formula for the transformation function and/or |
| 143 |
// fine tune this number in future iteratons. |
| 144 |
seats := uint64(math.Ceil(float64(estimatedObjectsToBeProcessed) / e.config.ObjectsPerSeat)) |
| 145 |
|
| 146 |
// make sure we never return a seat of zero |
| 147 |
if seats < minSeats { |
| 148 |
seats = minSeats |
| 149 |
} |
| 150 |
if seats > maxSeats { |
| 151 |
seats = maxSeats |
| 152 |
} |
| 153 |
return WorkEstimate{InitialSeats: seats} |
| 154 |
} |
| 155 |
|
| 156 |
func key(requestInfo *apirequest.RequestInfo) string { |
| 157 |
groupResource := &schema.GroupResource{ |
| 158 |
Group: requestInfo.APIGroup, |
| 159 |
Resource: requestInfo.Resource, |
| 160 |
} |
| 161 |
return groupResource.String() |
| 162 |
} |