Debian Codegraph @

pkg/server/deleted_kinds.go

raw
1 /*
2 Copyright 2020 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package server
18
19 import (
20 "fmt"
21 "os"
22 "regexp"
23 "sort"
24 "strconv"
25 "strings"
26
27 "k8s.io/apimachinery/pkg/runtime"
28 "k8s.io/apimachinery/pkg/runtime/schema"
29 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
30 "k8s.io/apimachinery/pkg/util/sets"
31 apimachineryversion "k8s.io/apimachinery/pkg/util/version"
32 "k8s.io/apimachinery/pkg/version"
33 "k8s.io/apiserver/pkg/registry/rest"
34 serverstorage "k8s.io/apiserver/pkg/server/storage"
35 "k8s.io/klog/v2"
36 )
37
38 var alphaPattern = regexp.MustCompile(`^v\d+alpha\d+$`)
39
40 // resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
41 type resourceExpirationEvaluator struct {
42 currentVersion *apimachineryversion.Version
43 emulationForwardCompatible bool
44 runtimeConfigEmulationForwardCompatible bool
45 isAlpha bool
46 // Special flag checking for the existence of alpha.0
47 // alpha.0 is a special case where everything merged to master is auto propagated to the release-1.n branch
48 isAlphaZero bool
49 // This is usually set for testing for which tests need to be removed. This prevent insta-failing CI.
50 // Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta
51 // This flag only takes effect during alpha but not alphaZero.
52 strictRemovedHandlingInAlpha bool
53 // This is usually set by a cluster-admin looking for a short-term escape hatch after something bad happened.
54 // This should be made a flag before merge
55 // Set KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE to prevent removing APIs for one more release.
56 serveRemovedAPIsOneMoreRelease bool
57 }
58
59 // ResourceExpirationEvaluator indicates whether or not a resource should be served.
60 type ResourceExpirationEvaluator interface {
61 // RemoveUnavailableKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted or are introduced after the current version.
62 // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
63 RemoveUnavailableKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error
64 // ShouldServeForVersion returns true if a particular version cut off is after the current version
65 ShouldServeForVersion(majorRemoved, minorRemoved int) bool
66 }
67
68 type ResourceExpirationEvaluatorOptions struct {
69 // CurrentVersion is the current version of the apiserver.
70 CurrentVersion *apimachineryversion.Version
71 // Prerelease holds an optional prerelease portion of the version.
72 // This is used to determine if the current binary is an alpha.
73 Prerelease string
74 // EmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
75 // when resources of the same group and resource name but with lower priority are served.
76 // Not applicable to alpha APIs.
77 EmulationForwardCompatible bool
78 // RuntimeConfigEmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
79 // when the resource is explicitly enabled in runtime-config.
80 RuntimeConfigEmulationForwardCompatible bool
81 }
82
83 func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) {
84 opts := ResourceExpirationEvaluatorOptions{
85 CurrentVersion: apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor()),
86 Prerelease: currentVersion.PreRelease(),
87 }
88 return NewResourceExpirationEvaluatorFromOptions(opts)
89 }
90
91 func NewResourceExpirationEvaluatorFromOptions(opts ResourceExpirationEvaluatorOptions) (ResourceExpirationEvaluator, error) {
92 currentVersion := opts.CurrentVersion
93 if currentVersion == nil {
94 return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion")
95 }
96 klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion)
97 ret := &resourceExpirationEvaluator{
98 strictRemovedHandlingInAlpha: false,
99 emulationForwardCompatible: opts.EmulationForwardCompatible,
100 runtimeConfigEmulationForwardCompatible: opts.RuntimeConfigEmulationForwardCompatible,
101 }
102 // Only keeps the major and minor versions from input version.
103 ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor())
104 ret.isAlpha = strings.Contains(opts.Prerelease, "alpha")
105 ret.isAlphaZero = strings.Contains(opts.Prerelease, "alpha.0")
106
107 if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok {
108 // do nothing
109 } else if envBool, err := strconv.ParseBool(envString); err != nil {
110 return nil, err
111 } else {
112 ret.strictRemovedHandlingInAlpha = envBool
113 }
114
115 if envString, ok := os.LookupEnv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE"); !ok {
116 // do nothing
117 } else if envBool, err := strconv.ParseBool(envString); err != nil {
118 return nil, err
119 } else {
120 ret.serveRemovedAPIsOneMoreRelease = envBool
121 }
122
123 return ret, nil
124 }
125
126 // isNotRemoved checks if a resource is removed due to the APILifecycleRemoved information.
127 func (e *resourceExpirationEvaluator) isNotRemoved(gv schema.GroupVersion, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage) bool {
128 internalPtr := resourceServingInfo.New()
129
130 target := gv
131 // honor storage that overrides group version (used for things like scale subresources)
132 if versionProvider, ok := resourceServingInfo.(rest.GroupVersionKindProvider); ok {
133 target = versionProvider.GroupVersionKind(target).GroupVersion()
134 }
135
136 versionedPtr, err := versioner.ConvertToVersion(internalPtr, target)
137 if err != nil {
138 utilruntime.HandleError(err)
139 return false
140 }
141
142 removed, ok := versionedPtr.(removedInterface)
143 if !ok {
144 return true
145 }
146 majorRemoved, minorRemoved := removed.APILifecycleRemoved()
147 return e.ShouldServeForVersion(majorRemoved, minorRemoved)
148 }
149
150 func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool {
151 removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved))
152 if removedVer.GreaterThan(e.currentVersion) {
153 return true
154 }
155 if removedVer.LessThan(e.currentVersion) {
156 return false
157 }
158 // at this point major and minor are equal, so this API should be removed when the current release GAs.
159 // If this is an alpha tag, don't remove by default, but allow the option.
160 // If the cluster-admin has requested serving one more release, allow it.
161 if e.isAlpha && !e.isAlphaZero && e.strictRemovedHandlingInAlpha { // don't serve in alpha.1+ if we want strict handling
162 return false
163 }
164 if e.isAlpha { // alphas are allowed to continue serving expired betas while we clean up the test
165 return true
166 }
167 if e.serveRemovedAPIsOneMoreRelease { // cluster-admins are allowed to kick the can one release down the road
168 return true
169 }
170 return false
171 }
172
173 type removedInterface interface {
174 APILifecycleRemoved() (major, minor int)
175 }
176
177 // Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
178 type introducedInterface interface {
179 APILifecycleIntroduced() (major, minor int)
180 }
181
182 // removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
183 // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
184 func (e *resourceExpirationEvaluator) removeDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
185 versionsToRemove := sets.NewString()
186 for apiVersion := range sets.StringKeySet(versionedResourcesStorageMap) {
187 versionToResource := versionedResourcesStorageMap[apiVersion]
188 resourcesToRemove := sets.NewString()
189 for resourceName, resourceServingInfo := range versionToResource {
190 if !e.isNotRemoved(schema.GroupVersion{Group: groupName, Version: apiVersion}, versioner, resourceServingInfo) {
191 resourcesToRemove.Insert(resourceName)
192 }
193 }
194
195 for resourceName := range versionedResourcesStorageMap[apiVersion] {
196 if !shouldRemoveResourceAndSubresources(resourcesToRemove, resourceName) {
197 continue
198 }
199
200 klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName)
201 storage := versionToResource[resourceName]
202 storage.Destroy()
203 delete(versionToResource, resourceName)
204 }
205 versionedResourcesStorageMap[apiVersion] = versionToResource
206
207 if len(versionedResourcesStorageMap[apiVersion]) == 0 {
208 versionsToRemove.Insert(apiVersion)
209 }
210 }
211
212 for _, apiVersion := range versionsToRemove.List() {
213 klog.V(1).Infof("Removing version %v.%v because it is time to stop serving it because it has no resources per APILifecycle.", apiVersion, groupName)
214 delete(versionedResourcesStorageMap, apiVersion)
215 }
216 }
217
218 func (e *resourceExpirationEvaluator) RemoveUnavailableKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error {
219 e.removeDeletedKinds(groupName, versioner, versionedResourcesStorageMap)
220 return e.removeUnintroducedKinds(groupName, versioner, versionedResourcesStorageMap, apiResourceConfigSource)
221 }
222
223 // removeUnintroducedKinds inspects the storage map and modifies it in place by removing storage for kinds that are introduced after the current version.
224 // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
225 func (e *resourceExpirationEvaluator) removeUnintroducedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error {
226 versionsToRemove := sets.NewString()
227 prioritizedVersions := versioner.PrioritizedVersionsForGroup(groupName)
228 sort.Slice(prioritizedVersions, func(i, j int) bool {
229 return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
230 })
231 enabledResources := sets.NewString()
232
233 // iterate from the end to the front, so that we remove the lower priority versions first.
234 for i := len(prioritizedVersions) - 1; i >= 0; i-- {
235 apiVersion := prioritizedVersions[i].Version
236 versionToResource := versionedResourcesStorageMap[apiVersion]
237 if len(versionToResource) == 0 {
238 continue
239 }
240 resourcesToRemove := sets.NewString()
241 for resourceName, resourceServingInfo := range versionToResource {
242 // we check the resource enablement from low priority to high priority.
243 // If the same resource with a different version that we have checked so far is already enabled, that means some resource with the same resourceName and a lower priority version has been enabled.
244 // Then emulation forward compatibility for the version being checked now is made based on this information.
245 lowerPriorityEnabled := enabledResources.Has(resourceName)
246 shouldKeep, err := e.shouldServeBasedOnVersionIntroduced(schema.GroupVersionResource{Group: groupName, Version: apiVersion, Resource: resourceName},
247 versioner, resourceServingInfo, apiResourceConfigSource, lowerPriorityEnabled)
248 if err != nil {
249 return err
250 }
251 if !shouldKeep {
252 resourcesToRemove.Insert(resourceName)
253 } else if !alphaPattern.MatchString(apiVersion) {
254 // enabledResources is passed onto the next iteration to check the enablement of higher priority resources for emulation forward compatibility.
255 // But enablement alpha apis do not affect the enablement of other versions because emulation forward compatibility is not applicable to alpha apis.
256 enabledResources.Insert(resourceName)
257 }
258 }
259
260 for resourceName := range versionedResourcesStorageMap[apiVersion] {
261 if !shouldRemoveResourceAndSubresources(resourcesToRemove, resourceName) {
262 continue
263 }
264
265 klog.V(1).Infof("Removing resource %v.%v.%v because it is introduced after the current version %s per APILifecycle.", resourceName, apiVersion, groupName, e.currentVersion.String())
266 storage := versionToResource[resourceName]
267 storage.Destroy()
268 delete(versionToResource, resourceName)
269 }
270 versionedResourcesStorageMap[apiVersion] = versionToResource
271
272 if len(versionedResourcesStorageMap[apiVersion]) == 0 {
273 versionsToRemove.Insert(apiVersion)
274 }
275 }
276
277 for _, apiVersion := range versionsToRemove.List() {
278 gv := schema.GroupVersion{Group: groupName, Version: apiVersion}
279 if apiResourceConfigSource != nil && apiResourceConfigSource.VersionExplicitlyEnabled(gv) {
280 return fmt.Errorf(
281 "cannot enable version %s in runtime-config because all the resources have been introduced after the current version %s. Consider setting --runtime-config-emulation-forward-compatible=true",
282 gv, e.currentVersion)
283 }
284 klog.V(1).Infof("Removing version %v.%v because it is introduced after the current version %s and because it has no resources per APILifecycle.", apiVersion, groupName, e.currentVersion.String())
285 delete(versionedResourcesStorageMap, apiVersion)
286 }
287 return nil
288 }
289
290 func (e *resourceExpirationEvaluator) shouldServeBasedOnVersionIntroduced(gvr schema.GroupVersionResource, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage,
291 apiResourceConfigSource serverstorage.APIResourceConfigSource, lowerPriorityEnabled bool) (bool, error) {
292 verIntroduced := apimachineryversion.MajorMinor(0, 0)
293 internalPtr := resourceServingInfo.New()
294
295 target := gvr.GroupVersion()
296 // honor storage that overrides group version (used for things like scale subresources)
297 if versionProvider, ok := resourceServingInfo.(rest.GroupVersionKindProvider); ok {
298 target = versionProvider.GroupVersionKind(target).GroupVersion()
299 }
300
301 versionedPtr, err := versioner.ConvertToVersion(internalPtr, target)
302 if err != nil {
303 utilruntime.HandleError(err)
304 return false, err
305 }
306
307 introduced, ok := versionedPtr.(introducedInterface)
308 if ok {
309 majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
310 verIntroduced = apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
311 }
312 // should serve resource introduced at or before the current version.
313 if e.currentVersion.AtLeast(verIntroduced) {
314 return true, nil
315 }
316 // the rest of the function is to determine if a resource introduced after current version should be served. (only applicable in emulation mode.)
317
318 // if a lower priority version of the resource has been enabled, the same resource with higher priority
319 // should also be enabled if emulationForwardCompatible = true.
320 if e.emulationForwardCompatible && lowerPriorityEnabled {
321 return true, nil
322 }
323 if apiResourceConfigSource == nil {
324 return false, nil
325 }
326 // could explicitly enable future resources in runtime-config forward compatible mode.
327 if e.runtimeConfigEmulationForwardCompatible && (apiResourceConfigSource.ResourceExplicitlyEnabled(gvr) || apiResourceConfigSource.VersionExplicitlyEnabled(gvr.GroupVersion())) {
328 return true, nil
329 }
330 // return error if a future resource is explicit enabled in runtime-config but runtimeConfigEmulationForwardCompatible is false.
331 if apiResourceConfigSource.ResourceExplicitlyEnabled(gvr) {
332 return false, fmt.Errorf("cannot enable resource %s in runtime-config because it is introduced at %s after the current version %s. Consider setting --runtime-config-emulation-forward-compatible=true",
333 gvr, verIntroduced, e.currentVersion)
334 }
335 return false, nil
336 }
337
338 func shouldRemoveResourceAndSubresources(resourcesToRemove sets.String, resourceName string) bool {
339 for _, resourceToRemove := range resourcesToRemove.List() {
340 if resourceName == resourceToRemove {
341 return true
342 }
343 // our API works on nesting, so you can have deployments, deployments/status, and deployments/scale. Not all subresources
344 // serve the parent type, but if the parent type (deployments in this case), has been removed, it's subresources should be removed too.
345 if strings.HasPrefix(resourceName, resourceToRemove+"/") {
346 return true
347 }
348 }
349 return false
350 }