Debian Codegraph @

pkg/endpoints/discovery/aggregated/handler.go

raw
1 /*
2 Copyright 2022 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package aggregated
18
19 import (
20 "fmt"
21 "net/http"
22 "reflect"
23 "sort"
24 "sync"
25
26 apidiscoveryv2 "k8s.io/api/apidiscovery/v2"
27 apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
28 "k8s.io/apimachinery/pkg/runtime/schema"
29 "k8s.io/apimachinery/pkg/runtime/serializer"
30 "k8s.io/apimachinery/pkg/version"
31 apidiscoveryv2conversion "k8s.io/apiserver/pkg/apis/apidiscovery/v2"
32 genericfeatures "k8s.io/apiserver/pkg/features"
33
34 "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
35
36 "k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
37 "k8s.io/apiserver/pkg/endpoints/metrics"
38
39 "sync/atomic"
40
41 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
42 "k8s.io/apimachinery/pkg/runtime"
43 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
44 utilfeature "k8s.io/apiserver/pkg/util/feature"
45 "k8s.io/klog/v2"
46 )
47
48 type Source uint
49
50 // The GroupVersion from the lowest Source takes precedence
51 const (
52 AggregatorSource Source = 0
53 BuiltinSource Source = 100
54 CRDSource Source = 200
55 )
56
57 // This handler serves the /apis endpoint for an aggregated list of
58 // api resources indexed by their group version.
59 type ResourceManager interface {
60 // Adds knowledge of the given groupversion to the discovery document
61 // If it was already being tracked, updates the stored APIVersionDiscovery
62 // Thread-safe
63 AddGroupVersion(groupName string, value apidiscoveryv2.APIVersionDiscovery)
64
65 // Sets a priority to be used while sorting a specific group and
66 // group-version. If two versions report different priorities for
67 // the group, the higher one will be used. If the group is not
68 // known, the priority is ignored. The priority for this version
69 // is forgotten once the group-version is forgotten
70 SetGroupVersionPriority(gv metav1.GroupVersion, grouppriority, versionpriority int)
71
72 // Removes all group versions for a given group
73 // Thread-safe
74 RemoveGroup(groupName string)
75
76 // Removes a specific groupversion. If all versions of a group have been
77 // removed, then the entire group is unlisted.
78 // Thread-safe
79 RemoveGroupVersion(gv metav1.GroupVersion)
80
81 // Resets the manager's known list of group-versions and replaces them
82 // with the given groups
83 // Thread-Safe
84 SetGroups([]apidiscoveryv2.APIGroupDiscovery)
85
86 // Returns the same resource manager using a different source
87 // The source is used to decide how to de-duplicate groups.
88 // The group from the least-numbered source is used
89 WithSource(source Source) ResourceManager
90
91 http.Handler
92 }
93
94 type resourceManager struct {
95 source Source
96 *resourceDiscoveryManager
97 }
98
99 func (rm resourceManager) AddGroupVersion(groupName string, value apidiscoveryv2.APIVersionDiscovery) {
100 rm.resourceDiscoveryManager.AddGroupVersion(rm.source, groupName, value)
101 }
102 func (rm resourceManager) SetGroupVersionPriority(gv metav1.GroupVersion, grouppriority, versionpriority int) {
103 rm.resourceDiscoveryManager.SetGroupVersionPriority(rm.source, gv, grouppriority, versionpriority)
104 }
105 func (rm resourceManager) RemoveGroup(groupName string) {
106 rm.resourceDiscoveryManager.RemoveGroup(rm.source, groupName)
107 }
108 func (rm resourceManager) RemoveGroupVersion(gv metav1.GroupVersion) {
109 rm.resourceDiscoveryManager.RemoveGroupVersion(rm.source, gv)
110 }
111 func (rm resourceManager) SetGroups(groups []apidiscoveryv2.APIGroupDiscovery) {
112 rm.resourceDiscoveryManager.SetGroups(rm.source, groups)
113 }
114
115 func (rm resourceManager) WithSource(source Source) ResourceManager {
116 return resourceManager{
117 source: source,
118 resourceDiscoveryManager: rm.resourceDiscoveryManager,
119 }
120 }
121
122 type groupKey struct {
123 name string
124
125 // Source identifies where this group came from and dictates which group
126 // among duplicates is chosen to be used for discovery.
127 source Source
128 }
129
130 type groupVersionKey struct {
131 metav1.GroupVersion
132 source Source
133 }
134
135 type resourceDiscoveryManager struct {
136 serializer runtime.NegotiatedSerializer
137 // cache is an atomic pointer to avoid the use of locks
138 cache atomic.Pointer[cachedGroupList]
139
140 serveHTTPFunc http.HandlerFunc
141
142 // Writes protected by the lock.
143 // List of all apigroups & resources indexed by the resource manager
144 lock sync.RWMutex
145 apiGroups map[groupKey]*apidiscoveryv2.APIGroupDiscovery
146 versionPriorities map[groupVersionKey]priorityInfo
147 }
148
149 type priorityInfo struct {
150 GroupPriorityMinimum int
151 VersionPriority int
152 }
153
154 func NewResourceManager(path string) ResourceManager {
155 scheme := runtime.NewScheme()
156 utilruntime.Must(apidiscoveryv2.AddToScheme(scheme))
157 utilruntime.Must(apidiscoveryv2beta1.AddToScheme(scheme))
158 // Register conversion for apidiscovery
159 utilruntime.Must(apidiscoveryv2conversion.RegisterConversions(scheme))
160
161 codecs := serializer.NewCodecFactory(scheme)
162 rdm := &resourceDiscoveryManager{
163 serializer: codecs,
164 versionPriorities: make(map[groupVersionKey]priorityInfo),
165 }
166 rdm.serveHTTPFunc = metrics.InstrumentHandlerFunc("GET",
167 /* group = */ "",
168 /* version = */ "",
169 /* resource = */ "",
170 /* subresource = */ path,
171 /* scope = */ "",
172 /* component = */ metrics.APIServerComponent,
173 /* deprecated */ false,
174 /* removedRelease */ "",
175 rdm.serveHTTP)
176 return resourceManager{
177 source: BuiltinSource,
178 resourceDiscoveryManager: rdm,
179 }
180 }
181
182 func (rdm *resourceDiscoveryManager) SetGroupVersionPriority(source Source, gv metav1.GroupVersion, groupPriorityMinimum, versionPriority int) {
183 rdm.lock.Lock()
184 defer rdm.lock.Unlock()
185
186 key := groupVersionKey{
187 GroupVersion: gv,
188 source: source,
189 }
190 rdm.versionPriorities[key] = priorityInfo{
191 GroupPriorityMinimum: groupPriorityMinimum,
192 VersionPriority: versionPriority,
193 }
194 rdm.cache.Store(nil)
195 }
196
197 func (rdm *resourceDiscoveryManager) SetGroups(source Source, groups []apidiscoveryv2.APIGroupDiscovery) {
198 rdm.lock.Lock()
199 defer rdm.lock.Unlock()
200
201 rdm.apiGroups = nil
202 rdm.cache.Store(nil)
203
204 for _, group := range groups {
205 for _, version := range group.Versions {
206 rdm.addGroupVersionLocked(source, group.Name, version)
207 }
208 }
209
210 // Filter unused out priority entries
211 for gv := range rdm.versionPriorities {
212 key := groupKey{
213 source: source,
214 name: gv.Group,
215 }
216 entry, exists := rdm.apiGroups[key]
217 if !exists {
218 delete(rdm.versionPriorities, gv)
219 continue
220 }
221
222 containsVersion := false
223
224 for _, v := range entry.Versions {
225 if v.Version == gv.Version {
226 containsVersion = true
227 break
228 }
229 }
230
231 if !containsVersion {
232 delete(rdm.versionPriorities, gv)
233 }
234 }
235 }
236
237 func (rdm *resourceDiscoveryManager) AddGroupVersion(source Source, groupName string, value apidiscoveryv2.APIVersionDiscovery) {
238 rdm.lock.Lock()
239 defer rdm.lock.Unlock()
240
241 rdm.addGroupVersionLocked(source, groupName, value)
242 }
243
244 func (rdm *resourceDiscoveryManager) addGroupVersionLocked(source Source, groupName string, value apidiscoveryv2.APIVersionDiscovery) {
245
246 if rdm.apiGroups == nil {
247 rdm.apiGroups = make(map[groupKey]*apidiscoveryv2.APIGroupDiscovery)
248 }
249
250 key := groupKey{
251 source: source,
252 name: groupName,
253 }
254
255 if existing, groupExists := rdm.apiGroups[key]; groupExists {
256 // If this version already exists, replace it
257 versionExists := false
258
259 // Not very efficient, but in practice there are generally not many versions
260 for i := range existing.Versions {
261 if existing.Versions[i].Version == value.Version {
262 // The new gv is the exact same as what is already in
263 // the map. This is a noop and cache should not be
264 // invalidated.
265 if reflect.DeepEqual(existing.Versions[i], value) {
266 return
267 }
268
269 existing.Versions[i] = value
270 versionExists = true
271 break
272 }
273 }
274
275 if !versionExists {
276 existing.Versions = append(existing.Versions, value)
277 }
278
279 } else {
280 group := &apidiscoveryv2.APIGroupDiscovery{
281 ObjectMeta: metav1.ObjectMeta{
282 Name: groupName,
283 },
284 Versions: []apidiscoveryv2.APIVersionDiscovery{value},
285 }
286 rdm.apiGroups[key] = group
287 }
288 klog.Infof("Adding GroupVersion %s %s to ResourceManager", groupName, value.Version)
289
290 gv := metav1.GroupVersion{Group: groupName, Version: value.Version}
291 gvKey := groupVersionKey{
292 GroupVersion: gv,
293 source: source,
294 }
295 if _, ok := rdm.versionPriorities[gvKey]; !ok {
296 rdm.versionPriorities[gvKey] = priorityInfo{
297 GroupPriorityMinimum: 1000,
298 VersionPriority: 15,
299 }
300 }
301
302 // Reset response document so it is recreated lazily
303 rdm.cache.Store(nil)
304 }
305
306 func (rdm *resourceDiscoveryManager) RemoveGroupVersion(source Source, apiGroup metav1.GroupVersion) {
307 rdm.lock.Lock()
308 defer rdm.lock.Unlock()
309
310 key := groupKey{
311 source: source,
312 name: apiGroup.Group,
313 }
314
315 group, exists := rdm.apiGroups[key]
316 if !exists {
317 return
318 }
319
320 modified := false
321 for i := range group.Versions {
322 if group.Versions[i].Version == apiGroup.Version {
323 group.Versions = append(group.Versions[:i], group.Versions[i+1:]...)
324 modified = true
325 break
326 }
327 }
328 // If no modification was done, cache does not need to be cleared
329 if !modified {
330 return
331 }
332
333 gvKey := groupVersionKey{
334 GroupVersion: apiGroup,
335 source: source,
336 }
337
338 delete(rdm.versionPriorities, gvKey)
339 if len(group.Versions) == 0 {
340 delete(rdm.apiGroups, key)
341 }
342
343 // Reset response document so it is recreated lazily
344 rdm.cache.Store(nil)
345 }
346
347 func (rdm *resourceDiscoveryManager) RemoveGroup(source Source, groupName string) {
348 rdm.lock.Lock()
349 defer rdm.lock.Unlock()
350
351 key := groupKey{
352 source: source,
353 name: groupName,
354 }
355
356 delete(rdm.apiGroups, key)
357
358 for k := range rdm.versionPriorities {
359 if k.Group == groupName && k.source == source {
360 delete(rdm.versionPriorities, k)
361 }
362 }
363
364 // Reset response document so it is recreated lazily
365 rdm.cache.Store(nil)
366 }
367
368 // Prepares the api group list for serving by converting them from map into
369 // list and sorting them according to insertion order
370 func (rdm *resourceDiscoveryManager) calculateAPIGroupsLocked() []apidiscoveryv2.APIGroupDiscovery {
371 regenerationCounter.Inc()
372 // Re-order the apiGroups by their priority.
373 groups := []apidiscoveryv2.APIGroupDiscovery{}
374
375 groupsToUse := map[string]apidiscoveryv2.APIGroupDiscovery{}
376 sourcesUsed := map[metav1.GroupVersion]Source{}
377
378 for key, group := range rdm.apiGroups {
379 if existing, ok := groupsToUse[key.name]; ok {
380 for _, v := range group.Versions {
381 gv := metav1.GroupVersion{Group: key.name, Version: v.Version}
382
383 // Skip groupversions we've already seen before. Only DefaultSource
384 // takes precedence
385 if usedSource, seen := sourcesUsed[gv]; seen && key.source >= usedSource {
386 continue
387 } else if seen {
388 // Find the index of the duplicate version and replace
389 for i := 0; i < len(existing.Versions); i++ {
390 if existing.Versions[i].Version == v.Version {
391 existing.Versions[i] = v
392 break
393 }
394 }
395
396 } else {
397 // New group-version, just append
398 existing.Versions = append(existing.Versions, v)
399 }
400
401 sourcesUsed[gv] = key.source
402 groupsToUse[key.name] = existing
403 }
404 // Check to see if we have overlapping versions. If we do, take the one
405 // with highest source precedence
406 } else {
407 groupsToUse[key.name] = *group.DeepCopy()
408 for _, v := range group.Versions {
409 gv := metav1.GroupVersion{Group: key.name, Version: v.Version}
410 sourcesUsed[gv] = key.source
411 }
412 }
413 }
414
415 for _, group := range groupsToUse {
416
417 // Re-order versions based on their priority. Use kube-aware string
418 // comparison as a tie breaker
419 sort.SliceStable(group.Versions, func(i, j int) bool {
420 iVersion := group.Versions[i].Version
421 jVersion := group.Versions[j].Version
422
423 iGV := metav1.GroupVersion{Group: group.Name, Version: iVersion}
424 jGV := metav1.GroupVersion{Group: group.Name, Version: jVersion}
425
426 iSource := sourcesUsed[iGV]
427 jSource := sourcesUsed[jGV]
428
429 iPriority := rdm.versionPriorities[groupVersionKey{iGV, iSource}].VersionPriority
430 jPriority := rdm.versionPriorities[groupVersionKey{jGV, jSource}].VersionPriority
431
432 // Sort by version string comparator if priority is equal
433 if iPriority == jPriority {
434 return version.CompareKubeAwareVersionStrings(iVersion, jVersion) > 0
435 }
436
437 // i sorts before j if it has a higher priority
438 return iPriority > jPriority
439 })
440
441 groups = append(groups, group)
442 }
443
444 // For each group, determine the highest minimum group priority and use that
445 priorities := map[string]int{}
446 for gv, info := range rdm.versionPriorities {
447 if source := sourcesUsed[gv.GroupVersion]; source != gv.source {
448 continue
449 }
450
451 if existing, exists := priorities[gv.Group]; exists {
452 if existing < info.GroupPriorityMinimum {
453 priorities[gv.Group] = info.GroupPriorityMinimum
454 }
455 } else {
456 priorities[gv.Group] = info.GroupPriorityMinimum
457 }
458 }
459
460 sort.SliceStable(groups, func(i, j int) bool {
461 iName := groups[i].Name
462 jName := groups[j].Name
463
464 // Default to 0 priority by default
465 iPriority := priorities[iName]
466 jPriority := priorities[jName]
467
468 // Sort discovery based on apiservice priority.
469 // Duplicated from staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helpers.go
470 if iPriority == jPriority {
471 // Equal priority uses name to break ties
472 return iName < jName
473 }
474
475 // i sorts before j if it has a higher priority
476 return iPriority > jPriority
477 })
478
479 return groups
480 }
481
482 // Fetches from cache if it exists. If cache is empty, create it.
483 func (rdm *resourceDiscoveryManager) fetchFromCache() *cachedGroupList {
484 rdm.lock.RLock()
485 defer rdm.lock.RUnlock()
486
487 cacheLoad := rdm.cache.Load()
488 if cacheLoad != nil {
489 return cacheLoad
490 }
491 response := apidiscoveryv2.APIGroupDiscoveryList{
492 Items: rdm.calculateAPIGroupsLocked(),
493 }
494 etag, err := calculateETag(response)
495 if err != nil {
496 klog.Errorf("failed to calculate etag for discovery document: %s", etag)
497 etag = ""
498 }
499 cached := &cachedGroupList{
500 cachedResponse: response,
501 cachedResponseETag: etag,
502 }
503 rdm.cache.Store(cached)
504 return cached
505 }
506
507 type cachedGroupList struct {
508 cachedResponse apidiscoveryv2.APIGroupDiscoveryList
509 // etag is calculated based on a SHA hash of only the JSON object.
510 // A response via different Accept encodings (eg: protobuf, json) will
511 // yield the same etag. This is okay because Accept is part of the Vary header.
512 // Per RFC7231 a client must only cache a response etag pair if the header field
513 // matches as indicated by the Vary field. Thus, protobuf and json and other Accept
514 // encodings will not be cached as the same response despite having the same etag.
515 cachedResponseETag string
516 }
517
518 func (rdm *resourceDiscoveryManager) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
519 rdm.serveHTTPFunc(resp, req)
520 }
521
522 func (rdm *resourceDiscoveryManager) serveHTTP(resp http.ResponseWriter, req *http.Request) {
523 cache := rdm.fetchFromCache()
524 response := cache.cachedResponse
525 etag := cache.cachedResponseETag
526
527 mediaType, _, err := negotiation.NegotiateOutputMediaType(req, rdm.serializer, DiscoveryEndpointRestrictions)
528 if err != nil {
529 // Should never happen. wrapper.go will only proxy requests to this
530 // handler if the media type passes DiscoveryEndpointRestrictions
531 utilruntime.HandleError(err)
532 resp.WriteHeader(http.StatusInternalServerError)
533 return
534 }
535 var targetGV schema.GroupVersion
536 if mediaType.Convert == nil ||
537 (mediaType.Convert.GroupVersion() != apidiscoveryv2.SchemeGroupVersion &&
538 mediaType.Convert.GroupVersion() != apidiscoveryv2beta1.SchemeGroupVersion) {
539 utilruntime.HandleError(fmt.Errorf("expected aggregated discovery group version, got group: %s, version %s", mediaType.Convert.Group, mediaType.Convert.Version))
540 resp.WriteHeader(http.StatusInternalServerError)
541 return
542 }
543
544 if mediaType.Convert.GroupVersion() == apidiscoveryv2beta1.SchemeGroupVersion &&
545 utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryRemoveBetaType) {
546 klog.Errorf("aggregated discovery version v2beta1 is removed. Please update to use v2")
547 resp.WriteHeader(http.StatusNotFound)
548 return
549 }
550
551 targetGV = mediaType.Convert.GroupVersion()
552
553 if len(etag) > 0 {
554 // Use proper e-tag headers if one is available
555 ServeHTTPWithETag(
556 &response,
557 etag,
558 targetGV,
559 rdm.serializer,
560 resp,
561 req,
562 )
563 } else {
564 // Default to normal response in rare case etag is
565 // not cached with the object for some reason.
566 responsewriters.WriteObjectNegotiated(
567 rdm.serializer,
568 DiscoveryEndpointRestrictions,
569 targetGV,
570 resp,
571 req,
572 http.StatusOK,
573 &response,
574 true,
575 )
576 }
577 }