Debian Codegraph @

pkg/server/genericapiserver.go

raw
1 /*
2 Copyright 2014 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package server
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "net/http"
24 gpath "path"
25 "strings"
26 "sync"
27 "time"
28
29 systemd "github.com/coreos/go-systemd/v22/daemon"
30
31 "golang.org/x/time/rate"
32 apidiscoveryv2 "k8s.io/api/apidiscovery/v2"
33 "k8s.io/apimachinery/pkg/api/meta"
34 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
35 "k8s.io/apimachinery/pkg/runtime"
36 "k8s.io/apimachinery/pkg/runtime/schema"
37 "k8s.io/apimachinery/pkg/runtime/serializer"
38 "k8s.io/apimachinery/pkg/runtime/serializer/cbor"
39 "k8s.io/apimachinery/pkg/util/managedfields"
40 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
41 "k8s.io/apimachinery/pkg/util/sets"
42 "k8s.io/apimachinery/pkg/util/wait"
43 utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
44 "k8s.io/apiserver/pkg/admission"
45 "k8s.io/apiserver/pkg/audit"
46 "k8s.io/apiserver/pkg/authorization/authorizer"
47 genericapi "k8s.io/apiserver/pkg/endpoints"
48 "k8s.io/apiserver/pkg/endpoints/discovery"
49 discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
50 "k8s.io/apiserver/pkg/features"
51 "k8s.io/apiserver/pkg/registry/rest"
52 "k8s.io/apiserver/pkg/server/healthz"
53 "k8s.io/apiserver/pkg/server/routes"
54 "k8s.io/apiserver/pkg/storageversion"
55 utilfeature "k8s.io/apiserver/pkg/util/feature"
56 restclient "k8s.io/client-go/rest"
57 basecompatibility "k8s.io/component-base/compatibility"
58 "k8s.io/component-base/featuregate"
59 "k8s.io/klog/v2"
60 openapibuilder3 "k8s.io/kube-openapi/pkg/builder3"
61 openapicommon "k8s.io/kube-openapi/pkg/common"
62 "k8s.io/kube-openapi/pkg/handler"
63 "k8s.io/kube-openapi/pkg/handler3"
64 openapiutil "k8s.io/kube-openapi/pkg/util"
65 "k8s.io/kube-openapi/pkg/validation/spec"
66 )
67
68 // Info about an API group.
69 type APIGroupInfo struct {
70 PrioritizedVersions []schema.GroupVersion
71 // Info about the resources in this group. It's a map from version to resource to the storage.
72 VersionedResourcesStorageMap map[string]map[string]rest.Storage
73 // OptionsExternalVersion controls the APIVersion used for common objects in the
74 // schema like api.Status, api.DeleteOptions, and metav1.ListOptions. Other implementors may
75 // define a version "v1beta1" but want to use the Kubernetes "v1" internal objects.
76 // If nil, defaults to groupMeta.GroupVersion.
77 // TODO: Remove this when https://github.com/kubernetes/kubernetes/issues/19018 is fixed.
78 OptionsExternalVersion *schema.GroupVersion
79 // MetaGroupVersion defaults to "meta.k8s.io/v1" and is the scheme group version used to decode
80 // common API implementations like ListOptions. Future changes will allow this to vary by group
81 // version (for when the inevitable meta/v2 group emerges).
82 MetaGroupVersion *schema.GroupVersion
83
84 // Scheme includes all of the types used by this group and how to convert between them (or
85 // to convert objects from outside of this group that are accepted in this API).
86 // TODO: replace with interfaces
87 Scheme *runtime.Scheme
88 // NegotiatedSerializer controls how this group encodes and decodes data
89 NegotiatedSerializer runtime.NegotiatedSerializer
90 // ParameterCodec performs conversions for query parameters passed to API calls
91 ParameterCodec runtime.ParameterCodec
92
93 // StaticOpenAPISpec is the spec derived from the definitions of all resources installed together.
94 // It is set during InstallAPIGroups, InstallAPIGroup, and InstallLegacyAPIGroup.
95 StaticOpenAPISpec map[string]*spec.Schema
96 }
97
98 func (a *APIGroupInfo) destroyStorage() {
99 for _, stores := range a.VersionedResourcesStorageMap {
100 for _, store := range stores {
101 store.Destroy()
102 }
103 }
104 }
105
106 // GenericAPIServer contains state for a Kubernetes cluster api server.
107 type GenericAPIServer struct {
108 // discoveryAddresses is used to build cluster IPs for discovery.
109 discoveryAddresses discovery.Addresses
110
111 // LoopbackClientConfig is a config for a privileged loopback connection to the API server
112 LoopbackClientConfig *restclient.Config
113
114 // minRequestTimeout is how short the request timeout can be. This is used to build the RESTHandler
115 minRequestTimeout time.Duration
116
117 // ShutdownTimeout is the timeout used for server shutdown. This specifies the timeout before server
118 // gracefully shutdown returns.
119 ShutdownTimeout time.Duration
120
121 // legacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
122 // to InstallLegacyAPIGroup
123 legacyAPIGroupPrefixes sets.String
124
125 // admissionControl is used to build the RESTStorage that backs an API Group.
126 admissionControl admission.Interface
127
128 // SecureServingInfo holds configuration of the TLS server.
129 SecureServingInfo *SecureServingInfo
130
131 // ExternalAddress is the address (hostname or IP and port) that should be used in
132 // external (public internet) URLs for this GenericAPIServer.
133 ExternalAddress string
134
135 // Serializer controls how common API objects not in a group/version prefix are serialized for this server.
136 // Individual APIGroups may define their own serializers.
137 Serializer runtime.NegotiatedSerializer
138
139 // "Outputs"
140 // Handler holds the handlers being used by this API server
141 Handler *APIServerHandler
142
143 // UnprotectedDebugSocket is used to serve pprof information in a unix-domain socket. This socket is
144 // not protected by authentication/authorization.
145 UnprotectedDebugSocket *routes.DebugSocket
146
147 // listedPathProvider is a lister which provides the set of paths to show at /
148 listedPathProvider routes.ListedPathProvider
149
150 // DiscoveryGroupManager serves /apis in an unaggregated form.
151 DiscoveryGroupManager discovery.GroupManager
152
153 // AggregatedDiscoveryGroupManager serves /apis in an aggregated form.
154 AggregatedDiscoveryGroupManager discoveryendpoint.ResourceManager
155
156 // AggregatedLegacyDiscoveryGroupManager serves /api in an aggregated form.
157 AggregatedLegacyDiscoveryGroupManager discoveryendpoint.ResourceManager
158
159 // Enable swagger and/or OpenAPI if these configs are non-nil.
160 openAPIConfig *openapicommon.Config
161
162 // Enable swagger and/or OpenAPI V3 if these configs are non-nil.
163 openAPIV3Config *openapicommon.OpenAPIV3Config
164
165 // SkipOpenAPIInstallation indicates not to install the OpenAPI handler
166 // during PrepareRun.
167 // Set this to true when the specific API Server has its own OpenAPI handler
168 // (e.g. kube-aggregator)
169 skipOpenAPIInstallation bool
170
171 // OpenAPIVersionedService controls the /openapi/v2 endpoint, and can be used to update the served spec.
172 // It is set during PrepareRun if `openAPIConfig` is non-nil unless `skipOpenAPIInstallation` is true.
173 OpenAPIVersionedService *handler.OpenAPIService
174
175 // OpenAPIV3VersionedService controls the /openapi/v3 endpoint and can be used to update the served spec.
176 // It is set during PrepareRun if `openAPIConfig` is non-nil unless `skipOpenAPIInstallation` is true.
177 OpenAPIV3VersionedService *handler3.OpenAPIService
178
179 // StaticOpenAPISpec is the spec derived from the restful container endpoints.
180 // It is set during PrepareRun.
181 StaticOpenAPISpec *spec.Swagger
182
183 // PostStartHooks are each called after the server has started listening, in a separate go func for each
184 // with no guarantee of ordering between them. The map key is a name used for error reporting.
185 // It may kill the process with a panic if it wishes to by returning an error.
186 postStartHookLock sync.Mutex
187 postStartHooks map[string]postStartHookEntry
188 postStartHooksCalled bool
189 disabledPostStartHooks sets.String
190
191 preShutdownHookLock sync.Mutex
192 preShutdownHooks map[string]preShutdownHookEntry
193 preShutdownHooksCalled bool
194
195 // healthz checks
196 healthzRegistry healthCheckRegistry
197 readyzRegistry healthCheckRegistry
198 livezRegistry healthCheckRegistry
199
200 livezGracePeriod time.Duration
201
202 // auditing. The backend is started before the server starts listening.
203 AuditBackend audit.Backend
204
205 // Authorizer determines whether a user is allowed to make a certain request. The Handler does a preliminary
206 // authorization check using the request URI but it may be necessary to make additional checks, such as in
207 // the create-on-update case
208 Authorizer authorizer.Authorizer
209
210 // EquivalentResourceRegistry provides information about resources equivalent to a given resource,
211 // and the kind associated with a given resource. As resources are installed, they are registered here.
212 EquivalentResourceRegistry runtime.EquivalentResourceRegistry
213
214 // delegationTarget is the next delegate in the chain. This is never nil.
215 delegationTarget DelegationTarget
216
217 // NonLongRunningRequestWaitGroup allows you to wait for all chain
218 // handlers associated with non long-running requests
219 // to complete while the server is shuting down.
220 NonLongRunningRequestWaitGroup *utilwaitgroup.SafeWaitGroup
221 // WatchRequestWaitGroup allows us to wait for all chain
222 // handlers associated with active watch requests to
223 // complete while the server is shuting down.
224 WatchRequestWaitGroup *utilwaitgroup.RateLimitedSafeWaitGroup
225
226 // ShutdownDelayDuration allows to block shutdown for some time, e.g. until endpoints pointing to this API server
227 // have converged on all node. During this time, the API server keeps serving, /healthz will return 200,
228 // but /readyz will return failure.
229 ShutdownDelayDuration time.Duration
230
231 // The limit on the request body size that would be accepted and decoded in a write request.
232 // 0 means no limit.
233 maxRequestBodyBytes int64
234
235 // APIServerID is the ID of this API server
236 APIServerID string
237
238 // StorageReadinessHook implements post-start-hook functionality for checking readiness
239 // of underlying storage for registered resources.
240 StorageReadinessHook *StorageReadinessHook
241
242 // StorageVersionManager holds the storage versions of the API resources installed by this server.
243 StorageVersionManager storageversion.Manager
244
245 // EffectiveVersion determines which apis and features are available
246 // based on when the api/feature lifecyle.
247 EffectiveVersion basecompatibility.EffectiveVersion
248 // EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
249 // have higher priority than APIs of the same group resource enabled at the emulation version.
250 // If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
251 // This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
252 // Not applicable to alpha APIs.
253 EmulationForwardCompatible bool
254 // RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
255 // If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.
256 // If false, error would be thrown if any GroupVersion or GroupVersionResource explicitly enabled in the --runtime-config flag is introduced after the emulation version.
257 RuntimeConfigEmulationForwardCompatible bool
258
259 // FeatureGate is a way to plumb feature gate through if you have them.
260 FeatureGate featuregate.FeatureGate
261
262 // lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver.
263 lifecycleSignals lifecycleSignals
264
265 // destroyFns contains a list of functions that should be called on shutdown to clean up resources.
266 destroyFns []func()
267
268 // muxAndDiscoveryCompleteSignals holds signals that indicate all known HTTP paths have been registered.
269 // it exists primarily to avoid returning a 404 response when a resource actually exists but we haven't installed the path to a handler.
270 // it is exposed for easier composition of the individual servers.
271 // the primary users of this field are the WithMuxCompleteProtection filter and the NotFoundHandler
272 muxAndDiscoveryCompleteSignals map[string]<-chan struct{}
273
274 // ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
275 // Server during the graceful termination of the apiserver. If true, we wait
276 // for non longrunning requests in flight to be drained and then initiate a
277 // shutdown of the HTTP Server. If false, we initiate a shutdown of the HTTP
278 // Server as soon as ShutdownDelayDuration has elapsed.
279 // If enabled, after ShutdownDelayDuration elapses, any incoming request is
280 // rejected with a 429 status code and a 'Retry-After' response.
281 ShutdownSendRetryAfter bool
282
283 // ShutdownWatchTerminationGracePeriod, if set to a positive value,
284 // is the maximum duration the apiserver will wait for all active
285 // watch request(s) to drain.
286 // Once this grace period elapses, the apiserver will no longer
287 // wait for any active watch request(s) in flight to drain, it will
288 // proceed to the next step in the graceful server shutdown process.
289 // If set to a positive value, the apiserver will keep track of the
290 // number of active watch request(s) in flight and during shutdown
291 // it will wait, at most, for the specified duration and allow these
292 // active watch requests to drain with some rate limiting in effect.
293 // The default is zero, which implies the apiserver will not keep
294 // track of active watch request(s) in flight and will not wait
295 // for them to drain, this maintains backward compatibility.
296 // This grace period is orthogonal to other grace periods, and
297 // it is not overridden by any other grace period.
298 ShutdownWatchTerminationGracePeriod time.Duration
299 }
300
301 // DelegationTarget is an interface which allows for composition of API servers with top level handling that works
302 // as expected.
303 type DelegationTarget interface {
304 // UnprotectedHandler returns a handler that is NOT protected by a normal chain
305 UnprotectedHandler() http.Handler
306
307 // PostStartHooks returns the post-start hooks that need to be combined
308 PostStartHooks() map[string]postStartHookEntry
309
310 // PreShutdownHooks returns the pre-stop hooks that need to be combined
311 PreShutdownHooks() map[string]preShutdownHookEntry
312
313 // HealthzChecks returns the healthz checks that need to be combined
314 HealthzChecks() []healthz.HealthChecker
315
316 // ListedPaths returns the paths for supporting an index
317 ListedPaths() []string
318
319 // NextDelegate returns the next delegationTarget in the chain of delegations
320 NextDelegate() DelegationTarget
321
322 // PrepareRun does post API installation setup steps. It calls recursively the same function of the delegates.
323 PrepareRun() preparedGenericAPIServer
324
325 // MuxAndDiscoveryCompleteSignals exposes registered signals that indicate if all known HTTP paths have been installed.
326 MuxAndDiscoveryCompleteSignals() map[string]<-chan struct{}
327
328 // Destroy cleans up its resources on shutdown.
329 // Destroy has to be implemented in thread-safe way and be prepared
330 // for being called more than once.
331 Destroy()
332 }
333
334 func (s *GenericAPIServer) UnprotectedHandler() http.Handler {
335 // when we delegate, we need the server we're delegating to choose whether or not to use gorestful
336 return s.Handler.Director
337 }
338 func (s *GenericAPIServer) PostStartHooks() map[string]postStartHookEntry {
339 return s.postStartHooks
340 }
341 func (s *GenericAPIServer) PreShutdownHooks() map[string]preShutdownHookEntry {
342 return s.preShutdownHooks
343 }
344 func (s *GenericAPIServer) HealthzChecks() []healthz.HealthChecker {
345 return s.healthzRegistry.checks
346 }
347 func (s *GenericAPIServer) ListedPaths() []string {
348 return s.listedPathProvider.ListedPaths()
349 }
350
351 func (s *GenericAPIServer) NextDelegate() DelegationTarget {
352 return s.delegationTarget
353 }
354
355 // RegisterMuxAndDiscoveryCompleteSignal registers the given signal that will be used to determine if all known
356 // HTTP paths have been registered. It is okay to call this method after instantiating the generic server but before running.
357 func (s *GenericAPIServer) RegisterMuxAndDiscoveryCompleteSignal(signalName string, signal <-chan struct{}) error {
358 if _, exists := s.muxAndDiscoveryCompleteSignals[signalName]; exists {
359 return fmt.Errorf("%s already registered", signalName)
360 }
361 s.muxAndDiscoveryCompleteSignals[signalName] = signal
362 return nil
363 }
364
365 func (s *GenericAPIServer) MuxAndDiscoveryCompleteSignals() map[string]<-chan struct{} {
366 return s.muxAndDiscoveryCompleteSignals
367 }
368
369 // RegisterDestroyFunc registers a function that will be called during Destroy().
370 // The function have to be idempotent and prepared to be called more than once.
371 func (s *GenericAPIServer) RegisterDestroyFunc(destroyFn func()) {
372 s.destroyFns = append(s.destroyFns, destroyFn)
373 }
374
375 // Destroy cleans up all its and its delegation target resources on shutdown.
376 // It starts with destroying its own resources and later proceeds with
377 // its delegation target.
378 func (s *GenericAPIServer) Destroy() {
379 for _, destroyFn := range s.destroyFns {
380 destroyFn()
381 }
382 if s.delegationTarget != nil {
383 s.delegationTarget.Destroy()
384 }
385 }
386
387 type emptyDelegate struct {
388 // handler is called at the end of the delegation chain
389 // when a request has been made against an unregistered HTTP path the individual servers will simply pass it through until it reaches the handler.
390 handler http.Handler
391 }
392
393 func NewEmptyDelegate() DelegationTarget {
394 return emptyDelegate{}
395 }
396
397 // NewEmptyDelegateWithCustomHandler allows for registering a custom handler usually for special handling of 404 requests
398 func NewEmptyDelegateWithCustomHandler(handler http.Handler) DelegationTarget {
399 return emptyDelegate{handler}
400 }
401
402 func (s emptyDelegate) UnprotectedHandler() http.Handler {
403 return s.handler
404 }
405 func (s emptyDelegate) PostStartHooks() map[string]postStartHookEntry {
406 return map[string]postStartHookEntry{}
407 }
408 func (s emptyDelegate) PreShutdownHooks() map[string]preShutdownHookEntry {
409 return map[string]preShutdownHookEntry{}
410 }
411 func (s emptyDelegate) HealthzChecks() []healthz.HealthChecker {
412 return []healthz.HealthChecker{}
413 }
414 func (s emptyDelegate) ListedPaths() []string {
415 return []string{}
416 }
417 func (s emptyDelegate) NextDelegate() DelegationTarget {
418 return nil
419 }
420 func (s emptyDelegate) PrepareRun() preparedGenericAPIServer {
421 return preparedGenericAPIServer{nil}
422 }
423 func (s emptyDelegate) MuxAndDiscoveryCompleteSignals() map[string]<-chan struct{} {
424 return map[string]<-chan struct{}{}
425 }
426 func (s emptyDelegate) Destroy() {
427 }
428
429 // preparedGenericAPIServer is a private wrapper that enforces a call of PrepareRun() before Run can be invoked.
430 type preparedGenericAPIServer struct {
431 *GenericAPIServer
432 }
433
434 // PrepareRun does post API installation setup steps. It calls recursively the same function of the delegates.
435 func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
436 s.delegationTarget.PrepareRun()
437
438 if s.openAPIConfig != nil && !s.skipOpenAPIInstallation {
439 s.OpenAPIVersionedService, s.StaticOpenAPISpec = routes.OpenAPI{
440 Config: s.openAPIConfig,
441 }.InstallV2(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux)
442 }
443
444 if s.openAPIV3Config != nil && !s.skipOpenAPIInstallation {
445 s.OpenAPIV3VersionedService = routes.OpenAPI{
446 V3Config: s.openAPIV3Config,
447 }.InstallV3(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux)
448 }
449
450 s.installHealthz()
451 s.installLivez()
452
453 // as soon as shutdown is initiated, readiness should start failing
454 readinessStopCh := s.lifecycleSignals.ShutdownInitiated.Signaled()
455 err := s.addReadyzShutdownCheck(readinessStopCh)
456 if err != nil {
457 klog.Errorf("Failed to install readyz shutdown check %s", err)
458 }
459 s.installReadyz()
460
461 return preparedGenericAPIServer{s}
462 }
463
464 // Run spawns the secure http server. It only returns if stopCh is closed
465 // or the secure port cannot be listened on initially.
466 //
467 // Deprecated: use RunWithContext instead. Run will not get removed to avoid
468 // breaking consumers, but should not be used in new code.
469 func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
470 ctx := wait.ContextForChannel(stopCh)
471 return s.RunWithContext(ctx)
472 }
473
474 // RunWithContext spawns the secure http server. It only returns if ctx is canceled
475 // or the secure port cannot be listened on initially.
476 // This is the diagram of what contexts/channels/signals are dependent on each other:
477 //
478 // | ctx
479 // | |
480 // | ---------------------------------------------------------
481 // | | |
482 // | ShutdownInitiated (shutdownInitiatedCh) |
483 // | | |
484 // | (ShutdownDelayDuration) (PreShutdownHooks)
485 // | | |
486 // | AfterShutdownDelayDuration (delayedStopCh) PreShutdownHooksStopped (preShutdownHooksHasStoppedCh)
487 // | | |
488 // | |-------------------------------------------------------|
489 // | |
490 // | |
491 // | NotAcceptingNewRequest (notAcceptingNewRequestCh)
492 // | |
493 // | |
494 // | |----------------------------------------------------------------------------------|
495 // | | | | |
496 // | [without [with | |
497 // | ShutdownSendRetryAfter] ShutdownSendRetryAfter] | |
498 // | | | | |
499 // | | ---------------| |
500 // | | | |
501 // | | |----------------|-----------------------| |
502 // | | | | |
503 // | | (NonLongRunningRequestWaitGroup::Wait) (WatchRequestWaitGroup::Wait) |
504 // | | | | |
505 // | | |------------------|---------------------| |
506 // | | | |
507 // | | InFlightRequestsDrained (drainedCh) |
508 // | | | |
509 // | |-------------------|---------------------|----------------------------------------|
510 // | | |
511 // | stopHttpServerCtx (AuditBackend::Shutdown())
512 // | |
513 // | listenerStoppedCh
514 // | |
515 // | HTTPServerStoppedListening (httpServerStoppedListeningCh)
516 func (s preparedGenericAPIServer) RunWithContext(ctx context.Context) error {
517 stopCh := ctx.Done()
518 delayedStopCh := s.lifecycleSignals.AfterShutdownDelayDuration
519 shutdownInitiatedCh := s.lifecycleSignals.ShutdownInitiated
520
521 // Clean up resources on shutdown.
522 defer s.Destroy()
523
524 // If UDS profiling is enabled, start a local http server listening on that socket
525 if s.UnprotectedDebugSocket != nil {
526 go func() {
527 defer utilruntime.HandleCrash()
528 klog.Error(s.UnprotectedDebugSocket.Run(stopCh))
529 }()
530 }
531
532 // spawn a new goroutine for closing the MuxAndDiscoveryComplete signal
533 // registration happens during construction of the generic api server
534 // the last server in the chain aggregates signals from the previous instances
535 go func() {
536 for _, muxAndDiscoveryCompletedSignal := range s.GenericAPIServer.MuxAndDiscoveryCompleteSignals() {
537 select {
538 case <-muxAndDiscoveryCompletedSignal:
539 continue
540 case <-stopCh:
541 klog.V(1).Infof("haven't completed %s, stop requested", s.lifecycleSignals.MuxAndDiscoveryComplete.Name())
542 return
543 }
544 }
545 s.lifecycleSignals.MuxAndDiscoveryComplete.Signal()
546 klog.V(1).Infof("%s has all endpoints registered and discovery information is complete", s.lifecycleSignals.MuxAndDiscoveryComplete.Name())
547 }()
548
549 go func() {
550 defer delayedStopCh.Signal()
551 defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", delayedStopCh.Name())
552
553 <-stopCh
554
555 // As soon as shutdown is initiated, /readyz should start returning failure.
556 // This gives the load balancer a window defined by ShutdownDelayDuration to detect that /readyz is red
557 // and stop sending traffic to this server.
558 shutdownInitiatedCh.Signal()
559 klog.V(1).InfoS("[graceful-termination] shutdown event", "name", shutdownInitiatedCh.Name())
560
561 time.Sleep(s.ShutdownDelayDuration)
562 }()
563
564 // close socket after delayed stopCh
565 shutdownTimeout := s.ShutdownTimeout
566 if s.ShutdownSendRetryAfter {
567 // when this mode is enabled, we do the following:
568 // - the server will continue to listen until all existing requests in flight
569 // (not including active long running requests) have been drained.
570 // - once drained, http Server Shutdown is invoked with a timeout of 2s,
571 // net/http waits for 1s for the peer to respond to a GO_AWAY frame, so
572 // we should wait for a minimum of 2s
573 shutdownTimeout = 2 * time.Second
574 klog.V(1).InfoS("[graceful-termination] using HTTP Server shutdown timeout", "shutdownTimeout", shutdownTimeout)
575 }
576
577 notAcceptingNewRequestCh := s.lifecycleSignals.NotAcceptingNewRequest
578 drainedCh := s.lifecycleSignals.InFlightRequestsDrained
579 // Canceling the parent context does not immediately cancel the HTTP server.
580 // We only inherit context values here and deal with cancellation ourselves.
581 stopHTTPServerCtx, stopHTTPServer := context.WithCancelCause(context.WithoutCancel(ctx))
582 go func() {
583 defer stopHTTPServer(errors.New("time to stop HTTP server"))
584
585 timeToStopHttpServerCh := notAcceptingNewRequestCh.Signaled()
586 if s.ShutdownSendRetryAfter {
587 timeToStopHttpServerCh = drainedCh.Signaled()
588 }
589
590 <-timeToStopHttpServerCh
591 }()
592
593 // Start the audit backend before any request comes in. This means we must call Backend.Run
594 // before http server start serving. Otherwise the Backend.ProcessEvents call might block.
595 // AuditBackend.Run will stop as soon as all in-flight requests are drained.
596 if s.AuditBackend != nil {
597 if err := s.AuditBackend.Run(drainedCh.Signaled()); err != nil {
598 return fmt.Errorf("failed to run the audit backend: %v", err)
599 }
600 }
601
602 stoppedCh, listenerStoppedCh, err := s.NonBlockingRunWithContext(stopHTTPServerCtx, shutdownTimeout)
603 if err != nil {
604 return err
605 }
606
607 httpServerStoppedListeningCh := s.lifecycleSignals.HTTPServerStoppedListening
608 go func() {
609 <-listenerStoppedCh
610 httpServerStoppedListeningCh.Signal()
611 klog.V(1).InfoS("[graceful-termination] shutdown event", "name", httpServerStoppedListeningCh.Name())
612 }()
613
614 // we don't accept new request as soon as both ShutdownDelayDuration has
615 // elapsed and preshutdown hooks have completed.
616 preShutdownHooksHasStoppedCh := s.lifecycleSignals.PreShutdownHooksStopped
617 go func() {
618 defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", notAcceptingNewRequestCh.Name())
619 defer notAcceptingNewRequestCh.Signal()
620
621 // wait for the delayed stopCh before closing the handler chain
622 <-delayedStopCh.Signaled()
623
624 // Additionally wait for preshutdown hooks to also be finished, as some of them need
625 // to send API calls to clean up after themselves (e.g. lease reconcilers removing
626 // itself from the active servers).
627 <-preShutdownHooksHasStoppedCh.Signaled()
628 }()
629
630 // wait for all in-flight non-long running requests to finish
631 nonLongRunningRequestDrainedCh := make(chan struct{})
632 go func() {
633 defer close(nonLongRunningRequestDrainedCh)
634 defer klog.V(1).Info("[graceful-termination] in-flight non long-running request(s) have drained")
635
636 // wait for the delayed stopCh before closing the handler chain (it rejects everything after Wait has been called).
637 <-notAcceptingNewRequestCh.Signaled()
638
639 // Wait for all requests to finish, which are bounded by the RequestTimeout variable.
640 // once NonLongRunningRequestWaitGroup.Wait is invoked, the apiserver is
641 // expected to reject any incoming request with a {503, Retry-After}
642 // response via the WithWaitGroup filter. On the contrary, we observe
643 // that incoming request(s) get a 'connection refused' error, this is
644 // because, at this point, we have called 'Server.Shutdown' and
645 // net/http server has stopped listening. This causes incoming
646 // request to get a 'connection refused' error.
647 // On the other hand, if 'ShutdownSendRetryAfter' is enabled incoming
648 // requests will be rejected with a {429, Retry-After} since
649 // 'Server.Shutdown' will be invoked only after in-flight requests
650 // have been drained.
651 // TODO: can we consolidate these two modes of graceful termination?
652 s.NonLongRunningRequestWaitGroup.Wait()
653 }()
654
655 // wait for all in-flight watches to finish
656 activeWatchesDrainedCh := make(chan struct{})
657 go func() {
658 defer close(activeWatchesDrainedCh)
659
660 <-notAcceptingNewRequestCh.Signaled()
661 if s.ShutdownWatchTerminationGracePeriod <= time.Duration(0) {
662 klog.V(1).InfoS("[graceful-termination] not going to wait for active watch request(s) to drain")
663 return
664 }
665
666 // Wait for all active watches to finish
667 grace := s.ShutdownWatchTerminationGracePeriod
668 activeBefore, activeAfter, err := s.WatchRequestWaitGroup.Wait(func(count int) (utilwaitgroup.RateLimiter, context.Context, context.CancelFunc) {
669 qps := float64(count) / grace.Seconds()
670 // TODO: we don't want the QPS (max requests drained per second) to
671 // get below a certain floor value, since we want the server to
672 // drain the active watch requests as soon as possible.
673 // For now, it's hard coded to 200, and it is subject to change
674 // based on the result from the scale testing.
675 if qps < 200 {
676 qps = 200
677 }
678
679 ctx, cancel := context.WithTimeout(context.Background(), grace)
680 // We don't expect more than one token to be consumed
681 // in a single Wait call, so setting burst to 1.
682 return rate.NewLimiter(rate.Limit(qps), 1), ctx, cancel
683 })
684 klog.V(1).InfoS("[graceful-termination] active watch request(s) have drained",
685 "duration", grace, "activeWatchesBefore", activeBefore, "activeWatchesAfter", activeAfter, "error", err)
686 }()
687
688 go func() {
689 defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", drainedCh.Name())
690 defer drainedCh.Signal()
691
692 <-nonLongRunningRequestDrainedCh
693 <-activeWatchesDrainedCh
694 }()
695
696 klog.V(1).Info("[graceful-termination] waiting for shutdown to be initiated")
697 <-stopCh
698
699 // run shutdown hooks directly. This includes deregistering from
700 // the kubernetes endpoint in case of kube-apiserver.
701 func() {
702 defer func() {
703 preShutdownHooksHasStoppedCh.Signal()
704 klog.V(1).InfoS("[graceful-termination] pre-shutdown hooks completed", "name", preShutdownHooksHasStoppedCh.Name())
705 }()
706 err = s.RunPreShutdownHooks()
707 }()
708 if err != nil {
709 return err
710 }
711
712 // Wait for all requests in flight to drain, bounded by the RequestTimeout variable.
713 <-drainedCh.Signaled()
714
715 if s.AuditBackend != nil {
716 s.AuditBackend.Shutdown()
717 klog.V(1).InfoS("[graceful-termination] audit backend shutdown completed")
718 }
719
720 // wait for stoppedCh that is closed when the graceful termination (server.Shutdown) is finished.
721 <-listenerStoppedCh
722 <-stoppedCh
723
724 klog.V(1).Info("[graceful-termination] apiserver is exiting")
725 return nil
726 }
727
728 // NonBlockingRun spawns the secure http server. An error is
729 // returned if the secure port cannot be listened on.
730 // The returned channel is closed when the (asynchronous) termination is finished.
731 //
732 // Deprecated: use RunWithContext instead. Run will not get removed to avoid
733 // breaking consumers, but should not be used in new code.
734 func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}, shutdownTimeout time.Duration) (<-chan struct{}, <-chan struct{}, error) {
735 ctx := wait.ContextForChannel(stopCh)
736 return s.NonBlockingRunWithContext(ctx, shutdownTimeout)
737 }
738
739 // NonBlockingRunWithContext spawns the secure http server. An error is
740 // returned if the secure port cannot be listened on.
741 // The returned channel is closed when the (asynchronous) termination is finished.
742 func (s preparedGenericAPIServer) NonBlockingRunWithContext(ctx context.Context, shutdownTimeout time.Duration) (<-chan struct{}, <-chan struct{}, error) {
743 // Use an internal stop channel to allow cleanup of the listeners on error.
744 internalStopCh := make(chan struct{})
745 var stoppedCh <-chan struct{}
746 var listenerStoppedCh <-chan struct{}
747 if s.SecureServingInfo != nil && s.Handler != nil {
748 var err error
749 stoppedCh, listenerStoppedCh, err = s.SecureServingInfo.Serve(s.Handler, shutdownTimeout, internalStopCh)
750 if err != nil {
751 close(internalStopCh)
752 return nil, nil, err
753 }
754 }
755
756 // Now that listener have bound successfully, it is the
757 // responsibility of the caller to close the provided channel to
758 // ensure cleanup.
759 go func() {
760 <-ctx.Done()
761 close(internalStopCh)
762 }()
763
764 s.RunPostStartHooks(ctx)
765
766 if _, err := systemd.SdNotify(true, "READY=1\n"); err != nil {
767 klog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
768 }
769
770 return stoppedCh, listenerStoppedCh, nil
771 }
772
773 // installAPIResources is a private method for installing the REST storage backing each api groupversionresource
774 func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo, typeConverter managedfields.TypeConverter) error {
775 var resourceInfos []*storageversion.ResourceInfo
776 for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
777 if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
778 klog.Warningf("Skipping API %v because it has no resources.", groupVersion)
779 continue
780 }
781
782 apiGroupVersion, err := s.getAPIGroupVersion(apiGroupInfo, groupVersion, apiPrefix)
783 if err != nil {
784 return err
785 }
786 if apiGroupInfo.OptionsExternalVersion != nil {
787 apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion
788 }
789 apiGroupVersion.TypeConverter = typeConverter
790 apiGroupVersion.MaxRequestBodyBytes = s.maxRequestBodyBytes
791
792 discoveryAPIResources, r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer)
793
794 if err != nil {
795 return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err)
796 }
797 resourceInfos = append(resourceInfos, r...)
798
799 // Aggregated discovery only aggregates resources under /apis
800 if apiPrefix == APIGroupPrefix {
801 s.AggregatedDiscoveryGroupManager.AddGroupVersion(
802 groupVersion.Group,
803 apidiscoveryv2.APIVersionDiscovery{
804 Freshness: apidiscoveryv2.DiscoveryFreshnessCurrent,
805 Version: groupVersion.Version,
806 Resources: discoveryAPIResources,
807 },
808 )
809 } else {
810 // There is only one group version for legacy resources, priority can be defaulted to 0.
811 s.AggregatedLegacyDiscoveryGroupManager.AddGroupVersion(
812 groupVersion.Group,
813 apidiscoveryv2.APIVersionDiscovery{
814 Freshness: apidiscoveryv2.DiscoveryFreshnessCurrent,
815 Version: groupVersion.Version,
816 Resources: discoveryAPIResources,
817 },
818 )
819 }
820
821 }
822
823 s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)
824
825 if s.FeatureGate.Enabled(features.StorageVersionAPI) &&
826 s.FeatureGate.Enabled(features.APIServerIdentity) {
827 // API installation happens before we start listening on the handlers,
828 // therefore it is safe to register ResourceInfos here. The handler will block
829 // write requests until the storage versions of the targeting resources are updated.
830 s.StorageVersionManager.AddResourceInfo(resourceInfos...)
831 }
832
833 return nil
834 }
835
836 // InstallLegacyAPIGroup exposes the given legacy api group in the API.
837 // The <apiGroupInfo> passed into this function shouldn't be used elsewhere as the
838 // underlying storage will be destroyed on this servers shutdown.
839 func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
840 if !s.legacyAPIGroupPrefixes.Has(apiPrefix) {
841 return fmt.Errorf("%q is not in the allowed legacy API prefixes: %v", apiPrefix, s.legacyAPIGroupPrefixes.List())
842 }
843
844 openAPIModels, err := s.getOpenAPIModels(apiPrefix, apiGroupInfo)
845 if err != nil {
846 return fmt.Errorf("unable to get openapi models: %v", err)
847 }
848
849 if err := s.installAPIResources(apiPrefix, apiGroupInfo, openAPIModels); err != nil {
850 return err
851 }
852
853 // Install the version handler.
854 // Add a handler at /<apiPrefix> to enumerate the supported api versions.
855 legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix)
856 wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager)
857 s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{}))
858 s.registerStorageReadinessCheck("", apiGroupInfo)
859
860 return nil
861 }
862
863 // InstallAPIGroups exposes given api groups in the API.
864 // The <apiGroupInfos> passed into this function shouldn't be used elsewhere as the
865 // underlying storage will be destroyed on this servers shutdown.
866 func (s *GenericAPIServer) InstallAPIGroups(apiGroupInfos ...*APIGroupInfo) error {
867 for _, apiGroupInfo := range apiGroupInfos {
868 if len(apiGroupInfo.PrioritizedVersions) == 0 {
869 return fmt.Errorf("no version priority set for %#v", *apiGroupInfo)
870 }
871 // Do not register empty group or empty version. Doing so claims /apis/ for the wrong entity to be returned.
872 // Catching these here places the error much closer to its origin
873 if len(apiGroupInfo.PrioritizedVersions[0].Group) == 0 {
874 return fmt.Errorf("cannot register handler with an empty group for %#v", *apiGroupInfo)
875 }
876 if len(apiGroupInfo.PrioritizedVersions[0].Version) == 0 {
877 return fmt.Errorf("cannot register handler with an empty version for %#v", *apiGroupInfo)
878 }
879 }
880
881 openAPIModels, err := s.getOpenAPIModels(APIGroupPrefix, apiGroupInfos...)
882 if err != nil {
883 return fmt.Errorf("unable to get openapi models: %v", err)
884 }
885
886 for _, apiGroupInfo := range apiGroupInfos {
887 if err := s.installAPIResources(APIGroupPrefix, apiGroupInfo, openAPIModels); err != nil {
888 return fmt.Errorf("unable to install api resources: %v", err)
889 }
890
891 // setup discovery
892 // Install the version handler.
893 // Add a handler at /apis/<groupName> to enumerate all versions supported by this group.
894 apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{}
895 for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
896 // Check the config to make sure that we elide versions that don't have any resources
897 if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
898 continue
899 }
900 apiVersionsForDiscovery = append(apiVersionsForDiscovery, metav1.GroupVersionForDiscovery{
901 GroupVersion: groupVersion.String(),
902 Version: groupVersion.Version,
903 })
904 }
905 preferredVersionForDiscovery := metav1.GroupVersionForDiscovery{
906 GroupVersion: apiGroupInfo.PrioritizedVersions[0].String(),
907 Version: apiGroupInfo.PrioritizedVersions[0].Version,
908 }
909 apiGroup := metav1.APIGroup{
910 Name: apiGroupInfo.PrioritizedVersions[0].Group,
911 Versions: apiVersionsForDiscovery,
912 PreferredVersion: preferredVersionForDiscovery,
913 }
914
915 s.DiscoveryGroupManager.AddGroup(apiGroup)
916 s.Handler.GoRestfulContainer.Add(discovery.NewAPIGroupHandler(s.Serializer, apiGroup).WebService())
917 s.registerStorageReadinessCheck(apiGroupInfo.PrioritizedVersions[0].Group, apiGroupInfo)
918 }
919 return nil
920 }
921
922 // registerStorageReadinessCheck registers the readiness checks for all underlying storages
923 // for a given APIGroup.
924 func (s *GenericAPIServer) registerStorageReadinessCheck(groupName string, apiGroupInfo *APIGroupInfo) {
925 for version, storageMap := range apiGroupInfo.VersionedResourcesStorageMap {
926 for resource, storage := range storageMap {
927 if withReadiness, ok := storage.(rest.StorageWithReadiness); ok {
928 gvr := metav1.GroupVersionResource{
929 Group: groupName,
930 Version: version,
931 Resource: resource,
932 }
933 s.StorageReadinessHook.RegisterStorage(gvr, withReadiness)
934 }
935 }
936 }
937 }
938
939 // InstallAPIGroup exposes the given api group in the API.
940 // The <apiGroupInfo> passed into this function shouldn't be used elsewhere as the
941 // underlying storage will be destroyed on this servers shutdown.
942 func (s *GenericAPIServer) InstallAPIGroup(apiGroupInfo *APIGroupInfo) error {
943 return s.InstallAPIGroups(apiGroupInfo)
944 }
945
946 func (s *GenericAPIServer) getAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion, apiPrefix string) (*genericapi.APIGroupVersion, error) {
947 storage := make(map[string]rest.Storage)
948 for k, v := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] {
949 if strings.ToLower(k) != k {
950 return nil, fmt.Errorf("resource names must be lowercase only, not %q", k)
951 }
952 storage[k] = v
953 }
954 version := s.newAPIGroupVersion(apiGroupInfo, groupVersion)
955 version.Root = apiPrefix
956 version.Storage = storage
957 return version, nil
958 }
959
960 func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion) *genericapi.APIGroupVersion {
961
962 allServedVersionsByResource := map[string][]string{}
963 for version, resourcesInVersion := range apiGroupInfo.VersionedResourcesStorageMap {
964 for resource := range resourcesInVersion {
965 if len(groupVersion.Group) == 0 {
966 allServedVersionsByResource[resource] = append(allServedVersionsByResource[resource], version)
967 } else {
968 allServedVersionsByResource[resource] = append(allServedVersionsByResource[resource], fmt.Sprintf("%s/%s", groupVersion.Group, version))
969 }
970 }
971 }
972
973 return &genericapi.APIGroupVersion{
974 GroupVersion: groupVersion,
975 AllServedVersionsByResource: allServedVersionsByResource,
976 MetaGroupVersion: apiGroupInfo.MetaGroupVersion,
977
978 ParameterCodec: apiGroupInfo.ParameterCodec,
979 Serializer: apiGroupInfo.NegotiatedSerializer,
980 Creater: apiGroupInfo.Scheme,
981 Convertor: apiGroupInfo.Scheme,
982 ConvertabilityChecker: apiGroupInfo.Scheme,
983 UnsafeConvertor: runtime.UnsafeObjectConvertor(apiGroupInfo.Scheme),
984 Defaulter: apiGroupInfo.Scheme,
985 Typer: apiGroupInfo.Scheme,
986 Namer: runtime.Namer(meta.NewAccessor()),
987
988 EquivalentResourceRegistry: s.EquivalentResourceRegistry,
989
990 Admit: s.admissionControl,
991 MinRequestTimeout: s.minRequestTimeout,
992 Authorizer: s.Authorizer,
993 }
994 }
995
996 // NewDefaultAPIGroupInfo returns an APIGroupInfo stubbed with "normal" values
997 // exposed for easier composition from other packages
998 func NewDefaultAPIGroupInfo(group string, scheme *runtime.Scheme, parameterCodec runtime.ParameterCodec, codecs serializer.CodecFactory) APIGroupInfo {
999 opts := []serializer.CodecFactoryOptionsMutator{}
1000 if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) {
1001 opts = append(opts, serializer.WithSerializer(cbor.NewSerializerInfo))
1002 }
1003 if utilfeature.DefaultFeatureGate.Enabled(features.StreamingCollectionEncodingToJSON) {
1004 opts = append(opts, serializer.WithStreamingCollectionEncodingToJSON())
1005 }
1006 if utilfeature.DefaultFeatureGate.Enabled(features.StreamingCollectionEncodingToProtobuf) {
1007 opts = append(opts, serializer.WithStreamingCollectionEncodingToProtobuf())
1008 }
1009 if len(opts) != 0 {
1010 codecs = serializer.NewCodecFactory(scheme, opts...)
1011 }
1012 return APIGroupInfo{
1013 PrioritizedVersions: scheme.PrioritizedVersionsForGroup(group),
1014 VersionedResourcesStorageMap: map[string]map[string]rest.Storage{},
1015 // TODO unhardcode this. It was hardcoded before, but we need to re-evaluate
1016 OptionsExternalVersion: &schema.GroupVersion{Version: "v1"},
1017 Scheme: scheme,
1018 ParameterCodec: parameterCodec,
1019 NegotiatedSerializer: codecs,
1020 }
1021 }
1022
1023 // getOpenAPIModels is a private method for getting the OpenAPI models
1024 func (s *GenericAPIServer) getOpenAPIModels(apiPrefix string, apiGroupInfos ...*APIGroupInfo) (managedfields.TypeConverter, error) {
1025 if s.openAPIV3Config == nil {
1026 // SSA is GA and requires OpenAPI config to be set
1027 // to create models.
1028 return nil, errors.New("OpenAPIV3 config must not be nil")
1029 }
1030 pathsToIgnore := openapiutil.NewTrie(s.openAPIV3Config.IgnorePrefixes)
1031 resourceNames := make([]string, 0)
1032 for _, apiGroupInfo := range apiGroupInfos {
1033 groupResources, err := getResourceNamesForGroup(apiPrefix, apiGroupInfo, pathsToIgnore)
1034 if err != nil {
1035 return nil, err
1036 }
1037 resourceNames = append(resourceNames, groupResources...)
1038 }
1039
1040 // Build the openapi definitions for those resources and convert it to proto models
1041 openAPISpec, err := openapibuilder3.BuildOpenAPIDefinitionsForResources(s.openAPIV3Config, resourceNames...)
1042 if err != nil {
1043 return nil, err
1044 }
1045 for _, apiGroupInfo := range apiGroupInfos {
1046 apiGroupInfo.StaticOpenAPISpec = openAPISpec
1047 }
1048
1049 typeConverter, err := managedfields.NewTypeConverter(openAPISpec, false)
1050 if err != nil {
1051 return nil, err
1052 }
1053
1054 return typeConverter, nil
1055 }
1056
1057 // getResourceNamesForGroup is a private method for getting the canonical names for each resource to build in an api group
1058 func getResourceNamesForGroup(apiPrefix string, apiGroupInfo *APIGroupInfo, pathsToIgnore openapiutil.Trie) ([]string, error) {
1059 // Get the canonical names of every resource we need to build in this api group
1060 resourceNames := make([]string, 0)
1061 for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
1062 for resource, storage := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] {
1063 path := gpath.Join(apiPrefix, groupVersion.Group, groupVersion.Version, resource)
1064 if !pathsToIgnore.HasPrefix(path) {
1065 kind, err := genericapi.GetResourceKind(groupVersion, storage, apiGroupInfo.Scheme)
1066 if err != nil {
1067 return nil, err
1068 }
1069 sampleObject, err := apiGroupInfo.Scheme.New(kind)
1070 if err != nil {
1071 return nil, err
1072 }
1073 name := openapiutil.GetCanonicalTypeName(sampleObject)
1074 resourceNames = append(resourceNames, name)
1075 }
1076 }
1077 }
1078
1079 return resourceNames, nil
1080 }