| 1 |
/* |
| 2 |
Copyright 2014 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package server |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"errors" |
| 22 |
"fmt" |
| 23 |
"net/http" |
| 24 |
gpath "path" |
| 25 |
"strings" |
| 26 |
"sync" |
| 27 |
"time" |
| 28 |
|
| 29 |
systemd "github.com/coreos/go-systemd/v22/daemon" |
| 30 |
|
| 31 |
"golang.org/x/time/rate" |
| 32 |
apidiscoveryv2 "k8s.io/api/apidiscovery/v2" |
| 33 |
"k8s.io/apimachinery/pkg/api/meta" |
| 34 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 35 |
"k8s.io/apimachinery/pkg/runtime" |
| 36 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 37 |
"k8s.io/apimachinery/pkg/runtime/serializer" |
| 38 |
"k8s.io/apimachinery/pkg/runtime/serializer/cbor" |
| 39 |
"k8s.io/apimachinery/pkg/util/managedfields" |
| 40 |
utilruntime "k8s.io/apimachinery/pkg/util/runtime" |
| 41 |
"k8s.io/apimachinery/pkg/util/sets" |
| 42 |
"k8s.io/apimachinery/pkg/util/wait" |
| 43 |
utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" |
| 44 |
"k8s.io/apiserver/pkg/admission" |
| 45 |
"k8s.io/apiserver/pkg/audit" |
| 46 |
"k8s.io/apiserver/pkg/authorization/authorizer" |
| 47 |
genericapi "k8s.io/apiserver/pkg/endpoints" |
| 48 |
"k8s.io/apiserver/pkg/endpoints/discovery" |
| 49 |
discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" |
| 50 |
"k8s.io/apiserver/pkg/features" |
| 51 |
"k8s.io/apiserver/pkg/registry/rest" |
| 52 |
"k8s.io/apiserver/pkg/server/healthz" |
| 53 |
"k8s.io/apiserver/pkg/server/routes" |
| 54 |
"k8s.io/apiserver/pkg/storageversion" |
| 55 |
utilfeature "k8s.io/apiserver/pkg/util/feature" |
| 56 |
restclient "k8s.io/client-go/rest" |
| 57 |
basecompatibility "k8s.io/component-base/compatibility" |
| 58 |
"k8s.io/component-base/featuregate" |
| 59 |
"k8s.io/klog/v2" |
| 60 |
openapibuilder3 "k8s.io/kube-openapi/pkg/builder3" |
| 61 |
openapicommon "k8s.io/kube-openapi/pkg/common" |
| 62 |
"k8s.io/kube-openapi/pkg/handler" |
| 63 |
"k8s.io/kube-openapi/pkg/handler3" |
| 64 |
openapiutil "k8s.io/kube-openapi/pkg/util" |
| 65 |
"k8s.io/kube-openapi/pkg/validation/spec" |
| 66 |
) |
| 67 |
|
| 68 |
// Info about an API group. |
| 69 |
type APIGroupInfo struct { |
| 70 |
PrioritizedVersions []schema.GroupVersion |
| 71 |
// Info about the resources in this group. It's a map from version to resource to the storage. |
| 72 |
VersionedResourcesStorageMap map[string]map[string]rest.Storage |
| 73 |
// OptionsExternalVersion controls the APIVersion used for common objects in the |
| 74 |
// schema like api.Status, api.DeleteOptions, and metav1.ListOptions. Other implementors may |
| 75 |
// define a version "v1beta1" but want to use the Kubernetes "v1" internal objects. |
| 76 |
// If nil, defaults to groupMeta.GroupVersion. |
| 77 |
// TODO: Remove this when https://github.com/kubernetes/kubernetes/issues/19018 is fixed. |
| 78 |
OptionsExternalVersion *schema.GroupVersion |
| 79 |
// MetaGroupVersion defaults to "meta.k8s.io/v1" and is the scheme group version used to decode |
| 80 |
// common API implementations like ListOptions. Future changes will allow this to vary by group |
| 81 |
// version (for when the inevitable meta/v2 group emerges). |
| 82 |
MetaGroupVersion *schema.GroupVersion |
| 83 |
|
| 84 |
// Scheme includes all of the types used by this group and how to convert between them (or |
| 85 |
// to convert objects from outside of this group that are accepted in this API). |
| 86 |
// TODO: replace with interfaces |
| 87 |
Scheme *runtime.Scheme |
| 88 |
// NegotiatedSerializer controls how this group encodes and decodes data |
| 89 |
NegotiatedSerializer runtime.NegotiatedSerializer |
| 90 |
// ParameterCodec performs conversions for query parameters passed to API calls |
| 91 |
ParameterCodec runtime.ParameterCodec |
| 92 |
|
| 93 |
// StaticOpenAPISpec is the spec derived from the definitions of all resources installed together. |
| 94 |
// It is set during InstallAPIGroups, InstallAPIGroup, and InstallLegacyAPIGroup. |
| 95 |
StaticOpenAPISpec map[string]*spec.Schema |
| 96 |
} |
| 97 |
|
| 98 |
func (a *APIGroupInfo) destroyStorage() { |
| 99 |
for _, stores := range a.VersionedResourcesStorageMap { |
| 100 |
for _, store := range stores { |
| 101 |
store.Destroy() |
| 102 |
} |
| 103 |
} |
| 104 |
} |
| 105 |
|
| 106 |
// GenericAPIServer contains state for a Kubernetes cluster api server. |
| 107 |
type GenericAPIServer struct { |
| 108 |
// discoveryAddresses is used to build cluster IPs for discovery. |
| 109 |
discoveryAddresses discovery.Addresses |
| 110 |
|
| 111 |
// LoopbackClientConfig is a config for a privileged loopback connection to the API server |
| 112 |
LoopbackClientConfig *restclient.Config |
| 113 |
|
| 114 |
// minRequestTimeout is how short the request timeout can be. This is used to build the RESTHandler |
| 115 |
minRequestTimeout time.Duration |
| 116 |
|
| 117 |
// ShutdownTimeout is the timeout used for server shutdown. This specifies the timeout before server |
| 118 |
// gracefully shutdown returns. |
| 119 |
ShutdownTimeout time.Duration |
| 120 |
|
| 121 |
// legacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests |
| 122 |
// to InstallLegacyAPIGroup |
| 123 |
legacyAPIGroupPrefixes sets.String |
| 124 |
|
| 125 |
// admissionControl is used to build the RESTStorage that backs an API Group. |
| 126 |
admissionControl admission.Interface |
| 127 |
|
| 128 |
// SecureServingInfo holds configuration of the TLS server. |
| 129 |
SecureServingInfo *SecureServingInfo |
| 130 |
|
| 131 |
// ExternalAddress is the address (hostname or IP and port) that should be used in |
| 132 |
// external (public internet) URLs for this GenericAPIServer. |
| 133 |
ExternalAddress string |
| 134 |
|
| 135 |
// Serializer controls how common API objects not in a group/version prefix are serialized for this server. |
| 136 |
// Individual APIGroups may define their own serializers. |
| 137 |
Serializer runtime.NegotiatedSerializer |
| 138 |
|
| 139 |
// "Outputs" |
| 140 |
// Handler holds the handlers being used by this API server |
| 141 |
Handler *APIServerHandler |
| 142 |
|
| 143 |
// UnprotectedDebugSocket is used to serve pprof information in a unix-domain socket. This socket is |
| 144 |
// not protected by authentication/authorization. |
| 145 |
UnprotectedDebugSocket *routes.DebugSocket |
| 146 |
|
| 147 |
// listedPathProvider is a lister which provides the set of paths to show at / |
| 148 |
listedPathProvider routes.ListedPathProvider |
| 149 |
|
| 150 |
// DiscoveryGroupManager serves /apis in an unaggregated form. |
| 151 |
DiscoveryGroupManager discovery.GroupManager |
| 152 |
|
| 153 |
// AggregatedDiscoveryGroupManager serves /apis in an aggregated form. |
| 154 |
AggregatedDiscoveryGroupManager discoveryendpoint.ResourceManager |
| 155 |
|
| 156 |
// AggregatedLegacyDiscoveryGroupManager serves /api in an aggregated form. |
| 157 |
AggregatedLegacyDiscoveryGroupManager discoveryendpoint.ResourceManager |
| 158 |
|
| 159 |
// Enable swagger and/or OpenAPI if these configs are non-nil. |
| 160 |
openAPIConfig *openapicommon.Config |
| 161 |
|
| 162 |
// Enable swagger and/or OpenAPI V3 if these configs are non-nil. |
| 163 |
openAPIV3Config *openapicommon.OpenAPIV3Config |
| 164 |
|
| 165 |
// SkipOpenAPIInstallation indicates not to install the OpenAPI handler |
| 166 |
// during PrepareRun. |
| 167 |
// Set this to true when the specific API Server has its own OpenAPI handler |
| 168 |
// (e.g. kube-aggregator) |
| 169 |
skipOpenAPIInstallation bool |
| 170 |
|
| 171 |
// OpenAPIVersionedService controls the /openapi/v2 endpoint, and can be used to update the served spec. |
| 172 |
// It is set during PrepareRun if `openAPIConfig` is non-nil unless `skipOpenAPIInstallation` is true. |
| 173 |
OpenAPIVersionedService *handler.OpenAPIService |
| 174 |
|
| 175 |
// OpenAPIV3VersionedService controls the /openapi/v3 endpoint and can be used to update the served spec. |
| 176 |
// It is set during PrepareRun if `openAPIConfig` is non-nil unless `skipOpenAPIInstallation` is true. |
| 177 |
OpenAPIV3VersionedService *handler3.OpenAPIService |
| 178 |
|
| 179 |
// StaticOpenAPISpec is the spec derived from the restful container endpoints. |
| 180 |
// It is set during PrepareRun. |
| 181 |
StaticOpenAPISpec *spec.Swagger |
| 182 |
|
| 183 |
// PostStartHooks are each called after the server has started listening, in a separate go func for each |
| 184 |
// with no guarantee of ordering between them. The map key is a name used for error reporting. |
| 185 |
// It may kill the process with a panic if it wishes to by returning an error. |
| 186 |
postStartHookLock sync.Mutex |
| 187 |
postStartHooks map[string]postStartHookEntry |
| 188 |
postStartHooksCalled bool |
| 189 |
disabledPostStartHooks sets.String |
| 190 |
|
| 191 |
preShutdownHookLock sync.Mutex |
| 192 |
preShutdownHooks map[string]preShutdownHookEntry |
| 193 |
preShutdownHooksCalled bool |
| 194 |
|
| 195 |
// healthz checks |
| 196 |
healthzRegistry healthCheckRegistry |
| 197 |
readyzRegistry healthCheckRegistry |
| 198 |
livezRegistry healthCheckRegistry |
| 199 |
|
| 200 |
livezGracePeriod time.Duration |
| 201 |
|
| 202 |
// auditing. The backend is started before the server starts listening. |
| 203 |
AuditBackend audit.Backend |
| 204 |
|
| 205 |
// Authorizer determines whether a user is allowed to make a certain request. The Handler does a preliminary |
| 206 |
// authorization check using the request URI but it may be necessary to make additional checks, such as in |
| 207 |
// the create-on-update case |
| 208 |
Authorizer authorizer.Authorizer |
| 209 |
|
| 210 |
// EquivalentResourceRegistry provides information about resources equivalent to a given resource, |
| 211 |
// and the kind associated with a given resource. As resources are installed, they are registered here. |
| 212 |
EquivalentResourceRegistry runtime.EquivalentResourceRegistry |
| 213 |
|
| 214 |
// delegationTarget is the next delegate in the chain. This is never nil. |
| 215 |
delegationTarget DelegationTarget |
| 216 |
|
| 217 |
// NonLongRunningRequestWaitGroup allows you to wait for all chain |
| 218 |
// handlers associated with non long-running requests |
| 219 |
// to complete while the server is shuting down. |
| 220 |
NonLongRunningRequestWaitGroup *utilwaitgroup.SafeWaitGroup |
| 221 |
// WatchRequestWaitGroup allows us to wait for all chain |
| 222 |
// handlers associated with active watch requests to |
| 223 |
// complete while the server is shuting down. |
| 224 |
WatchRequestWaitGroup *utilwaitgroup.RateLimitedSafeWaitGroup |
| 225 |
|
| 226 |
// ShutdownDelayDuration allows to block shutdown for some time, e.g. until endpoints pointing to this API server |
| 227 |
// have converged on all node. During this time, the API server keeps serving, /healthz will return 200, |
| 228 |
// but /readyz will return failure. |
| 229 |
ShutdownDelayDuration time.Duration |
| 230 |
|
| 231 |
// The limit on the request body size that would be accepted and decoded in a write request. |
| 232 |
// 0 means no limit. |
| 233 |
maxRequestBodyBytes int64 |
| 234 |
|
| 235 |
// APIServerID is the ID of this API server |
| 236 |
APIServerID string |
| 237 |
|
| 238 |
// StorageReadinessHook implements post-start-hook functionality for checking readiness |
| 239 |
// of underlying storage for registered resources. |
| 240 |
StorageReadinessHook *StorageReadinessHook |
| 241 |
|
| 242 |
// StorageVersionManager holds the storage versions of the API resources installed by this server. |
| 243 |
StorageVersionManager storageversion.Manager |
| 244 |
|
| 245 |
// EffectiveVersion determines which apis and features are available |
| 246 |
// based on when the api/feature lifecyle. |
| 247 |
EffectiveVersion basecompatibility.EffectiveVersion |
| 248 |
// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and |
| 249 |
// have higher priority than APIs of the same group resource enabled at the emulation version. |
| 250 |
// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed. |
| 251 |
// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set. |
| 252 |
// Not applicable to alpha APIs. |
| 253 |
EmulationForwardCompatible bool |
| 254 |
// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config. |
| 255 |
// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version. |
| 256 |
// If false, error would be thrown if any GroupVersion or GroupVersionResource explicitly enabled in the --runtime-config flag is introduced after the emulation version. |
| 257 |
RuntimeConfigEmulationForwardCompatible bool |
| 258 |
|
| 259 |
// FeatureGate is a way to plumb feature gate through if you have them. |
| 260 |
FeatureGate featuregate.FeatureGate |
| 261 |
|
| 262 |
// lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver. |
| 263 |
lifecycleSignals lifecycleSignals |
| 264 |
|
| 265 |
// destroyFns contains a list of functions that should be called on shutdown to clean up resources. |
| 266 |
destroyFns []func() |
| 267 |
|
| 268 |
// muxAndDiscoveryCompleteSignals holds signals that indicate all known HTTP paths have been registered. |
| 269 |
// it exists primarily to avoid returning a 404 response when a resource actually exists but we haven't installed the path to a handler. |
| 270 |
// it is exposed for easier composition of the individual servers. |
| 271 |
// the primary users of this field are the WithMuxCompleteProtection filter and the NotFoundHandler |
| 272 |
muxAndDiscoveryCompleteSignals map[string]<-chan struct{} |
| 273 |
|
| 274 |
// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP |
| 275 |
// Server during the graceful termination of the apiserver. If true, we wait |
| 276 |
// for non longrunning requests in flight to be drained and then initiate a |
| 277 |
// shutdown of the HTTP Server. If false, we initiate a shutdown of the HTTP |
| 278 |
// Server as soon as ShutdownDelayDuration has elapsed. |
| 279 |
// If enabled, after ShutdownDelayDuration elapses, any incoming request is |
| 280 |
// rejected with a 429 status code and a 'Retry-After' response. |
| 281 |
ShutdownSendRetryAfter bool |
| 282 |
|
| 283 |
// ShutdownWatchTerminationGracePeriod, if set to a positive value, |
| 284 |
// is the maximum duration the apiserver will wait for all active |
| 285 |
// watch request(s) to drain. |
| 286 |
// Once this grace period elapses, the apiserver will no longer |
| 287 |
// wait for any active watch request(s) in flight to drain, it will |
| 288 |
// proceed to the next step in the graceful server shutdown process. |
| 289 |
// If set to a positive value, the apiserver will keep track of the |
| 290 |
// number of active watch request(s) in flight and during shutdown |
| 291 |
// it will wait, at most, for the specified duration and allow these |
| 292 |
// active watch requests to drain with some rate limiting in effect. |
| 293 |
// The default is zero, which implies the apiserver will not keep |
| 294 |
// track of active watch request(s) in flight and will not wait |
| 295 |
// for them to drain, this maintains backward compatibility. |
| 296 |
// This grace period is orthogonal to other grace periods, and |
| 297 |
// it is not overridden by any other grace period. |
| 298 |
ShutdownWatchTerminationGracePeriod time.Duration |
| 299 |
} |
| 300 |
|
| 301 |
// DelegationTarget is an interface which allows for composition of API servers with top level handling that works |
| 302 |
// as expected. |
| 303 |
type DelegationTarget interface { |
| 304 |
// UnprotectedHandler returns a handler that is NOT protected by a normal chain |
| 305 |
UnprotectedHandler() http.Handler |
| 306 |
|
| 307 |
// PostStartHooks returns the post-start hooks that need to be combined |
| 308 |
PostStartHooks() map[string]postStartHookEntry |
| 309 |
|
| 310 |
// PreShutdownHooks returns the pre-stop hooks that need to be combined |
| 311 |
PreShutdownHooks() map[string]preShutdownHookEntry |
| 312 |
|
| 313 |
// HealthzChecks returns the healthz checks that need to be combined |
| 314 |
HealthzChecks() []healthz.HealthChecker |
| 315 |
|
| 316 |
// ListedPaths returns the paths for supporting an index |
| 317 |
ListedPaths() []string |
| 318 |
|
| 319 |
// NextDelegate returns the next delegationTarget in the chain of delegations |
| 320 |
NextDelegate() DelegationTarget |
| 321 |
|
| 322 |
// PrepareRun does post API installation setup steps. It calls recursively the same function of the delegates. |
| 323 |
PrepareRun() preparedGenericAPIServer |
| 324 |
|
| 325 |
// MuxAndDiscoveryCompleteSignals exposes registered signals that indicate if all known HTTP paths have been installed. |
| 326 |
MuxAndDiscoveryCompleteSignals() map[string]<-chan struct{} |
| 327 |
|
| 328 |
// Destroy cleans up its resources on shutdown. |
| 329 |
// Destroy has to be implemented in thread-safe way and be prepared |
| 330 |
// for being called more than once. |
| 331 |
Destroy() |
| 332 |
} |
| 333 |
|
| 334 |
func (s *GenericAPIServer) UnprotectedHandler() http.Handler { |
| 335 |
// when we delegate, we need the server we're delegating to choose whether or not to use gorestful |
| 336 |
return s.Handler.Director |
| 337 |
} |
| 338 |
func (s *GenericAPIServer) PostStartHooks() map[string]postStartHookEntry { |
| 339 |
return s.postStartHooks |
| 340 |
} |
| 341 |
func (s *GenericAPIServer) PreShutdownHooks() map[string]preShutdownHookEntry { |
| 342 |
return s.preShutdownHooks |
| 343 |
} |
| 344 |
func (s *GenericAPIServer) HealthzChecks() []healthz.HealthChecker { |
| 345 |
return s.healthzRegistry.checks |
| 346 |
} |
| 347 |
func (s *GenericAPIServer) ListedPaths() []string { |
| 348 |
return s.listedPathProvider.ListedPaths() |
| 349 |
} |
| 350 |
|
| 351 |
func (s *GenericAPIServer) NextDelegate() DelegationTarget { |
| 352 |
return s.delegationTarget |
| 353 |
} |
| 354 |
|
| 355 |
// RegisterMuxAndDiscoveryCompleteSignal registers the given signal that will be used to determine if all known |
| 356 |
// HTTP paths have been registered. It is okay to call this method after instantiating the generic server but before running. |
| 357 |
func (s *GenericAPIServer) RegisterMuxAndDiscoveryCompleteSignal(signalName string, signal <-chan struct{}) error { |
| 358 |
if _, exists := s.muxAndDiscoveryCompleteSignals[signalName]; exists { |
| 359 |
return fmt.Errorf("%s already registered", signalName) |
| 360 |
} |
| 361 |
s.muxAndDiscoveryCompleteSignals[signalName] = signal |
| 362 |
return nil |
| 363 |
} |
| 364 |
|
| 365 |
func (s *GenericAPIServer) MuxAndDiscoveryCompleteSignals() map[string]<-chan struct{} { |
| 366 |
return s.muxAndDiscoveryCompleteSignals |
| 367 |
} |
| 368 |
|
| 369 |
// RegisterDestroyFunc registers a function that will be called during Destroy(). |
| 370 |
// The function have to be idempotent and prepared to be called more than once. |
| 371 |
func (s *GenericAPIServer) RegisterDestroyFunc(destroyFn func()) { |
| 372 |
s.destroyFns = append(s.destroyFns, destroyFn) |
| 373 |
} |
| 374 |
|
| 375 |
// Destroy cleans up all its and its delegation target resources on shutdown. |
| 376 |
// It starts with destroying its own resources and later proceeds with |
| 377 |
// its delegation target. |
| 378 |
func (s *GenericAPIServer) Destroy() { |
| 379 |
for _, destroyFn := range s.destroyFns { |
| 380 |
destroyFn() |
| 381 |
} |
| 382 |
if s.delegationTarget != nil { |
| 383 |
s.delegationTarget.Destroy() |
| 384 |
} |
| 385 |
} |
| 386 |
|
| 387 |
type emptyDelegate struct { |
| 388 |
// handler is called at the end of the delegation chain |
| 389 |
// when a request has been made against an unregistered HTTP path the individual servers will simply pass it through until it reaches the handler. |
| 390 |
handler http.Handler |
| 391 |
} |
| 392 |
|
| 393 |
func NewEmptyDelegate() DelegationTarget { |
| 394 |
return emptyDelegate{} |
| 395 |
} |
| 396 |
|
| 397 |
// NewEmptyDelegateWithCustomHandler allows for registering a custom handler usually for special handling of 404 requests |
| 398 |
func NewEmptyDelegateWithCustomHandler(handler http.Handler) DelegationTarget { |
| 399 |
return emptyDelegate{handler} |
| 400 |
} |
| 401 |
|
| 402 |
func (s emptyDelegate) UnprotectedHandler() http.Handler { |
| 403 |
return s.handler |
| 404 |
} |
| 405 |
func (s emptyDelegate) PostStartHooks() map[string]postStartHookEntry { |
| 406 |
return map[string]postStartHookEntry{} |
| 407 |
} |
| 408 |
func (s emptyDelegate) PreShutdownHooks() map[string]preShutdownHookEntry { |
| 409 |
return map[string]preShutdownHookEntry{} |
| 410 |
} |
| 411 |
func (s emptyDelegate) HealthzChecks() []healthz.HealthChecker { |
| 412 |
return []healthz.HealthChecker{} |
| 413 |
} |
| 414 |
func (s emptyDelegate) ListedPaths() []string { |
| 415 |
return []string{} |
| 416 |
} |
| 417 |
func (s emptyDelegate) NextDelegate() DelegationTarget { |
| 418 |
return nil |
| 419 |
} |
| 420 |
func (s emptyDelegate) PrepareRun() preparedGenericAPIServer { |
| 421 |
return preparedGenericAPIServer{nil} |
| 422 |
} |
| 423 |
func (s emptyDelegate) MuxAndDiscoveryCompleteSignals() map[string]<-chan struct{} { |
| 424 |
return map[string]<-chan struct{}{} |
| 425 |
} |
| 426 |
func (s emptyDelegate) Destroy() { |
| 427 |
} |
| 428 |
|
| 429 |
// preparedGenericAPIServer is a private wrapper that enforces a call of PrepareRun() before Run can be invoked. |
| 430 |
type preparedGenericAPIServer struct { |
| 431 |
*GenericAPIServer |
| 432 |
} |
| 433 |
|
| 434 |
// PrepareRun does post API installation setup steps. It calls recursively the same function of the delegates. |
| 435 |
func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer { |
| 436 |
s.delegationTarget.PrepareRun() |
| 437 |
|
| 438 |
if s.openAPIConfig != nil && !s.skipOpenAPIInstallation { |
| 439 |
s.OpenAPIVersionedService, s.StaticOpenAPISpec = routes.OpenAPI{ |
| 440 |
Config: s.openAPIConfig, |
| 441 |
}.InstallV2(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux) |
| 442 |
} |
| 443 |
|
| 444 |
if s.openAPIV3Config != nil && !s.skipOpenAPIInstallation { |
| 445 |
s.OpenAPIV3VersionedService = routes.OpenAPI{ |
| 446 |
V3Config: s.openAPIV3Config, |
| 447 |
}.InstallV3(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux) |
| 448 |
} |
| 449 |
|
| 450 |
s.installHealthz() |
| 451 |
s.installLivez() |
| 452 |
|
| 453 |
// as soon as shutdown is initiated, readiness should start failing |
| 454 |
readinessStopCh := s.lifecycleSignals.ShutdownInitiated.Signaled() |
| 455 |
err := s.addReadyzShutdownCheck(readinessStopCh) |
| 456 |
if err != nil { |
| 457 |
klog.Errorf("Failed to install readyz shutdown check %s", err) |
| 458 |
} |
| 459 |
s.installReadyz() |
| 460 |
|
| 461 |
return preparedGenericAPIServer{s} |
| 462 |
} |
| 463 |
|
| 464 |
// Run spawns the secure http server. It only returns if stopCh is closed |
| 465 |
// or the secure port cannot be listened on initially. |
| 466 |
// |
| 467 |
// Deprecated: use RunWithContext instead. Run will not get removed to avoid |
| 468 |
// breaking consumers, but should not be used in new code. |
| 469 |
func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error { |
| 470 |
ctx := wait.ContextForChannel(stopCh) |
| 471 |
return s.RunWithContext(ctx) |
| 472 |
} |
| 473 |
|
| 474 |
// RunWithContext spawns the secure http server. It only returns if ctx is canceled |
| 475 |
// or the secure port cannot be listened on initially. |
| 476 |
// This is the diagram of what contexts/channels/signals are dependent on each other: |
| 477 |
// |
| 478 |
// | ctx |
| 479 |
// | | |
| 480 |
// | --------------------------------------------------------- |
| 481 |
// | | | |
| 482 |
// | ShutdownInitiated (shutdownInitiatedCh) | |
| 483 |
// | | | |
| 484 |
// | (ShutdownDelayDuration) (PreShutdownHooks) |
| 485 |
// | | | |
| 486 |
// | AfterShutdownDelayDuration (delayedStopCh) PreShutdownHooksStopped (preShutdownHooksHasStoppedCh) |
| 487 |
// | | | |
| 488 |
// | |-------------------------------------------------------| |
| 489 |
// | | |
| 490 |
// | | |
| 491 |
// | NotAcceptingNewRequest (notAcceptingNewRequestCh) |
| 492 |
// | | |
| 493 |
// | | |
| 494 |
// | |----------------------------------------------------------------------------------| |
| 495 |
// | | | | | |
| 496 |
// | [without [with | | |
| 497 |
// | ShutdownSendRetryAfter] ShutdownSendRetryAfter] | | |
| 498 |
// | | | | | |
| 499 |
// | | ---------------| | |
| 500 |
// | | | | |
| 501 |
// | | |----------------|-----------------------| | |
| 502 |
// | | | | | |
| 503 |
// | | (NonLongRunningRequestWaitGroup::Wait) (WatchRequestWaitGroup::Wait) | |
| 504 |
// | | | | | |
| 505 |
// | | |------------------|---------------------| | |
| 506 |
// | | | | |
| 507 |
// | | InFlightRequestsDrained (drainedCh) | |
| 508 |
// | | | | |
| 509 |
// | |-------------------|---------------------|----------------------------------------| |
| 510 |
// | | | |
| 511 |
// | stopHttpServerCtx (AuditBackend::Shutdown()) |
| 512 |
// | | |
| 513 |
// | listenerStoppedCh |
| 514 |
// | | |
| 515 |
// | HTTPServerStoppedListening (httpServerStoppedListeningCh) |
| 516 |
func (s preparedGenericAPIServer) RunWithContext(ctx context.Context) error { |
| 517 |
stopCh := ctx.Done() |
| 518 |
delayedStopCh := s.lifecycleSignals.AfterShutdownDelayDuration |
| 519 |
shutdownInitiatedCh := s.lifecycleSignals.ShutdownInitiated |
| 520 |
|
| 521 |
// Clean up resources on shutdown. |
| 522 |
defer s.Destroy() |
| 523 |
|
| 524 |
// If UDS profiling is enabled, start a local http server listening on that socket |
| 525 |
if s.UnprotectedDebugSocket != nil { |
| 526 |
go func() { |
| 527 |
defer utilruntime.HandleCrash() |
| 528 |
klog.Error(s.UnprotectedDebugSocket.Run(stopCh)) |
| 529 |
}() |
| 530 |
} |
| 531 |
|
| 532 |
// spawn a new goroutine for closing the MuxAndDiscoveryComplete signal |
| 533 |
// registration happens during construction of the generic api server |
| 534 |
// the last server in the chain aggregates signals from the previous instances |
| 535 |
go func() { |
| 536 |
for _, muxAndDiscoveryCompletedSignal := range s.GenericAPIServer.MuxAndDiscoveryCompleteSignals() { |
| 537 |
select { |
| 538 |
case <-muxAndDiscoveryCompletedSignal: |
| 539 |
continue |
| 540 |
case <-stopCh: |
| 541 |
klog.V(1).Infof("haven't completed %s, stop requested", s.lifecycleSignals.MuxAndDiscoveryComplete.Name()) |
| 542 |
return |
| 543 |
} |
| 544 |
} |
| 545 |
s.lifecycleSignals.MuxAndDiscoveryComplete.Signal() |
| 546 |
klog.V(1).Infof("%s has all endpoints registered and discovery information is complete", s.lifecycleSignals.MuxAndDiscoveryComplete.Name()) |
| 547 |
}() |
| 548 |
|
| 549 |
go func() { |
| 550 |
defer delayedStopCh.Signal() |
| 551 |
defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", delayedStopCh.Name()) |
| 552 |
|
| 553 |
<-stopCh |
| 554 |
|
| 555 |
// As soon as shutdown is initiated, /readyz should start returning failure. |
| 556 |
// This gives the load balancer a window defined by ShutdownDelayDuration to detect that /readyz is red |
| 557 |
// and stop sending traffic to this server. |
| 558 |
shutdownInitiatedCh.Signal() |
| 559 |
klog.V(1).InfoS("[graceful-termination] shutdown event", "name", shutdownInitiatedCh.Name()) |
| 560 |
|
| 561 |
time.Sleep(s.ShutdownDelayDuration) |
| 562 |
}() |
| 563 |
|
| 564 |
// close socket after delayed stopCh |
| 565 |
shutdownTimeout := s.ShutdownTimeout |
| 566 |
if s.ShutdownSendRetryAfter { |
| 567 |
// when this mode is enabled, we do the following: |
| 568 |
// - the server will continue to listen until all existing requests in flight |
| 569 |
// (not including active long running requests) have been drained. |
| 570 |
// - once drained, http Server Shutdown is invoked with a timeout of 2s, |
| 571 |
// net/http waits for 1s for the peer to respond to a GO_AWAY frame, so |
| 572 |
// we should wait for a minimum of 2s |
| 573 |
shutdownTimeout = 2 * time.Second |
| 574 |
klog.V(1).InfoS("[graceful-termination] using HTTP Server shutdown timeout", "shutdownTimeout", shutdownTimeout) |
| 575 |
} |
| 576 |
|
| 577 |
notAcceptingNewRequestCh := s.lifecycleSignals.NotAcceptingNewRequest |
| 578 |
drainedCh := s.lifecycleSignals.InFlightRequestsDrained |
| 579 |
// Canceling the parent context does not immediately cancel the HTTP server. |
| 580 |
// We only inherit context values here and deal with cancellation ourselves. |
| 581 |
stopHTTPServerCtx, stopHTTPServer := context.WithCancelCause(context.WithoutCancel(ctx)) |
| 582 |
go func() { |
| 583 |
defer stopHTTPServer(errors.New("time to stop HTTP server")) |
| 584 |
|
| 585 |
timeToStopHttpServerCh := notAcceptingNewRequestCh.Signaled() |
| 586 |
if s.ShutdownSendRetryAfter { |
| 587 |
timeToStopHttpServerCh = drainedCh.Signaled() |
| 588 |
} |
| 589 |
|
| 590 |
<-timeToStopHttpServerCh |
| 591 |
}() |
| 592 |
|
| 593 |
// Start the audit backend before any request comes in. This means we must call Backend.Run |
| 594 |
// before http server start serving. Otherwise the Backend.ProcessEvents call might block. |
| 595 |
// AuditBackend.Run will stop as soon as all in-flight requests are drained. |
| 596 |
if s.AuditBackend != nil { |
| 597 |
if err := s.AuditBackend.Run(drainedCh.Signaled()); err != nil { |
| 598 |
return fmt.Errorf("failed to run the audit backend: %v", err) |
| 599 |
} |
| 600 |
} |
| 601 |
|
| 602 |
stoppedCh, listenerStoppedCh, err := s.NonBlockingRunWithContext(stopHTTPServerCtx, shutdownTimeout) |
| 603 |
if err != nil { |
| 604 |
return err |
| 605 |
} |
| 606 |
|
| 607 |
httpServerStoppedListeningCh := s.lifecycleSignals.HTTPServerStoppedListening |
| 608 |
go func() { |
| 609 |
<-listenerStoppedCh |
| 610 |
httpServerStoppedListeningCh.Signal() |
| 611 |
klog.V(1).InfoS("[graceful-termination] shutdown event", "name", httpServerStoppedListeningCh.Name()) |
| 612 |
}() |
| 613 |
|
| 614 |
// we don't accept new request as soon as both ShutdownDelayDuration has |
| 615 |
// elapsed and preshutdown hooks have completed. |
| 616 |
preShutdownHooksHasStoppedCh := s.lifecycleSignals.PreShutdownHooksStopped |
| 617 |
go func() { |
| 618 |
defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", notAcceptingNewRequestCh.Name()) |
| 619 |
defer notAcceptingNewRequestCh.Signal() |
| 620 |
|
| 621 |
// wait for the delayed stopCh before closing the handler chain |
| 622 |
<-delayedStopCh.Signaled() |
| 623 |
|
| 624 |
// Additionally wait for preshutdown hooks to also be finished, as some of them need |
| 625 |
// to send API calls to clean up after themselves (e.g. lease reconcilers removing |
| 626 |
// itself from the active servers). |
| 627 |
<-preShutdownHooksHasStoppedCh.Signaled() |
| 628 |
}() |
| 629 |
|
| 630 |
// wait for all in-flight non-long running requests to finish |
| 631 |
nonLongRunningRequestDrainedCh := make(chan struct{}) |
| 632 |
go func() { |
| 633 |
defer close(nonLongRunningRequestDrainedCh) |
| 634 |
defer klog.V(1).Info("[graceful-termination] in-flight non long-running request(s) have drained") |
| 635 |
|
| 636 |
// wait for the delayed stopCh before closing the handler chain (it rejects everything after Wait has been called). |
| 637 |
<-notAcceptingNewRequestCh.Signaled() |
| 638 |
|
| 639 |
// Wait for all requests to finish, which are bounded by the RequestTimeout variable. |
| 640 |
// once NonLongRunningRequestWaitGroup.Wait is invoked, the apiserver is |
| 641 |
// expected to reject any incoming request with a {503, Retry-After} |
| 642 |
// response via the WithWaitGroup filter. On the contrary, we observe |
| 643 |
// that incoming request(s) get a 'connection refused' error, this is |
| 644 |
// because, at this point, we have called 'Server.Shutdown' and |
| 645 |
// net/http server has stopped listening. This causes incoming |
| 646 |
// request to get a 'connection refused' error. |
| 647 |
// On the other hand, if 'ShutdownSendRetryAfter' is enabled incoming |
| 648 |
// requests will be rejected with a {429, Retry-After} since |
| 649 |
// 'Server.Shutdown' will be invoked only after in-flight requests |
| 650 |
// have been drained. |
| 651 |
// TODO: can we consolidate these two modes of graceful termination? |
| 652 |
s.NonLongRunningRequestWaitGroup.Wait() |
| 653 |
}() |
| 654 |
|
| 655 |
// wait for all in-flight watches to finish |
| 656 |
activeWatchesDrainedCh := make(chan struct{}) |
| 657 |
go func() { |
| 658 |
defer close(activeWatchesDrainedCh) |
| 659 |
|
| 660 |
<-notAcceptingNewRequestCh.Signaled() |
| 661 |
if s.ShutdownWatchTerminationGracePeriod <= time.Duration(0) { |
| 662 |
klog.V(1).InfoS("[graceful-termination] not going to wait for active watch request(s) to drain") |
| 663 |
return |
| 664 |
} |
| 665 |
|
| 666 |
// Wait for all active watches to finish |
| 667 |
grace := s.ShutdownWatchTerminationGracePeriod |
| 668 |
activeBefore, activeAfter, err := s.WatchRequestWaitGroup.Wait(func(count int) (utilwaitgroup.RateLimiter, context.Context, context.CancelFunc) { |
| 669 |
qps := float64(count) / grace.Seconds() |
| 670 |
// TODO: we don't want the QPS (max requests drained per second) to |
| 671 |
// get below a certain floor value, since we want the server to |
| 672 |
// drain the active watch requests as soon as possible. |
| 673 |
// For now, it's hard coded to 200, and it is subject to change |
| 674 |
// based on the result from the scale testing. |
| 675 |
if qps < 200 { |
| 676 |
qps = 200 |
| 677 |
} |
| 678 |
|
| 679 |
ctx, cancel := context.WithTimeout(context.Background(), grace) |
| 680 |
// We don't expect more than one token to be consumed |
| 681 |
// in a single Wait call, so setting burst to 1. |
| 682 |
return rate.NewLimiter(rate.Limit(qps), 1), ctx, cancel |
| 683 |
}) |
| 684 |
klog.V(1).InfoS("[graceful-termination] active watch request(s) have drained", |
| 685 |
"duration", grace, "activeWatchesBefore", activeBefore, "activeWatchesAfter", activeAfter, "error", err) |
| 686 |
}() |
| 687 |
|
| 688 |
go func() { |
| 689 |
defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", drainedCh.Name()) |
| 690 |
defer drainedCh.Signal() |
| 691 |
|
| 692 |
<-nonLongRunningRequestDrainedCh |
| 693 |
<-activeWatchesDrainedCh |
| 694 |
}() |
| 695 |
|
| 696 |
klog.V(1).Info("[graceful-termination] waiting for shutdown to be initiated") |
| 697 |
<-stopCh |
| 698 |
|
| 699 |
// run shutdown hooks directly. This includes deregistering from |
| 700 |
// the kubernetes endpoint in case of kube-apiserver. |
| 701 |
func() { |
| 702 |
defer func() { |
| 703 |
preShutdownHooksHasStoppedCh.Signal() |
| 704 |
klog.V(1).InfoS("[graceful-termination] pre-shutdown hooks completed", "name", preShutdownHooksHasStoppedCh.Name()) |
| 705 |
}() |
| 706 |
err = s.RunPreShutdownHooks() |
| 707 |
}() |
| 708 |
if err != nil { |
| 709 |
return err |
| 710 |
} |
| 711 |
|
| 712 |
// Wait for all requests in flight to drain, bounded by the RequestTimeout variable. |
| 713 |
<-drainedCh.Signaled() |
| 714 |
|
| 715 |
if s.AuditBackend != nil { |
| 716 |
s.AuditBackend.Shutdown() |
| 717 |
klog.V(1).InfoS("[graceful-termination] audit backend shutdown completed") |
| 718 |
} |
| 719 |
|
| 720 |
// wait for stoppedCh that is closed when the graceful termination (server.Shutdown) is finished. |
| 721 |
<-listenerStoppedCh |
| 722 |
<-stoppedCh |
| 723 |
|
| 724 |
klog.V(1).Info("[graceful-termination] apiserver is exiting") |
| 725 |
return nil |
| 726 |
} |
| 727 |
|
| 728 |
// NonBlockingRun spawns the secure http server. An error is |
| 729 |
// returned if the secure port cannot be listened on. |
| 730 |
// The returned channel is closed when the (asynchronous) termination is finished. |
| 731 |
// |
| 732 |
// Deprecated: use RunWithContext instead. Run will not get removed to avoid |
| 733 |
// breaking consumers, but should not be used in new code. |
| 734 |
func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}, shutdownTimeout time.Duration) (<-chan struct{}, <-chan struct{}, error) { |
| 735 |
ctx := wait.ContextForChannel(stopCh) |
| 736 |
return s.NonBlockingRunWithContext(ctx, shutdownTimeout) |
| 737 |
} |
| 738 |
|
| 739 |
// NonBlockingRunWithContext spawns the secure http server. An error is |
| 740 |
// returned if the secure port cannot be listened on. |
| 741 |
// The returned channel is closed when the (asynchronous) termination is finished. |
| 742 |
func (s preparedGenericAPIServer) NonBlockingRunWithContext(ctx context.Context, shutdownTimeout time.Duration) (<-chan struct{}, <-chan struct{}, error) { |
| 743 |
// Use an internal stop channel to allow cleanup of the listeners on error. |
| 744 |
internalStopCh := make(chan struct{}) |
| 745 |
var stoppedCh <-chan struct{} |
| 746 |
var listenerStoppedCh <-chan struct{} |
| 747 |
if s.SecureServingInfo != nil && s.Handler != nil { |
| 748 |
var err error |
| 749 |
stoppedCh, listenerStoppedCh, err = s.SecureServingInfo.Serve(s.Handler, shutdownTimeout, internalStopCh) |
| 750 |
if err != nil { |
| 751 |
close(internalStopCh) |
| 752 |
return nil, nil, err |
| 753 |
} |
| 754 |
} |
| 755 |
|
| 756 |
// Now that listener have bound successfully, it is the |
| 757 |
// responsibility of the caller to close the provided channel to |
| 758 |
// ensure cleanup. |
| 759 |
go func() { |
| 760 |
<-ctx.Done() |
| 761 |
close(internalStopCh) |
| 762 |
}() |
| 763 |
|
| 764 |
s.RunPostStartHooks(ctx) |
| 765 |
|
| 766 |
if _, err := systemd.SdNotify(true, "READY=1\n"); err != nil { |
| 767 |
klog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) |
| 768 |
} |
| 769 |
|
| 770 |
return stoppedCh, listenerStoppedCh, nil |
| 771 |
} |
| 772 |
|
| 773 |
// installAPIResources is a private method for installing the REST storage backing each api groupversionresource |
| 774 |
func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo, typeConverter managedfields.TypeConverter) error { |
| 775 |
var resourceInfos []*storageversion.ResourceInfo |
| 776 |
for _, groupVersion := range apiGroupInfo.PrioritizedVersions { |
| 777 |
if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 { |
| 778 |
klog.Warningf("Skipping API %v because it has no resources.", groupVersion) |
| 779 |
continue |
| 780 |
} |
| 781 |
|
| 782 |
apiGroupVersion, err := s.getAPIGroupVersion(apiGroupInfo, groupVersion, apiPrefix) |
| 783 |
if err != nil { |
| 784 |
return err |
| 785 |
} |
| 786 |
if apiGroupInfo.OptionsExternalVersion != nil { |
| 787 |
apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion |
| 788 |
} |
| 789 |
apiGroupVersion.TypeConverter = typeConverter |
| 790 |
apiGroupVersion.MaxRequestBodyBytes = s.maxRequestBodyBytes |
| 791 |
|
| 792 |
discoveryAPIResources, r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer) |
| 793 |
|
| 794 |
if err != nil { |
| 795 |
return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err) |
| 796 |
} |
| 797 |
resourceInfos = append(resourceInfos, r...) |
| 798 |
|
| 799 |
// Aggregated discovery only aggregates resources under /apis |
| 800 |
if apiPrefix == APIGroupPrefix { |
| 801 |
s.AggregatedDiscoveryGroupManager.AddGroupVersion( |
| 802 |
groupVersion.Group, |
| 803 |
apidiscoveryv2.APIVersionDiscovery{ |
| 804 |
Freshness: apidiscoveryv2.DiscoveryFreshnessCurrent, |
| 805 |
Version: groupVersion.Version, |
| 806 |
Resources: discoveryAPIResources, |
| 807 |
}, |
| 808 |
) |
| 809 |
} else { |
| 810 |
// There is only one group version for legacy resources, priority can be defaulted to 0. |
| 811 |
s.AggregatedLegacyDiscoveryGroupManager.AddGroupVersion( |
| 812 |
groupVersion.Group, |
| 813 |
apidiscoveryv2.APIVersionDiscovery{ |
| 814 |
Freshness: apidiscoveryv2.DiscoveryFreshnessCurrent, |
| 815 |
Version: groupVersion.Version, |
| 816 |
Resources: discoveryAPIResources, |
| 817 |
}, |
| 818 |
) |
| 819 |
} |
| 820 |
|
| 821 |
} |
| 822 |
|
| 823 |
s.RegisterDestroyFunc(apiGroupInfo.destroyStorage) |
| 824 |
|
| 825 |
if s.FeatureGate.Enabled(features.StorageVersionAPI) && |
| 826 |
s.FeatureGate.Enabled(features.APIServerIdentity) { |
| 827 |
// API installation happens before we start listening on the handlers, |
| 828 |
// therefore it is safe to register ResourceInfos here. The handler will block |
| 829 |
// write requests until the storage versions of the targeting resources are updated. |
| 830 |
s.StorageVersionManager.AddResourceInfo(resourceInfos...) |
| 831 |
} |
| 832 |
|
| 833 |
return nil |
| 834 |
} |
| 835 |
|
| 836 |
// InstallLegacyAPIGroup exposes the given legacy api group in the API. |
| 837 |
// The <apiGroupInfo> passed into this function shouldn't be used elsewhere as the |
| 838 |
// underlying storage will be destroyed on this servers shutdown. |
| 839 |
func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo *APIGroupInfo) error { |
| 840 |
if !s.legacyAPIGroupPrefixes.Has(apiPrefix) { |
| 841 |
return fmt.Errorf("%q is not in the allowed legacy API prefixes: %v", apiPrefix, s.legacyAPIGroupPrefixes.List()) |
| 842 |
} |
| 843 |
|
| 844 |
openAPIModels, err := s.getOpenAPIModels(apiPrefix, apiGroupInfo) |
| 845 |
if err != nil { |
| 846 |
return fmt.Errorf("unable to get openapi models: %v", err) |
| 847 |
} |
| 848 |
|
| 849 |
if err := s.installAPIResources(apiPrefix, apiGroupInfo, openAPIModels); err != nil { |
| 850 |
return err |
| 851 |
} |
| 852 |
|
| 853 |
// Install the version handler. |
| 854 |
// Add a handler at /<apiPrefix> to enumerate the supported api versions. |
| 855 |
legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix) |
| 856 |
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager) |
| 857 |
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{})) |
| 858 |
s.registerStorageReadinessCheck("", apiGroupInfo) |
| 859 |
|
| 860 |
return nil |
| 861 |
} |
| 862 |
|
| 863 |
// InstallAPIGroups exposes given api groups in the API. |
| 864 |
// The <apiGroupInfos> passed into this function shouldn't be used elsewhere as the |
| 865 |
// underlying storage will be destroyed on this servers shutdown. |
| 866 |
func (s *GenericAPIServer) InstallAPIGroups(apiGroupInfos ...*APIGroupInfo) error { |
| 867 |
for _, apiGroupInfo := range apiGroupInfos { |
| 868 |
if len(apiGroupInfo.PrioritizedVersions) == 0 { |
| 869 |
return fmt.Errorf("no version priority set for %#v", *apiGroupInfo) |
| 870 |
} |
| 871 |
// Do not register empty group or empty version. Doing so claims /apis/ for the wrong entity to be returned. |
| 872 |
// Catching these here places the error much closer to its origin |
| 873 |
if len(apiGroupInfo.PrioritizedVersions[0].Group) == 0 { |
| 874 |
return fmt.Errorf("cannot register handler with an empty group for %#v", *apiGroupInfo) |
| 875 |
} |
| 876 |
if len(apiGroupInfo.PrioritizedVersions[0].Version) == 0 { |
| 877 |
return fmt.Errorf("cannot register handler with an empty version for %#v", *apiGroupInfo) |
| 878 |
} |
| 879 |
} |
| 880 |
|
| 881 |
openAPIModels, err := s.getOpenAPIModels(APIGroupPrefix, apiGroupInfos...) |
| 882 |
if err != nil { |
| 883 |
return fmt.Errorf("unable to get openapi models: %v", err) |
| 884 |
} |
| 885 |
|
| 886 |
for _, apiGroupInfo := range apiGroupInfos { |
| 887 |
if err := s.installAPIResources(APIGroupPrefix, apiGroupInfo, openAPIModels); err != nil { |
| 888 |
return fmt.Errorf("unable to install api resources: %v", err) |
| 889 |
} |
| 890 |
|
| 891 |
// setup discovery |
| 892 |
// Install the version handler. |
| 893 |
// Add a handler at /apis/<groupName> to enumerate all versions supported by this group. |
| 894 |
apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{} |
| 895 |
for _, groupVersion := range apiGroupInfo.PrioritizedVersions { |
| 896 |
// Check the config to make sure that we elide versions that don't have any resources |
| 897 |
if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 { |
| 898 |
continue |
| 899 |
} |
| 900 |
apiVersionsForDiscovery = append(apiVersionsForDiscovery, metav1.GroupVersionForDiscovery{ |
| 901 |
GroupVersion: groupVersion.String(), |
| 902 |
Version: groupVersion.Version, |
| 903 |
}) |
| 904 |
} |
| 905 |
preferredVersionForDiscovery := metav1.GroupVersionForDiscovery{ |
| 906 |
GroupVersion: apiGroupInfo.PrioritizedVersions[0].String(), |
| 907 |
Version: apiGroupInfo.PrioritizedVersions[0].Version, |
| 908 |
} |
| 909 |
apiGroup := metav1.APIGroup{ |
| 910 |
Name: apiGroupInfo.PrioritizedVersions[0].Group, |
| 911 |
Versions: apiVersionsForDiscovery, |
| 912 |
PreferredVersion: preferredVersionForDiscovery, |
| 913 |
} |
| 914 |
|
| 915 |
s.DiscoveryGroupManager.AddGroup(apiGroup) |
| 916 |
s.Handler.GoRestfulContainer.Add(discovery.NewAPIGroupHandler(s.Serializer, apiGroup).WebService()) |
| 917 |
s.registerStorageReadinessCheck(apiGroupInfo.PrioritizedVersions[0].Group, apiGroupInfo) |
| 918 |
} |
| 919 |
return nil |
| 920 |
} |
| 921 |
|
| 922 |
// registerStorageReadinessCheck registers the readiness checks for all underlying storages |
| 923 |
// for a given APIGroup. |
| 924 |
func (s *GenericAPIServer) registerStorageReadinessCheck(groupName string, apiGroupInfo *APIGroupInfo) { |
| 925 |
for version, storageMap := range apiGroupInfo.VersionedResourcesStorageMap { |
| 926 |
for resource, storage := range storageMap { |
| 927 |
if withReadiness, ok := storage.(rest.StorageWithReadiness); ok { |
| 928 |
gvr := metav1.GroupVersionResource{ |
| 929 |
Group: groupName, |
| 930 |
Version: version, |
| 931 |
Resource: resource, |
| 932 |
} |
| 933 |
s.StorageReadinessHook.RegisterStorage(gvr, withReadiness) |
| 934 |
} |
| 935 |
} |
| 936 |
} |
| 937 |
} |
| 938 |
|
| 939 |
// InstallAPIGroup exposes the given api group in the API. |
| 940 |
// The <apiGroupInfo> passed into this function shouldn't be used elsewhere as the |
| 941 |
// underlying storage will be destroyed on this servers shutdown. |
| 942 |
func (s *GenericAPIServer) InstallAPIGroup(apiGroupInfo *APIGroupInfo) error { |
| 943 |
return s.InstallAPIGroups(apiGroupInfo) |
| 944 |
} |
| 945 |
|
| 946 |
func (s *GenericAPIServer) getAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion, apiPrefix string) (*genericapi.APIGroupVersion, error) { |
| 947 |
storage := make(map[string]rest.Storage) |
| 948 |
for k, v := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] { |
| 949 |
if strings.ToLower(k) != k { |
| 950 |
return nil, fmt.Errorf("resource names must be lowercase only, not %q", k) |
| 951 |
} |
| 952 |
storage[k] = v |
| 953 |
} |
| 954 |
version := s.newAPIGroupVersion(apiGroupInfo, groupVersion) |
| 955 |
version.Root = apiPrefix |
| 956 |
version.Storage = storage |
| 957 |
return version, nil |
| 958 |
} |
| 959 |
|
| 960 |
func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion) *genericapi.APIGroupVersion { |
| 961 |
|
| 962 |
allServedVersionsByResource := map[string][]string{} |
| 963 |
for version, resourcesInVersion := range apiGroupInfo.VersionedResourcesStorageMap { |
| 964 |
for resource := range resourcesInVersion { |
| 965 |
if len(groupVersion.Group) == 0 { |
| 966 |
allServedVersionsByResource[resource] = append(allServedVersionsByResource[resource], version) |
| 967 |
} else { |
| 968 |
allServedVersionsByResource[resource] = append(allServedVersionsByResource[resource], fmt.Sprintf("%s/%s", groupVersion.Group, version)) |
| 969 |
} |
| 970 |
} |
| 971 |
} |
| 972 |
|
| 973 |
return &genericapi.APIGroupVersion{ |
| 974 |
GroupVersion: groupVersion, |
| 975 |
AllServedVersionsByResource: allServedVersionsByResource, |
| 976 |
MetaGroupVersion: apiGroupInfo.MetaGroupVersion, |
| 977 |
|
| 978 |
ParameterCodec: apiGroupInfo.ParameterCodec, |
| 979 |
Serializer: apiGroupInfo.NegotiatedSerializer, |
| 980 |
Creater: apiGroupInfo.Scheme, |
| 981 |
Convertor: apiGroupInfo.Scheme, |
| 982 |
ConvertabilityChecker: apiGroupInfo.Scheme, |
| 983 |
UnsafeConvertor: runtime.UnsafeObjectConvertor(apiGroupInfo.Scheme), |
| 984 |
Defaulter: apiGroupInfo.Scheme, |
| 985 |
Typer: apiGroupInfo.Scheme, |
| 986 |
Namer: runtime.Namer(meta.NewAccessor()), |
| 987 |
|
| 988 |
EquivalentResourceRegistry: s.EquivalentResourceRegistry, |
| 989 |
|
| 990 |
Admit: s.admissionControl, |
| 991 |
MinRequestTimeout: s.minRequestTimeout, |
| 992 |
Authorizer: s.Authorizer, |
| 993 |
} |
| 994 |
} |
| 995 |
|
| 996 |
// NewDefaultAPIGroupInfo returns an APIGroupInfo stubbed with "normal" values |
| 997 |
// exposed for easier composition from other packages |
| 998 |
func NewDefaultAPIGroupInfo(group string, scheme *runtime.Scheme, parameterCodec runtime.ParameterCodec, codecs serializer.CodecFactory) APIGroupInfo { |
| 999 |
opts := []serializer.CodecFactoryOptionsMutator{} |
| 1000 |
if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) { |
| 1001 |
opts = append(opts, serializer.WithSerializer(cbor.NewSerializerInfo)) |
| 1002 |
} |
| 1003 |
if utilfeature.DefaultFeatureGate.Enabled(features.StreamingCollectionEncodingToJSON) { |
| 1004 |
opts = append(opts, serializer.WithStreamingCollectionEncodingToJSON()) |
| 1005 |
} |
| 1006 |
if utilfeature.DefaultFeatureGate.Enabled(features.StreamingCollectionEncodingToProtobuf) { |
| 1007 |
opts = append(opts, serializer.WithStreamingCollectionEncodingToProtobuf()) |
| 1008 |
} |
| 1009 |
if len(opts) != 0 { |
| 1010 |
codecs = serializer.NewCodecFactory(scheme, opts...) |
| 1011 |
} |
| 1012 |
return APIGroupInfo{ |
| 1013 |
PrioritizedVersions: scheme.PrioritizedVersionsForGroup(group), |
| 1014 |
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{}, |
| 1015 |
// TODO unhardcode this. It was hardcoded before, but we need to re-evaluate |
| 1016 |
OptionsExternalVersion: &schema.GroupVersion{Version: "v1"}, |
| 1017 |
Scheme: scheme, |
| 1018 |
ParameterCodec: parameterCodec, |
| 1019 |
NegotiatedSerializer: codecs, |
| 1020 |
} |
| 1021 |
} |
| 1022 |
|
| 1023 |
// getOpenAPIModels is a private method for getting the OpenAPI models |
| 1024 |
func (s *GenericAPIServer) getOpenAPIModels(apiPrefix string, apiGroupInfos ...*APIGroupInfo) (managedfields.TypeConverter, error) { |
| 1025 |
if s.openAPIV3Config == nil { |
| 1026 |
// SSA is GA and requires OpenAPI config to be set |
| 1027 |
// to create models. |
| 1028 |
return nil, errors.New("OpenAPIV3 config must not be nil") |
| 1029 |
} |
| 1030 |
pathsToIgnore := openapiutil.NewTrie(s.openAPIV3Config.IgnorePrefixes) |
| 1031 |
resourceNames := make([]string, 0) |
| 1032 |
for _, apiGroupInfo := range apiGroupInfos { |
| 1033 |
groupResources, err := getResourceNamesForGroup(apiPrefix, apiGroupInfo, pathsToIgnore) |
| 1034 |
if err != nil { |
| 1035 |
return nil, err |
| 1036 |
} |
| 1037 |
resourceNames = append(resourceNames, groupResources...) |
| 1038 |
} |
| 1039 |
|
| 1040 |
// Build the openapi definitions for those resources and convert it to proto models |
| 1041 |
openAPISpec, err := openapibuilder3.BuildOpenAPIDefinitionsForResources(s.openAPIV3Config, resourceNames...) |
| 1042 |
if err != nil { |
| 1043 |
return nil, err |
| 1044 |
} |
| 1045 |
for _, apiGroupInfo := range apiGroupInfos { |
| 1046 |
apiGroupInfo.StaticOpenAPISpec = openAPISpec |
| 1047 |
} |
| 1048 |
|
| 1049 |
typeConverter, err := managedfields.NewTypeConverter(openAPISpec, false) |
| 1050 |
if err != nil { |
| 1051 |
return nil, err |
| 1052 |
} |
| 1053 |
|
| 1054 |
return typeConverter, nil |
| 1055 |
} |
| 1056 |
|
| 1057 |
// getResourceNamesForGroup is a private method for getting the canonical names for each resource to build in an api group |
| 1058 |
func getResourceNamesForGroup(apiPrefix string, apiGroupInfo *APIGroupInfo, pathsToIgnore openapiutil.Trie) ([]string, error) { |
| 1059 |
// Get the canonical names of every resource we need to build in this api group |
| 1060 |
resourceNames := make([]string, 0) |
| 1061 |
for _, groupVersion := range apiGroupInfo.PrioritizedVersions { |
| 1062 |
for resource, storage := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] { |
| 1063 |
path := gpath.Join(apiPrefix, groupVersion.Group, groupVersion.Version, resource) |
| 1064 |
if !pathsToIgnore.HasPrefix(path) { |
| 1065 |
kind, err := genericapi.GetResourceKind(groupVersion, storage, apiGroupInfo.Scheme) |
| 1066 |
if err != nil { |
| 1067 |
return nil, err |
| 1068 |
} |
| 1069 |
sampleObject, err := apiGroupInfo.Scheme.New(kind) |
| 1070 |
if err != nil { |
| 1071 |
return nil, err |
| 1072 |
} |
| 1073 |
name := openapiutil.GetCanonicalTypeName(sampleObject) |
| 1074 |
resourceNames = append(resourceNames, name) |
| 1075 |
} |
| 1076 |
} |
| 1077 |
} |
| 1078 |
|
| 1079 |
return resourceNames, nil |
| 1080 |
} |