| 1 |
/* |
| 2 |
Copyright 2016 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package server |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"crypto/sha256" |
| 22 |
"encoding/base32" |
| 23 |
"fmt" |
| 24 |
"net" |
| 25 |
"net/http" |
| 26 |
"os" |
| 27 |
goruntime "runtime" |
| 28 |
"runtime/debug" |
| 29 |
"sort" |
| 30 |
"strconv" |
| 31 |
"strings" |
| 32 |
"sync/atomic" |
| 33 |
"time" |
| 34 |
|
| 35 |
"github.com/google/uuid" |
| 36 |
"golang.org/x/crypto/cryptobyte" |
| 37 |
jsonpatch "gopkg.in/evanphx/json-patch.v4" |
| 38 |
|
| 39 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 40 |
"k8s.io/apimachinery/pkg/runtime" |
| 41 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 42 |
"k8s.io/apimachinery/pkg/runtime/serializer" |
| 43 |
utilruntime "k8s.io/apimachinery/pkg/util/runtime" |
| 44 |
"k8s.io/apimachinery/pkg/util/sets" |
| 45 |
"k8s.io/apimachinery/pkg/util/version" |
| 46 |
utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" |
| 47 |
"k8s.io/apiserver/pkg/admission" |
| 48 |
"k8s.io/apiserver/pkg/audit" |
| 49 |
"k8s.io/apiserver/pkg/authentication/authenticator" |
| 50 |
"k8s.io/apiserver/pkg/authentication/authenticatorfactory" |
| 51 |
authenticatorunion "k8s.io/apiserver/pkg/authentication/request/union" |
| 52 |
"k8s.io/apiserver/pkg/authentication/user" |
| 53 |
"k8s.io/apiserver/pkg/authorization/authorizer" |
| 54 |
"k8s.io/apiserver/pkg/endpoints/discovery" |
| 55 |
discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" |
| 56 |
"k8s.io/apiserver/pkg/endpoints/filterlatency" |
| 57 |
genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" |
| 58 |
apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi" |
| 59 |
apirequest "k8s.io/apiserver/pkg/endpoints/request" |
| 60 |
genericfeatures "k8s.io/apiserver/pkg/features" |
| 61 |
genericregistry "k8s.io/apiserver/pkg/registry/generic" |
| 62 |
"k8s.io/apiserver/pkg/server/dynamiccertificates" |
| 63 |
"k8s.io/apiserver/pkg/server/egressselector" |
| 64 |
genericfilters "k8s.io/apiserver/pkg/server/filters" |
| 65 |
"k8s.io/apiserver/pkg/server/healthz" |
| 66 |
"k8s.io/apiserver/pkg/server/routes" |
| 67 |
"k8s.io/apiserver/pkg/server/routine" |
| 68 |
serverstore "k8s.io/apiserver/pkg/server/storage" |
| 69 |
storagevalue "k8s.io/apiserver/pkg/storage/value" |
| 70 |
"k8s.io/apiserver/pkg/storageversion" |
| 71 |
utilfeature "k8s.io/apiserver/pkg/util/feature" |
| 72 |
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" |
| 73 |
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" |
| 74 |
"k8s.io/client-go/informers" |
| 75 |
restclient "k8s.io/client-go/rest" |
| 76 |
basecompatibility "k8s.io/component-base/compatibility" |
| 77 |
"k8s.io/component-base/featuregate" |
| 78 |
"k8s.io/component-base/logs" |
| 79 |
"k8s.io/component-base/metrics/features" |
| 80 |
"k8s.io/component-base/metrics/prometheus/slis" |
| 81 |
"k8s.io/component-base/tracing" |
| 82 |
"k8s.io/component-base/zpages/flagz" |
| 83 |
"k8s.io/klog/v2" |
| 84 |
openapicommon "k8s.io/kube-openapi/pkg/common" |
| 85 |
"k8s.io/kube-openapi/pkg/spec3" |
| 86 |
"k8s.io/kube-openapi/pkg/validation/spec" |
| 87 |
"k8s.io/utils/clock" |
| 88 |
utilsnet "k8s.io/utils/net" |
| 89 |
|
| 90 |
// install apis |
| 91 |
_ "k8s.io/apiserver/pkg/apis/apiserver/install" |
| 92 |
) |
| 93 |
|
| 94 |
// hostnameFunc is a function to set the hostnameFunc of this apiserver. |
| 95 |
// To be used for testing purpose only, to simulate scenarios where multiple apiservers |
| 96 |
// exist. In such cases we want to ensure unique apiserver IDs which are a hash of hostnameFunc. |
| 97 |
var ( |
| 98 |
hostnameFunc = os.Hostname |
| 99 |
) |
| 100 |
|
| 101 |
const ( |
| 102 |
// DefaultLegacyAPIPrefix is where the legacy APIs will be located. |
| 103 |
DefaultLegacyAPIPrefix = "/api" |
| 104 |
|
| 105 |
// APIGroupPrefix is where non-legacy API group will be located. |
| 106 |
APIGroupPrefix = "/apis" |
| 107 |
) |
| 108 |
|
| 109 |
// Config is a structure used to configure a GenericAPIServer. |
| 110 |
// Its members are sorted roughly in order of importance for composers. |
| 111 |
type Config struct { |
| 112 |
// SecureServing is required to serve https |
| 113 |
SecureServing *SecureServingInfo |
| 114 |
|
| 115 |
// Authentication is the configuration for authentication |
| 116 |
Authentication AuthenticationInfo |
| 117 |
|
| 118 |
// Authorization is the configuration for authorization |
| 119 |
Authorization AuthorizationInfo |
| 120 |
|
| 121 |
// LoopbackClientConfig is a config for a privileged loopback connection to the API server |
| 122 |
// This is required for proper functioning of the PostStartHooks on a GenericAPIServer |
| 123 |
// TODO: move into SecureServing(WithLoopback) as soon as insecure serving is gone |
| 124 |
LoopbackClientConfig *restclient.Config |
| 125 |
|
| 126 |
// EgressSelector provides a lookup mechanism for dialing outbound connections. |
| 127 |
// It does so based on a EgressSelectorConfiguration which was read at startup. |
| 128 |
EgressSelector *egressselector.EgressSelector |
| 129 |
|
| 130 |
// RuleResolver is required to get the list of rules that apply to a given user |
| 131 |
// in a given namespace |
| 132 |
RuleResolver authorizer.RuleResolver |
| 133 |
// AdmissionControl performs deep inspection of a given request (including content) |
| 134 |
// to set values and determine whether its allowed |
| 135 |
AdmissionControl admission.Interface |
| 136 |
CorsAllowedOriginList []string |
| 137 |
HSTSDirectives []string |
| 138 |
// FlowControl, if not nil, gives priority and fairness to request handling |
| 139 |
FlowControl utilflowcontrol.Interface |
| 140 |
|
| 141 |
EnableIndex bool |
| 142 |
EnableProfiling bool |
| 143 |
DebugSocketPath string |
| 144 |
EnableDiscovery bool |
| 145 |
|
| 146 |
// Requires generic profiling enabled |
| 147 |
EnableContentionProfiling bool |
| 148 |
EnableMetrics bool |
| 149 |
|
| 150 |
DisabledPostStartHooks sets.String |
| 151 |
// done values in this values for this map are ignored. |
| 152 |
PostStartHooks map[string]PostStartHookConfigEntry |
| 153 |
|
| 154 |
// EffectiveVersion determines which apis and features are available |
| 155 |
// based on when the api/feature lifecyle. |
| 156 |
EffectiveVersion basecompatibility.EffectiveVersion |
| 157 |
// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and |
| 158 |
// have higher priority than APIs of the same group resource enabled at the emulation version. |
| 159 |
// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed. |
| 160 |
// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set. |
| 161 |
// Not applicable to alpha APIs. |
| 162 |
EmulationForwardCompatible bool |
| 163 |
// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config. |
| 164 |
// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version. |
| 165 |
// If false, error would be thrown if any GroupVersion or GroupVersionResource explicitly enabled in the --runtime-config flag is introduced after the emulation version. |
| 166 |
RuntimeConfigEmulationForwardCompatible bool |
| 167 |
// FeatureGate is a way to plumb feature gate through if you have them. |
| 168 |
FeatureGate featuregate.FeatureGate |
| 169 |
// AuditBackend is where audit events are sent to. |
| 170 |
AuditBackend audit.Backend |
| 171 |
// AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request. |
| 172 |
AuditPolicyRuleEvaluator audit.PolicyRuleEvaluator |
| 173 |
// ExternalAddress is the host name to use for external (public internet) facing URLs (e.g. Swagger) |
| 174 |
// Will default to a value based on secure serving info and available ipv4 IPs. |
| 175 |
ExternalAddress string |
| 176 |
|
| 177 |
// TracerProvider can provide a tracer, which records spans for distributed tracing. |
| 178 |
TracerProvider tracing.TracerProvider |
| 179 |
|
| 180 |
//=========================================================================== |
| 181 |
// Fields you probably don't care about changing |
| 182 |
//=========================================================================== |
| 183 |
|
| 184 |
// BuildHandlerChainFunc allows you to build custom handler chains by decorating the apiHandler. |
| 185 |
BuildHandlerChainFunc func(apiHandler http.Handler, c *Config) (secure http.Handler) |
| 186 |
// NonLongRunningRequestWaitGroup allows you to wait for all chain |
| 187 |
// handlers associated with non long-running requests |
| 188 |
// to complete while the server is shuting down. |
| 189 |
NonLongRunningRequestWaitGroup *utilwaitgroup.SafeWaitGroup |
| 190 |
// WatchRequestWaitGroup allows us to wait for all chain |
| 191 |
// handlers associated with active watch requests to |
| 192 |
// complete while the server is shuting down. |
| 193 |
WatchRequestWaitGroup *utilwaitgroup.RateLimitedSafeWaitGroup |
| 194 |
// DiscoveryAddresses is used to build the IPs pass to discovery. If nil, the ExternalAddress is |
| 195 |
// always reported |
| 196 |
DiscoveryAddresses discovery.Addresses |
| 197 |
// The default set of healthz checks. There might be more added via AddHealthChecks dynamically. |
| 198 |
HealthzChecks []healthz.HealthChecker |
| 199 |
// The default set of livez checks. There might be more added via AddHealthChecks dynamically. |
| 200 |
LivezChecks []healthz.HealthChecker |
| 201 |
// The default set of readyz-only checks. There might be more added via AddReadyzChecks dynamically. |
| 202 |
ReadyzChecks []healthz.HealthChecker |
| 203 |
Flagz flagz.Reader |
| 204 |
// LegacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests |
| 205 |
// to InstallLegacyAPIGroup. New API servers don't generally have legacy groups at all. |
| 206 |
LegacyAPIGroupPrefixes sets.String |
| 207 |
// RequestInfoResolver is used to assign attributes (used by admission and authorization) based on a request URL. |
| 208 |
// Use-cases that are like kubelets may need to customize this. |
| 209 |
RequestInfoResolver apirequest.RequestInfoResolver |
| 210 |
// Serializer is required and provides the interface for serializing and converting objects to and from the wire |
| 211 |
// The default (api.Codecs) usually works fine. |
| 212 |
Serializer runtime.NegotiatedSerializer |
| 213 |
// OpenAPIConfig will be used in generating OpenAPI spec. This is nil by default. Use DefaultOpenAPIConfig for "working" defaults. |
| 214 |
OpenAPIConfig *openapicommon.Config |
| 215 |
// OpenAPIV3Config will be used in generating OpenAPI V3 spec. This is nil by default. Use DefaultOpenAPIV3Config for "working" defaults. |
| 216 |
OpenAPIV3Config *openapicommon.OpenAPIV3Config |
| 217 |
// SkipOpenAPIInstallation avoids installing the OpenAPI handler if set to true. |
| 218 |
SkipOpenAPIInstallation bool |
| 219 |
|
| 220 |
// ResourceTransformers are used to transform resources from and to etcd, e.g. encryption. |
| 221 |
ResourceTransformers storagevalue.ResourceTransformers |
| 222 |
// RESTOptionsGetter is used to construct RESTStorage types via the generic registry. |
| 223 |
RESTOptionsGetter genericregistry.RESTOptionsGetter |
| 224 |
|
| 225 |
// If specified, all requests except those which match the LongRunningFunc predicate will timeout |
| 226 |
// after this duration. |
| 227 |
RequestTimeout time.Duration |
| 228 |
// If specified, long running requests such as watch will be allocated a random timeout between this value, and |
| 229 |
// twice this value. Note that it is up to the request handlers to ignore or honor this timeout. In seconds. |
| 230 |
MinRequestTimeout int |
| 231 |
|
| 232 |
// StorageInitializationTimeout defines the maximum amount of time to wait for storage initialization |
| 233 |
// before declaring apiserver ready. |
| 234 |
StorageInitializationTimeout time.Duration |
| 235 |
|
| 236 |
// This represents the maximum amount of time it should take for apiserver to complete its startup |
| 237 |
// sequence and become healthy. From apiserver's start time to when this amount of time has |
| 238 |
// elapsed, /livez will assume that unfinished post-start hooks will complete successfully and |
| 239 |
// therefore return true. |
| 240 |
LivezGracePeriod time.Duration |
| 241 |
// ShutdownDelayDuration allows to block shutdown for some time, e.g. until endpoints pointing to this API server |
| 242 |
// have converged on all node. During this time, the API server keeps serving, /healthz will return 200, |
| 243 |
// but /readyz will return failure. |
| 244 |
ShutdownDelayDuration time.Duration |
| 245 |
|
| 246 |
// The limit on the total size increase all "copy" operations in a json |
| 247 |
// patch may cause. |
| 248 |
// This affects all places that applies json patch in the binary. |
| 249 |
JSONPatchMaxCopyBytes int64 |
| 250 |
// The limit on the request size that would be accepted and decoded in a write request |
| 251 |
// 0 means no limit. |
| 252 |
MaxRequestBodyBytes int64 |
| 253 |
// MaxRequestsInFlight is the maximum number of parallel non-long-running requests. Every further |
| 254 |
// request has to wait. Applies only to non-mutating requests. |
| 255 |
MaxRequestsInFlight int |
| 256 |
// MaxMutatingRequestsInFlight is the maximum number of parallel mutating requests. Every further |
| 257 |
// request has to wait. |
| 258 |
MaxMutatingRequestsInFlight int |
| 259 |
// Predicate which is true for paths of long-running http requests |
| 260 |
LongRunningFunc apirequest.LongRunningRequestCheck |
| 261 |
|
| 262 |
// GoawayChance is the probability that send a GOAWAY to HTTP/2 clients. When client received |
| 263 |
// GOAWAY, the in-flight requests will not be affected and new requests will use |
| 264 |
// a new TCP connection to triggering re-balancing to another server behind the load balance. |
| 265 |
// Default to 0, means never send GOAWAY. Max is 0.02 to prevent break the apiserver. |
| 266 |
GoawayChance float64 |
| 267 |
|
| 268 |
// MergedResourceConfig indicates which groupVersion enabled and its resources enabled/disabled. |
| 269 |
// This is composed of genericapiserver defaultAPIResourceConfig and those parsed from flags. |
| 270 |
// If not specify any in flags, then genericapiserver will only enable defaultAPIResourceConfig. |
| 271 |
MergedResourceConfig *serverstore.ResourceConfig |
| 272 |
|
| 273 |
// lifecycleSignals provides access to the various signals |
| 274 |
// that happen during lifecycle of the apiserver. |
| 275 |
// it's intentionally marked private as it should never be overridden. |
| 276 |
lifecycleSignals lifecycleSignals |
| 277 |
|
| 278 |
// StorageObjectCountTracker is used to keep track of the total number of objects |
| 279 |
// in the storage per resource, so we can estimate width of incoming requests. |
| 280 |
StorageObjectCountTracker flowcontrolrequest.StorageObjectCountTracker |
| 281 |
|
| 282 |
// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP |
| 283 |
// Server during the graceful termination of the apiserver. If true, we wait |
| 284 |
// for non longrunning requests in flight to be drained and then initiate a |
| 285 |
// shutdown of the HTTP Server. If false, we initiate a shutdown of the HTTP |
| 286 |
// Server as soon as ShutdownDelayDuration has elapsed. |
| 287 |
// If enabled, after ShutdownDelayDuration elapses, any incoming request is |
| 288 |
// rejected with a 429 status code and a 'Retry-After' response. |
| 289 |
ShutdownSendRetryAfter bool |
| 290 |
|
| 291 |
//=========================================================================== |
| 292 |
// values below here are targets for removal |
| 293 |
//=========================================================================== |
| 294 |
|
| 295 |
// PublicAddress is the IP address where members of the cluster (kubelet, |
| 296 |
// kube-proxy, services, etc.) can reach the GenericAPIServer. |
| 297 |
// If nil or 0.0.0.0, the host's default interface will be used. |
| 298 |
PublicAddress net.IP |
| 299 |
|
| 300 |
// EquivalentResourceRegistry provides information about resources equivalent to a given resource, |
| 301 |
// and the kind associated with a given resource. As resources are installed, they are registered here. |
| 302 |
EquivalentResourceRegistry runtime.EquivalentResourceRegistry |
| 303 |
|
| 304 |
// APIServerID is the ID of this API server |
| 305 |
APIServerID string |
| 306 |
|
| 307 |
// StorageVersionManager holds the storage versions of the API resources installed by this server. |
| 308 |
StorageVersionManager storageversion.Manager |
| 309 |
|
| 310 |
// AggregatedDiscoveryGroupManager serves /apis in an aggregated form. |
| 311 |
AggregatedDiscoveryGroupManager discoveryendpoint.ResourceManager |
| 312 |
|
| 313 |
// ShutdownWatchTerminationGracePeriod, if set to a positive value, |
| 314 |
// is the maximum duration the apiserver will wait for all active |
| 315 |
// watch request(s) to drain. |
| 316 |
// Once this grace period elapses, the apiserver will no longer |
| 317 |
// wait for any active watch request(s) in flight to drain, it will |
| 318 |
// proceed to the next step in the graceful server shutdown process. |
| 319 |
// If set to a positive value, the apiserver will keep track of the |
| 320 |
// number of active watch request(s) in flight and during shutdown |
| 321 |
// it will wait, at most, for the specified duration and allow these |
| 322 |
// active watch requests to drain with some rate limiting in effect. |
| 323 |
// The default is zero, which implies the apiserver will not keep |
| 324 |
// track of active watch request(s) in flight and will not wait |
| 325 |
// for them to drain, this maintains backward compatibility. |
| 326 |
// This grace period is orthogonal to other grace periods, and |
| 327 |
// it is not overridden by any other grace period. |
| 328 |
ShutdownWatchTerminationGracePeriod time.Duration |
| 329 |
} |
| 330 |
|
| 331 |
type RecommendedConfig struct { |
| 332 |
Config |
| 333 |
|
| 334 |
// SharedInformerFactory provides shared informers for Kubernetes resources. This value is set by |
| 335 |
// RecommendedOptions.CoreAPI.ApplyTo called by RecommendedOptions.ApplyTo. It uses an in-cluster client config |
| 336 |
// by default, or the kubeconfig given with kubeconfig command line flag. |
| 337 |
SharedInformerFactory informers.SharedInformerFactory |
| 338 |
|
| 339 |
// ClientConfig holds the kubernetes client configuration. |
| 340 |
// This value is set by RecommendedOptions.CoreAPI.ApplyTo called by RecommendedOptions.ApplyTo. |
| 341 |
// By default in-cluster client config is used. |
| 342 |
ClientConfig *restclient.Config |
| 343 |
} |
| 344 |
|
| 345 |
type SecureServingInfo struct { |
| 346 |
// Listener is the secure server network listener. |
| 347 |
Listener net.Listener |
| 348 |
|
| 349 |
// Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is |
| 350 |
// allowed to be in SNICerts. |
| 351 |
Cert dynamiccertificates.CertKeyContentProvider |
| 352 |
|
| 353 |
// SNICerts are the TLS certificates used for SNI. |
| 354 |
SNICerts []dynamiccertificates.SNICertKeyContentProvider |
| 355 |
|
| 356 |
// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates |
| 357 |
ClientCA dynamiccertificates.CAContentProvider |
| 358 |
|
| 359 |
// MinTLSVersion optionally overrides the minimum TLS version supported. |
| 360 |
// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). |
| 361 |
MinTLSVersion uint16 |
| 362 |
|
| 363 |
// CipherSuites optionally overrides the list of allowed cipher suites for the server. |
| 364 |
// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). |
| 365 |
CipherSuites []uint16 |
| 366 |
|
| 367 |
// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client. |
| 368 |
// A value of zero means to use the default provided by golang's HTTP/2 support. |
| 369 |
HTTP2MaxStreamsPerConnection int |
| 370 |
|
| 371 |
// DisableHTTP2 indicates that http2 should not be enabled. |
| 372 |
DisableHTTP2 bool |
| 373 |
} |
| 374 |
|
| 375 |
type AuthenticationInfo struct { |
| 376 |
// APIAudiences is a list of identifier that the API identifies as. This is |
| 377 |
// used by some authenticators to validate audience bound credentials. |
| 378 |
APIAudiences authenticator.Audiences |
| 379 |
// Authenticator determines which subject is making the request |
| 380 |
Authenticator authenticator.Request |
| 381 |
|
| 382 |
RequestHeaderConfig *authenticatorfactory.RequestHeaderConfig |
| 383 |
} |
| 384 |
|
| 385 |
type AuthorizationInfo struct { |
| 386 |
// Authorizer determines whether the subject is allowed to make the request based only |
| 387 |
// on the RequestURI |
| 388 |
Authorizer authorizer.Authorizer |
| 389 |
} |
| 390 |
|
| 391 |
func init() { |
| 392 |
utilruntime.Must(features.AddFeatureGates(utilfeature.DefaultMutableFeatureGate)) |
| 393 |
} |
| 394 |
|
| 395 |
// NewConfig returns a Config struct with the default values |
| 396 |
func NewConfig(codecs serializer.CodecFactory) *Config { |
| 397 |
defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz} |
| 398 |
var id string |
| 399 |
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) { |
| 400 |
hostname, err := hostnameFunc() |
| 401 |
if err != nil { |
| 402 |
klog.Fatalf("error getting hostname for apiserver identity: %v", err) |
| 403 |
} |
| 404 |
|
| 405 |
// Since the hash needs to be unique across each kube-apiserver and aggregated apiservers, |
| 406 |
// the hash used for the identity should include both the hostname and the identity value. |
| 407 |
// TODO: receive the identity value as a parameter once the apiserver identity lease controller |
| 408 |
// post start hook is moved to generic apiserver. |
| 409 |
b := cryptobyte.NewBuilder(nil) |
| 410 |
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { |
| 411 |
b.AddBytes([]byte(hostname)) |
| 412 |
}) |
| 413 |
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { |
| 414 |
b.AddBytes([]byte("kube-apiserver")) |
| 415 |
}) |
| 416 |
hashData, err := b.Bytes() |
| 417 |
if err != nil { |
| 418 |
klog.Fatalf("error building hash data for apiserver identity: %v", err) |
| 419 |
} |
| 420 |
|
| 421 |
hash := sha256.Sum256(hashData) |
| 422 |
id = "apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16])) |
| 423 |
} |
| 424 |
lifecycleSignals := newLifecycleSignals() |
| 425 |
|
| 426 |
return &Config{ |
| 427 |
Serializer: codecs, |
| 428 |
BuildHandlerChainFunc: DefaultBuildHandlerChain, |
| 429 |
NonLongRunningRequestWaitGroup: new(utilwaitgroup.SafeWaitGroup), |
| 430 |
WatchRequestWaitGroup: &utilwaitgroup.RateLimitedSafeWaitGroup{}, |
| 431 |
LegacyAPIGroupPrefixes: sets.NewString(DefaultLegacyAPIPrefix), |
| 432 |
DisabledPostStartHooks: sets.NewString(), |
| 433 |
PostStartHooks: map[string]PostStartHookConfigEntry{}, |
| 434 |
HealthzChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...), |
| 435 |
ReadyzChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...), |
| 436 |
LivezChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...), |
| 437 |
EnableIndex: true, |
| 438 |
EnableDiscovery: true, |
| 439 |
EnableProfiling: true, |
| 440 |
DebugSocketPath: "", |
| 441 |
EnableMetrics: true, |
| 442 |
MaxRequestsInFlight: 400, |
| 443 |
MaxMutatingRequestsInFlight: 200, |
| 444 |
RequestTimeout: time.Duration(60) * time.Second, |
| 445 |
MinRequestTimeout: 1800, |
| 446 |
StorageInitializationTimeout: time.Minute, |
| 447 |
LivezGracePeriod: time.Duration(0), |
| 448 |
ShutdownDelayDuration: time.Duration(0), |
| 449 |
// 1.5MB is the default client request size in bytes |
| 450 |
// the etcd server should accept. See |
| 451 |
// https://github.com/etcd-io/etcd/blob/release-3.4/embed/config.go#L56. |
| 452 |
// A request body might be encoded in json, and is converted to |
| 453 |
// proto when persisted in etcd, so we allow 2x as the largest size |
| 454 |
// increase the "copy" operations in a json patch may cause. |
| 455 |
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), |
| 456 |
// 1.5MB is the recommended client request size in byte |
| 457 |
// the etcd server should accept. See |
| 458 |
// https://github.com/etcd-io/etcd/blob/release-3.4/embed/config.go#L56. |
| 459 |
// A request body might be encoded in json, and is converted to |
| 460 |
// proto when persisted in etcd, so we allow 2x as the largest request |
| 461 |
// body size to be accepted and decoded in a write request. |
| 462 |
// If this constant is changed, DefaultMaxRequestSizeBytes in k8s.io/apiserver/pkg/cel/limits.go |
| 463 |
// should be changed to reflect the new value, if the two haven't |
| 464 |
// been wired together already somehow. |
| 465 |
MaxRequestBodyBytes: int64(3 * 1024 * 1024), |
| 466 |
|
| 467 |
// Default to treating watch as a long-running operation |
| 468 |
// Generic API servers have no inherent long-running subresources |
| 469 |
LongRunningFunc: genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()), |
| 470 |
lifecycleSignals: lifecycleSignals, |
| 471 |
StorageObjectCountTracker: flowcontrolrequest.NewStorageObjectCountTracker(), |
| 472 |
ShutdownWatchTerminationGracePeriod: time.Duration(0), |
| 473 |
|
| 474 |
APIServerID: id, |
| 475 |
StorageVersionManager: storageversion.NewDefaultManager(), |
| 476 |
TracerProvider: tracing.NewNoopTracerProvider(), |
| 477 |
} |
| 478 |
} |
| 479 |
|
| 480 |
// NewRecommendedConfig returns a RecommendedConfig struct with the default values |
| 481 |
func NewRecommendedConfig(codecs serializer.CodecFactory) *RecommendedConfig { |
| 482 |
return &RecommendedConfig{ |
| 483 |
Config: *NewConfig(codecs), |
| 484 |
} |
| 485 |
} |
| 486 |
|
| 487 |
// DefaultOpenAPIConfig provides the default OpenAPIConfig used to build the OpenAPI V2 spec |
| 488 |
func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.Config { |
| 489 |
return &openapicommon.Config{ |
| 490 |
ProtocolList: []string{"https"}, |
| 491 |
IgnorePrefixes: []string{}, |
| 492 |
Info: &spec.Info{ |
| 493 |
InfoProps: spec.InfoProps{ |
| 494 |
Title: "Generic API Server", |
| 495 |
}, |
| 496 |
}, |
| 497 |
DefaultResponse: &spec.Response{ |
| 498 |
ResponseProps: spec.ResponseProps{ |
| 499 |
Description: "Default Response.", |
| 500 |
}, |
| 501 |
}, |
| 502 |
GetOperationIDAndTags: apiopenapi.GetOperationIDAndTags, |
| 503 |
GetDefinitionName: defNamer.GetDefinitionName, |
| 504 |
GetDefinitions: getDefinitions, |
| 505 |
} |
| 506 |
} |
| 507 |
|
| 508 |
// DefaultOpenAPIV3Config provides the default OpenAPIV3Config used to build the OpenAPI V3 spec |
| 509 |
func DefaultOpenAPIV3Config(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.OpenAPIV3Config { |
| 510 |
defaultConfig := &openapicommon.OpenAPIV3Config{ |
| 511 |
IgnorePrefixes: []string{}, |
| 512 |
Info: &spec.Info{ |
| 513 |
InfoProps: spec.InfoProps{ |
| 514 |
Title: "Generic API Server", |
| 515 |
}, |
| 516 |
}, |
| 517 |
DefaultResponse: &spec3.Response{ |
| 518 |
ResponseProps: spec3.ResponseProps{ |
| 519 |
Description: "Default Response.", |
| 520 |
}, |
| 521 |
}, |
| 522 |
GetOperationIDAndTags: apiopenapi.GetOperationIDAndTags, |
| 523 |
GetDefinitionName: defNamer.GetDefinitionName, |
| 524 |
GetDefinitions: getDefinitions, |
| 525 |
} |
| 526 |
defaultConfig.Definitions = getDefinitions(func(name string) spec.Ref { |
| 527 |
defName, _ := defaultConfig.GetDefinitionName(name) |
| 528 |
return spec.MustCreateRef("#/components/schemas/" + openapicommon.EscapeJsonPointer(defName)) |
| 529 |
}) |
| 530 |
|
| 531 |
return defaultConfig |
| 532 |
} |
| 533 |
|
| 534 |
func (c *AuthenticationInfo) ApplyClientCert(clientCA dynamiccertificates.CAContentProvider, servingInfo *SecureServingInfo) error { |
| 535 |
if servingInfo == nil { |
| 536 |
return nil |
| 537 |
} |
| 538 |
if clientCA == nil { |
| 539 |
return nil |
| 540 |
} |
| 541 |
if servingInfo.ClientCA == nil { |
| 542 |
servingInfo.ClientCA = clientCA |
| 543 |
return nil |
| 544 |
} |
| 545 |
|
| 546 |
servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCA) |
| 547 |
return nil |
| 548 |
} |
| 549 |
|
| 550 |
type completedConfig struct { |
| 551 |
*Config |
| 552 |
|
| 553 |
//=========================================================================== |
| 554 |
// values below here are filled in during completion |
| 555 |
//=========================================================================== |
| 556 |
|
| 557 |
// SharedInformerFactory provides shared informers for resources |
| 558 |
SharedInformerFactory informers.SharedInformerFactory |
| 559 |
} |
| 560 |
|
| 561 |
type CompletedConfig struct { |
| 562 |
// Embed a private pointer that cannot be instantiated outside of this package. |
| 563 |
*completedConfig |
| 564 |
} |
| 565 |
|
| 566 |
// AddHealthChecks adds a health check to our config to be exposed by the health endpoints |
| 567 |
// of our configured apiserver. We should prefer this to adding healthChecks directly to |
| 568 |
// the config unless we explicitly want to add a healthcheck only to a specific health endpoint. |
| 569 |
func (c *Config) AddHealthChecks(healthChecks ...healthz.HealthChecker) { |
| 570 |
c.HealthzChecks = append(c.HealthzChecks, healthChecks...) |
| 571 |
c.LivezChecks = append(c.LivezChecks, healthChecks...) |
| 572 |
c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...) |
| 573 |
} |
| 574 |
|
| 575 |
// AddReadyzChecks adds a health check to our config to be exposed by the readyz endpoint |
| 576 |
// of our configured apiserver. |
| 577 |
func (c *Config) AddReadyzChecks(healthChecks ...healthz.HealthChecker) { |
| 578 |
c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...) |
| 579 |
} |
| 580 |
|
| 581 |
// AddPostStartHook allows you to add a PostStartHook that will later be added to the server itself in a New call. |
| 582 |
// Name conflicts will cause an error. |
| 583 |
func (c *Config) AddPostStartHook(name string, hook PostStartHookFunc) error { |
| 584 |
if len(name) == 0 { |
| 585 |
return fmt.Errorf("missing name") |
| 586 |
} |
| 587 |
if hook == nil { |
| 588 |
return fmt.Errorf("hook func may not be nil: %q", name) |
| 589 |
} |
| 590 |
if c.DisabledPostStartHooks.Has(name) { |
| 591 |
klog.V(1).Infof("skipping %q because it was explicitly disabled", name) |
| 592 |
return nil |
| 593 |
} |
| 594 |
|
| 595 |
if postStartHook, exists := c.PostStartHooks[name]; exists { |
| 596 |
// this is programmer error, but it can be hard to debug |
| 597 |
return fmt.Errorf("unable to add %q because it was already registered by: %s", name, postStartHook.originatingStack) |
| 598 |
} |
| 599 |
c.PostStartHooks[name] = PostStartHookConfigEntry{hook: hook, originatingStack: string(debug.Stack())} |
| 600 |
|
| 601 |
return nil |
| 602 |
} |
| 603 |
|
| 604 |
// AddPostStartHookOrDie allows you to add a PostStartHook, but dies on failure. |
| 605 |
func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) { |
| 606 |
if err := c.AddPostStartHook(name, hook); err != nil { |
| 607 |
klog.Fatalf("Error registering PostStartHook %q: %v", name, err) |
| 608 |
} |
| 609 |
} |
| 610 |
|
| 611 |
func completeOpenAPI(config *openapicommon.Config, version *version.Version) { |
| 612 |
if config == nil { |
| 613 |
return |
| 614 |
} |
| 615 |
if config.SecurityDefinitions != nil { |
| 616 |
// Setup OpenAPI security: all APIs will have the same authentication for now. |
| 617 |
config.DefaultSecurity = []map[string][]string{} |
| 618 |
keys := []string{} |
| 619 |
for k := range *config.SecurityDefinitions { |
| 620 |
keys = append(keys, k) |
| 621 |
} |
| 622 |
sort.Strings(keys) |
| 623 |
for _, k := range keys { |
| 624 |
config.DefaultSecurity = append(config.DefaultSecurity, map[string][]string{k: {}}) |
| 625 |
} |
| 626 |
if config.CommonResponses == nil { |
| 627 |
config.CommonResponses = map[int]spec.Response{} |
| 628 |
} |
| 629 |
if _, exists := config.CommonResponses[http.StatusUnauthorized]; !exists { |
| 630 |
config.CommonResponses[http.StatusUnauthorized] = spec.Response{ |
| 631 |
ResponseProps: spec.ResponseProps{ |
| 632 |
Description: "Unauthorized", |
| 633 |
}, |
| 634 |
} |
| 635 |
} |
| 636 |
} |
| 637 |
// make sure we populate info, and info.version, if not manually set |
| 638 |
if config.Info == nil { |
| 639 |
config.Info = &spec.Info{} |
| 640 |
} |
| 641 |
if config.Info.Version == "" { |
| 642 |
if version != nil { |
| 643 |
config.Info.Version = strings.Split(version.String(), "-")[0] |
| 644 |
} else { |
| 645 |
config.Info.Version = "unversioned" |
| 646 |
} |
| 647 |
} |
| 648 |
} |
| 649 |
|
| 650 |
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) { |
| 651 |
if config == nil { |
| 652 |
return |
| 653 |
} |
| 654 |
if config.SecuritySchemes != nil { |
| 655 |
// Setup OpenAPI security: all APIs will have the same authentication for now. |
| 656 |
config.DefaultSecurity = []map[string][]string{} |
| 657 |
keys := []string{} |
| 658 |
for k := range config.SecuritySchemes { |
| 659 |
keys = append(keys, k) |
| 660 |
} |
| 661 |
sort.Strings(keys) |
| 662 |
for _, k := range keys { |
| 663 |
config.DefaultSecurity = append(config.DefaultSecurity, map[string][]string{k: {}}) |
| 664 |
} |
| 665 |
if config.CommonResponses == nil { |
| 666 |
config.CommonResponses = map[int]*spec3.Response{} |
| 667 |
} |
| 668 |
if _, exists := config.CommonResponses[http.StatusUnauthorized]; !exists { |
| 669 |
config.CommonResponses[http.StatusUnauthorized] = &spec3.Response{ |
| 670 |
ResponseProps: spec3.ResponseProps{ |
| 671 |
Description: "Unauthorized", |
| 672 |
}, |
| 673 |
} |
| 674 |
} |
| 675 |
} |
| 676 |
// make sure we populate info, and info.version, if not manually set |
| 677 |
if config.Info == nil { |
| 678 |
config.Info = &spec.Info{} |
| 679 |
} |
| 680 |
if config.Info.Version == "" { |
| 681 |
if version != nil { |
| 682 |
config.Info.Version = strings.Split(version.String(), "-")[0] |
| 683 |
} else { |
| 684 |
config.Info.Version = "unversioned" |
| 685 |
} |
| 686 |
} |
| 687 |
} |
| 688 |
|
| 689 |
// DrainedNotify returns a lifecycle signal of genericapiserver already drained while shutting down. |
| 690 |
func (c *Config) DrainedNotify() <-chan struct{} { |
| 691 |
return c.lifecycleSignals.InFlightRequestsDrained.Signaled() |
| 692 |
} |
| 693 |
|
| 694 |
// ShutdownInitiated returns a lifecycle signal of apiserver shutdown having been initiated. |
| 695 |
func (c *Config) ShutdownInitiatedNotify() <-chan struct{} { |
| 696 |
return c.lifecycleSignals.ShutdownInitiated.Signaled() |
| 697 |
} |
| 698 |
|
| 699 |
// Complete fills in any fields not set that are required to have valid data and can be derived |
| 700 |
// from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. |
| 701 |
func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig { |
| 702 |
if c.FeatureGate == nil { |
| 703 |
c.FeatureGate = utilfeature.DefaultFeatureGate |
| 704 |
} |
| 705 |
if len(c.ExternalAddress) == 0 && c.PublicAddress != nil { |
| 706 |
c.ExternalAddress = c.PublicAddress.String() |
| 707 |
} |
| 708 |
|
| 709 |
// if there is no port, and we listen on one securely, use that one |
| 710 |
if _, _, err := net.SplitHostPort(c.ExternalAddress); err != nil { |
| 711 |
if c.SecureServing == nil { |
| 712 |
klog.Fatalf("cannot derive external address port without listening on a secure port.") |
| 713 |
} |
| 714 |
_, port, err := c.SecureServing.HostPort() |
| 715 |
if err != nil { |
| 716 |
klog.Fatalf("cannot derive external address from the secure port: %v", err) |
| 717 |
} |
| 718 |
c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port)) |
| 719 |
} |
| 720 |
completeOpenAPI(c.OpenAPIConfig, c.EffectiveVersion.EmulationVersion()) |
| 721 |
completeOpenAPIV3(c.OpenAPIV3Config, c.EffectiveVersion.EmulationVersion()) |
| 722 |
|
| 723 |
if c.DiscoveryAddresses == nil { |
| 724 |
c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress} |
| 725 |
} |
| 726 |
|
| 727 |
AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) |
| 728 |
|
| 729 |
if c.RequestInfoResolver == nil { |
| 730 |
c.RequestInfoResolver = NewRequestInfoResolver(c) |
| 731 |
} |
| 732 |
|
| 733 |
if c.EquivalentResourceRegistry == nil { |
| 734 |
if c.RESTOptionsGetter == nil { |
| 735 |
c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistry() |
| 736 |
} else { |
| 737 |
c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string { |
| 738 |
// use the storage prefix as the key if possible |
| 739 |
if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil { |
| 740 |
return opts.ResourcePrefix |
| 741 |
} |
| 742 |
// otherwise return "" to use the default key (parent GV name) |
| 743 |
return "" |
| 744 |
}) |
| 745 |
} |
| 746 |
} |
| 747 |
|
| 748 |
return CompletedConfig{&completedConfig{c, informers}} |
| 749 |
} |
| 750 |
|
| 751 |
// Complete fills in any fields not set that are required to have valid data and can be derived |
| 752 |
// from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. |
| 753 |
func (c *RecommendedConfig) Complete() CompletedConfig { |
| 754 |
return c.Config.Complete(c.SharedInformerFactory) |
| 755 |
} |
| 756 |
|
| 757 |
var defaultAllowedMediaTypes = []string{ |
| 758 |
runtime.ContentTypeJSON, |
| 759 |
runtime.ContentTypeYAML, |
| 760 |
runtime.ContentTypeProtobuf, |
| 761 |
} |
| 762 |
|
| 763 |
// New creates a new server which logically combines the handling chain with the passed server. |
| 764 |
// name is used to differentiate for logging. The handler chain in particular can be difficult as it starts delegating. |
| 765 |
// delegationTarget may not be nil. |
| 766 |
func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) { |
| 767 |
if c.Serializer == nil { |
| 768 |
return nil, fmt.Errorf("Genericapiserver.New() called with config.Serializer == nil") |
| 769 |
} |
| 770 |
allowedMediaTypes := defaultAllowedMediaTypes |
| 771 |
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.CBORServingAndStorage) { |
| 772 |
allowedMediaTypes = append(allowedMediaTypes, runtime.ContentTypeCBOR) |
| 773 |
} |
| 774 |
for _, info := range c.Serializer.SupportedMediaTypes() { |
| 775 |
var ok bool |
| 776 |
for _, mt := range allowedMediaTypes { |
| 777 |
if info.MediaType == mt { |
| 778 |
ok = true |
| 779 |
break |
| 780 |
} |
| 781 |
} |
| 782 |
if !ok { |
| 783 |
return nil, fmt.Errorf("refusing to create new apiserver %q with support for media type %q (allowed media types are: %s)", name, info.MediaType, strings.Join(allowedMediaTypes, ", ")) |
| 784 |
} |
| 785 |
} |
| 786 |
if c.LoopbackClientConfig == nil { |
| 787 |
return nil, fmt.Errorf("Genericapiserver.New() called with config.LoopbackClientConfig == nil") |
| 788 |
} |
| 789 |
if c.EquivalentResourceRegistry == nil { |
| 790 |
return nil, fmt.Errorf("Genericapiserver.New() called with config.EquivalentResourceRegistry == nil") |
| 791 |
} |
| 792 |
|
| 793 |
handlerChainBuilder := func(handler http.Handler) http.Handler { |
| 794 |
return c.BuildHandlerChainFunc(handler, c.Config) |
| 795 |
} |
| 796 |
|
| 797 |
var debugSocket *routes.DebugSocket |
| 798 |
if c.DebugSocketPath != "" { |
| 799 |
debugSocket = routes.NewDebugSocket(c.DebugSocketPath) |
| 800 |
} |
| 801 |
|
| 802 |
apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler()) |
| 803 |
|
| 804 |
s := &GenericAPIServer{ |
| 805 |
discoveryAddresses: c.DiscoveryAddresses, |
| 806 |
LoopbackClientConfig: c.LoopbackClientConfig, |
| 807 |
legacyAPIGroupPrefixes: c.LegacyAPIGroupPrefixes, |
| 808 |
admissionControl: c.AdmissionControl, |
| 809 |
Serializer: c.Serializer, |
| 810 |
AuditBackend: c.AuditBackend, |
| 811 |
Authorizer: c.Authorization.Authorizer, |
| 812 |
delegationTarget: delegationTarget, |
| 813 |
EquivalentResourceRegistry: c.EquivalentResourceRegistry, |
| 814 |
NonLongRunningRequestWaitGroup: c.NonLongRunningRequestWaitGroup, |
| 815 |
WatchRequestWaitGroup: c.WatchRequestWaitGroup, |
| 816 |
Handler: apiServerHandler, |
| 817 |
UnprotectedDebugSocket: debugSocket, |
| 818 |
|
| 819 |
listedPathProvider: apiServerHandler, |
| 820 |
|
| 821 |
minRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second, |
| 822 |
ShutdownTimeout: c.RequestTimeout, |
| 823 |
ShutdownDelayDuration: c.ShutdownDelayDuration, |
| 824 |
ShutdownWatchTerminationGracePeriod: c.ShutdownWatchTerminationGracePeriod, |
| 825 |
SecureServingInfo: c.SecureServing, |
| 826 |
ExternalAddress: c.ExternalAddress, |
| 827 |
|
| 828 |
openAPIConfig: c.OpenAPIConfig, |
| 829 |
openAPIV3Config: c.OpenAPIV3Config, |
| 830 |
skipOpenAPIInstallation: c.SkipOpenAPIInstallation, |
| 831 |
|
| 832 |
postStartHooks: map[string]postStartHookEntry{}, |
| 833 |
preShutdownHooks: map[string]preShutdownHookEntry{}, |
| 834 |
disabledPostStartHooks: c.DisabledPostStartHooks, |
| 835 |
|
| 836 |
healthzRegistry: healthCheckRegistry{path: "/healthz", checks: c.HealthzChecks}, |
| 837 |
livezRegistry: healthCheckRegistry{path: "/livez", checks: c.LivezChecks, clock: clock.RealClock{}}, |
| 838 |
readyzRegistry: healthCheckRegistry{path: "/readyz", checks: c.ReadyzChecks}, |
| 839 |
livezGracePeriod: c.LivezGracePeriod, |
| 840 |
|
| 841 |
DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer), |
| 842 |
|
| 843 |
maxRequestBodyBytes: c.MaxRequestBodyBytes, |
| 844 |
|
| 845 |
lifecycleSignals: c.lifecycleSignals, |
| 846 |
ShutdownSendRetryAfter: c.ShutdownSendRetryAfter, |
| 847 |
|
| 848 |
APIServerID: c.APIServerID, |
| 849 |
StorageReadinessHook: NewStorageReadinessHook(c.StorageInitializationTimeout), |
| 850 |
StorageVersionManager: c.StorageVersionManager, |
| 851 |
|
| 852 |
EffectiveVersion: c.EffectiveVersion, |
| 853 |
EmulationForwardCompatible: c.EmulationForwardCompatible, |
| 854 |
RuntimeConfigEmulationForwardCompatible: c.RuntimeConfigEmulationForwardCompatible, |
| 855 |
FeatureGate: c.FeatureGate, |
| 856 |
|
| 857 |
muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, |
| 858 |
} |
| 859 |
|
| 860 |
manager := c.AggregatedDiscoveryGroupManager |
| 861 |
if manager == nil { |
| 862 |
manager = discoveryendpoint.NewResourceManager("apis") |
| 863 |
} |
| 864 |
s.AggregatedDiscoveryGroupManager = manager |
| 865 |
s.AggregatedLegacyDiscoveryGroupManager = discoveryendpoint.NewResourceManager("api") |
| 866 |
for { |
| 867 |
if c.JSONPatchMaxCopyBytes <= 0 { |
| 868 |
break |
| 869 |
} |
| 870 |
existing := atomic.LoadInt64(&jsonpatch.AccumulatedCopySizeLimit) |
| 871 |
if existing > 0 && existing < c.JSONPatchMaxCopyBytes { |
| 872 |
break |
| 873 |
} |
| 874 |
if atomic.CompareAndSwapInt64(&jsonpatch.AccumulatedCopySizeLimit, existing, c.JSONPatchMaxCopyBytes) { |
| 875 |
break |
| 876 |
} |
| 877 |
} |
| 878 |
|
| 879 |
// first add poststarthooks from delegated targets |
| 880 |
for k, v := range delegationTarget.PostStartHooks() { |
| 881 |
s.postStartHooks[k] = v |
| 882 |
} |
| 883 |
|
| 884 |
for k, v := range delegationTarget.PreShutdownHooks() { |
| 885 |
s.preShutdownHooks[k] = v |
| 886 |
} |
| 887 |
|
| 888 |
// add poststarthooks that were preconfigured. Using the add method will give us an error if the same name has already been registered. |
| 889 |
for name, preconfiguredPostStartHook := range c.PostStartHooks { |
| 890 |
if err := s.AddPostStartHook(name, preconfiguredPostStartHook.hook); err != nil { |
| 891 |
return nil, err |
| 892 |
} |
| 893 |
} |
| 894 |
|
| 895 |
// register mux signals from the delegated server |
| 896 |
for k, v := range delegationTarget.MuxAndDiscoveryCompleteSignals() { |
| 897 |
if err := s.RegisterMuxAndDiscoveryCompleteSignal(k, v); err != nil { |
| 898 |
return nil, err |
| 899 |
} |
| 900 |
} |
| 901 |
|
| 902 |
genericApiServerHookName := "generic-apiserver-start-informers" |
| 903 |
if c.SharedInformerFactory != nil { |
| 904 |
if !s.isPostStartHookRegistered(genericApiServerHookName) { |
| 905 |
err := s.AddPostStartHook(genericApiServerHookName, func(hookContext PostStartHookContext) error { |
| 906 |
c.SharedInformerFactory.Start(hookContext.Done()) |
| 907 |
return nil |
| 908 |
}) |
| 909 |
if err != nil { |
| 910 |
return nil, err |
| 911 |
} |
| 912 |
} |
| 913 |
// TODO: Once we get rid of /healthz consider changing this to post-start-hook. |
| 914 |
err := s.AddReadyzChecks(healthz.NewInformerSyncHealthz(c.SharedInformerFactory)) |
| 915 |
if err != nil { |
| 916 |
return nil, err |
| 917 |
} |
| 918 |
} |
| 919 |
|
| 920 |
const priorityAndFairnessConfigConsumerHookName = "priority-and-fairness-config-consumer" |
| 921 |
if s.isPostStartHookRegistered(priorityAndFairnessConfigConsumerHookName) { |
| 922 |
} else if c.FlowControl != nil { |
| 923 |
err := s.AddPostStartHook(priorityAndFairnessConfigConsumerHookName, func(hookContext PostStartHookContext) error { |
| 924 |
go c.FlowControl.Run(hookContext.Done()) |
| 925 |
return nil |
| 926 |
}) |
| 927 |
if err != nil { |
| 928 |
return nil, err |
| 929 |
} |
| 930 |
// TODO(yue9944882): plumb pre-shutdown-hook for request-management system? |
| 931 |
} else { |
| 932 |
klog.V(3).Infof("Not requested to run hook %s", priorityAndFairnessConfigConsumerHookName) |
| 933 |
} |
| 934 |
|
| 935 |
// Add PostStartHooks for maintaining the watermarks for the Priority-and-Fairness and the Max-in-Flight filters. |
| 936 |
if c.FlowControl != nil { |
| 937 |
const priorityAndFairnessFilterHookName = "priority-and-fairness-filter" |
| 938 |
if !s.isPostStartHookRegistered(priorityAndFairnessFilterHookName) { |
| 939 |
err := s.AddPostStartHook(priorityAndFairnessFilterHookName, func(hookContext PostStartHookContext) error { |
| 940 |
genericfilters.StartPriorityAndFairnessWatermarkMaintenance(hookContext.Done()) |
| 941 |
return nil |
| 942 |
}) |
| 943 |
if err != nil { |
| 944 |
return nil, err |
| 945 |
} |
| 946 |
} |
| 947 |
} else { |
| 948 |
const maxInFlightFilterHookName = "max-in-flight-filter" |
| 949 |
if !s.isPostStartHookRegistered(maxInFlightFilterHookName) { |
| 950 |
err := s.AddPostStartHook(maxInFlightFilterHookName, func(hookContext PostStartHookContext) error { |
| 951 |
genericfilters.StartMaxInFlightWatermarkMaintenance(hookContext.Done()) |
| 952 |
return nil |
| 953 |
}) |
| 954 |
if err != nil { |
| 955 |
return nil, err |
| 956 |
} |
| 957 |
} |
| 958 |
} |
| 959 |
|
| 960 |
// Add PostStartHook for maintenaing the object count tracker. |
| 961 |
if c.StorageObjectCountTracker != nil { |
| 962 |
const storageObjectCountTrackerHookName = "storage-object-count-tracker-hook" |
| 963 |
if !s.isPostStartHookRegistered(storageObjectCountTrackerHookName) { |
| 964 |
if err := s.AddPostStartHook(storageObjectCountTrackerHookName, func(hookContext PostStartHookContext) error { |
| 965 |
go c.StorageObjectCountTracker.RunUntil(hookContext.Done()) |
| 966 |
return nil |
| 967 |
}); err != nil { |
| 968 |
return nil, err |
| 969 |
} |
| 970 |
} |
| 971 |
} |
| 972 |
|
| 973 |
for _, delegateCheck := range delegationTarget.HealthzChecks() { |
| 974 |
skip := false |
| 975 |
for _, existingCheck := range c.HealthzChecks { |
| 976 |
if existingCheck.Name() == delegateCheck.Name() { |
| 977 |
skip = true |
| 978 |
break |
| 979 |
} |
| 980 |
} |
| 981 |
if skip { |
| 982 |
continue |
| 983 |
} |
| 984 |
s.AddHealthChecks(delegateCheck) |
| 985 |
} |
| 986 |
s.RegisterDestroyFunc(func() { |
| 987 |
if err := c.Config.TracerProvider.Shutdown(context.Background()); err != nil { |
| 988 |
klog.Errorf("failed to shut down tracer provider: %v", err) |
| 989 |
} |
| 990 |
}) |
| 991 |
|
| 992 |
s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget} |
| 993 |
|
| 994 |
installAPI(name, s, c.Config) |
| 995 |
|
| 996 |
// use the UnprotectedHandler from the delegation target to ensure that we don't attempt to double authenticator, authorize, |
| 997 |
// or some other part of the filter chain in delegation cases. |
| 998 |
if delegationTarget.UnprotectedHandler() == nil && c.EnableIndex { |
| 999 |
s.Handler.NonGoRestfulMux.NotFoundHandler(routes.IndexLister{ |
| 1000 |
StatusCode: http.StatusNotFound, |
| 1001 |
PathProvider: s.listedPathProvider, |
| 1002 |
}) |
| 1003 |
} |
| 1004 |
|
| 1005 |
return s, nil |
| 1006 |
} |
| 1007 |
|
| 1008 |
func BuildHandlerChainWithStorageVersionPrecondition(apiHandler http.Handler, c *Config) http.Handler { |
| 1009 |
// WithStorageVersionPrecondition needs the WithRequestInfo to run first |
| 1010 |
handler := genericapifilters.WithStorageVersionPrecondition(apiHandler, c.StorageVersionManager, c.Serializer) |
| 1011 |
return DefaultBuildHandlerChain(handler, c) |
| 1012 |
} |
| 1013 |
|
| 1014 |
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler { |
| 1015 |
handler := apiHandler |
| 1016 |
|
| 1017 |
handler = filterlatency.TrackCompleted(handler) |
| 1018 |
handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer) |
| 1019 |
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authorization") |
| 1020 |
|
| 1021 |
if c.FlowControl != nil { |
| 1022 |
workEstimatorCfg := flowcontrolrequest.DefaultWorkEstimatorConfig() |
| 1023 |
requestWorkEstimator := flowcontrolrequest.NewWorkEstimator( |
| 1024 |
c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg, c.FlowControl.GetMaxSeats) |
| 1025 |
handler = filterlatency.TrackCompleted(handler) |
| 1026 |
handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator, c.RequestTimeout/4) |
| 1027 |
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "priorityandfairness") |
| 1028 |
} else { |
| 1029 |
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc) |
| 1030 |
} |
| 1031 |
|
| 1032 |
handler = filterlatency.TrackCompleted(handler) |
| 1033 |
handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer) |
| 1034 |
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "impersonation") |
| 1035 |
|
| 1036 |
handler = filterlatency.TrackCompleted(handler) |
| 1037 |
handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator, c.LongRunningFunc) |
| 1038 |
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "audit") |
| 1039 |
|
| 1040 |
failedHandler := genericapifilters.Unauthorized(c.Serializer) |
| 1041 |
failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyRuleEvaluator) |
| 1042 |
|
| 1043 |
// WithTracing comes after authentication so we can allow authenticated |
| 1044 |
// clients to influence sampling. |
| 1045 |
if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) { |
| 1046 |
handler = genericapifilters.WithTracing(handler, c.TracerProvider) |
| 1047 |
} |
| 1048 |
failedHandler = filterlatency.TrackCompleted(failedHandler) |
| 1049 |
handler = filterlatency.TrackCompleted(handler) |
| 1050 |
handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences, c.Authentication.RequestHeaderConfig) |
| 1051 |
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authentication") |
| 1052 |
|
| 1053 |
handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true") |
| 1054 |
|
| 1055 |
// WithWarningRecorder must be wrapped by the timeout handler |
| 1056 |
// to make the addition of warning headers threadsafe |
| 1057 |
handler = genericapifilters.WithWarningRecorder(handler) |
| 1058 |
|
| 1059 |
// WithTimeoutForNonLongRunningRequests will call the rest of the request handling in a go-routine with the |
| 1060 |
// context with deadline. The go-routine can keep running, while the timeout logic will return a timeout to the client. |
| 1061 |
handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc) |
| 1062 |
|
| 1063 |
handler = genericapifilters.WithRequestDeadline(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator, |
| 1064 |
c.LongRunningFunc, c.Serializer, c.RequestTimeout) |
| 1065 |
handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.NonLongRunningRequestWaitGroup) |
| 1066 |
if c.ShutdownWatchTerminationGracePeriod > 0 { |
| 1067 |
handler = genericfilters.WithWatchTerminationDuringShutdown(handler, c.lifecycleSignals, c.WatchRequestWaitGroup) |
| 1068 |
} |
| 1069 |
if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 { |
| 1070 |
handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance) |
| 1071 |
} |
| 1072 |
handler = genericapifilters.WithCacheControl(handler) |
| 1073 |
handler = genericfilters.WithHSTS(handler, c.HSTSDirectives) |
| 1074 |
if c.ShutdownSendRetryAfter { |
| 1075 |
handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled()) |
| 1076 |
} |
| 1077 |
handler = genericfilters.WithHTTPLogging(handler) |
| 1078 |
handler = genericapifilters.WithLatencyTrackers(handler) |
| 1079 |
// WithRoutine will execute future handlers in a separate goroutine and serving |
| 1080 |
// handler in current goroutine to minimize the stack memory usage. It must be |
| 1081 |
// after WithPanicRecover() to be protected from panics. |
| 1082 |
if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) { |
| 1083 |
handler = routine.WithRoutine(handler, c.LongRunningFunc) |
| 1084 |
} |
| 1085 |
handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver) |
| 1086 |
handler = genericapifilters.WithRequestReceivedTimestamp(handler) |
| 1087 |
handler = genericapifilters.WithMuxAndDiscoveryComplete(handler, c.lifecycleSignals.MuxAndDiscoveryComplete.Signaled()) |
| 1088 |
handler = genericfilters.WithPanicRecovery(handler, c.RequestInfoResolver) |
| 1089 |
handler = genericapifilters.WithAuditInit(handler) |
| 1090 |
return handler |
| 1091 |
} |
| 1092 |
|
| 1093 |
func installAPI(name string, s *GenericAPIServer, c *Config) { |
| 1094 |
if c.EnableIndex { |
| 1095 |
routes.Index{}.Install(s.listedPathProvider, s.Handler.NonGoRestfulMux) |
| 1096 |
} |
| 1097 |
if c.EnableProfiling { |
| 1098 |
routes.Profiling{}.Install(s.Handler.NonGoRestfulMux) |
| 1099 |
if c.EnableContentionProfiling { |
| 1100 |
goruntime.SetBlockProfileRate(1) |
| 1101 |
} |
| 1102 |
// so far, only logging related endpoints are considered valid to add for these debug flags. |
| 1103 |
routes.DebugFlags{}.Install(s.Handler.NonGoRestfulMux, "v", routes.StringFlagPutHandler(logs.GlogSetter)) |
| 1104 |
} |
| 1105 |
if s.UnprotectedDebugSocket != nil { |
| 1106 |
s.UnprotectedDebugSocket.InstallProfiling() |
| 1107 |
s.UnprotectedDebugSocket.InstallDebugFlag("v", routes.StringFlagPutHandler(logs.GlogSetter)) |
| 1108 |
if c.EnableContentionProfiling { |
| 1109 |
goruntime.SetBlockProfileRate(1) |
| 1110 |
} |
| 1111 |
} |
| 1112 |
|
| 1113 |
if c.EnableMetrics { |
| 1114 |
if c.EnableProfiling { |
| 1115 |
routes.MetricsWithReset{}.Install(s.Handler.NonGoRestfulMux) |
| 1116 |
slis.SLIMetricsWithReset{}.Install(s.Handler.NonGoRestfulMux) |
| 1117 |
} else { |
| 1118 |
routes.DefaultMetrics{}.Install(s.Handler.NonGoRestfulMux) |
| 1119 |
slis.SLIMetrics{}.Install(s.Handler.NonGoRestfulMux) |
| 1120 |
} |
| 1121 |
} |
| 1122 |
|
| 1123 |
routes.Version{Version: c.EffectiveVersion.Info()}.Install(s.Handler.GoRestfulContainer) |
| 1124 |
|
| 1125 |
if c.EnableDiscovery { |
| 1126 |
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager) |
| 1127 |
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{})) |
| 1128 |
} |
| 1129 |
if c.FlowControl != nil { |
| 1130 |
c.FlowControl.Install(s.Handler.NonGoRestfulMux) |
| 1131 |
} |
| 1132 |
} |
| 1133 |
|
| 1134 |
func NewRequestInfoResolver(c *Config) *apirequest.RequestInfoFactory { |
| 1135 |
apiPrefixes := sets.NewString(strings.Trim(APIGroupPrefix, "/")) // all possible API prefixes |
| 1136 |
legacyAPIPrefixes := sets.String{} // APIPrefixes that won't have groups (legacy) |
| 1137 |
for legacyAPIPrefix := range c.LegacyAPIGroupPrefixes { |
| 1138 |
apiPrefixes.Insert(strings.Trim(legacyAPIPrefix, "/")) |
| 1139 |
legacyAPIPrefixes.Insert(strings.Trim(legacyAPIPrefix, "/")) |
| 1140 |
} |
| 1141 |
|
| 1142 |
return &apirequest.RequestInfoFactory{ |
| 1143 |
APIPrefixes: apiPrefixes, |
| 1144 |
GrouplessAPIPrefixes: legacyAPIPrefixes, |
| 1145 |
} |
| 1146 |
} |
| 1147 |
|
| 1148 |
func (s *SecureServingInfo) HostPort() (string, int, error) { |
| 1149 |
if s == nil || s.Listener == nil { |
| 1150 |
return "", 0, fmt.Errorf("no listener found") |
| 1151 |
} |
| 1152 |
addr := s.Listener.Addr().String() |
| 1153 |
host, portStr, err := net.SplitHostPort(addr) |
| 1154 |
if err != nil { |
| 1155 |
return "", 0, fmt.Errorf("failed to get port from listener address %q: %v", addr, err) |
| 1156 |
} |
| 1157 |
port, err := utilsnet.ParsePort(portStr, true) |
| 1158 |
if err != nil { |
| 1159 |
return "", 0, fmt.Errorf("invalid non-numeric port %q", portStr) |
| 1160 |
} |
| 1161 |
return host, port, nil |
| 1162 |
} |
| 1163 |
|
| 1164 |
// AuthorizeClientBearerToken wraps the authenticator and authorizer in loopback authentication logic |
| 1165 |
// if the loopback client config is specified AND it has a bearer token. Note that if either authn or |
| 1166 |
// authz is nil, this function won't add a token authenticator or authorizer. |
| 1167 |
func AuthorizeClientBearerToken(loopback *restclient.Config, authn *AuthenticationInfo, authz *AuthorizationInfo) { |
| 1168 |
if loopback == nil || len(loopback.BearerToken) == 0 { |
| 1169 |
return |
| 1170 |
} |
| 1171 |
if authn == nil || authz == nil { |
| 1172 |
// prevent nil pointer panic |
| 1173 |
return |
| 1174 |
} |
| 1175 |
if authn.Authenticator == nil || authz.Authorizer == nil { |
| 1176 |
// authenticator or authorizer might be nil if we want to bypass authz/authn |
| 1177 |
// and we also do nothing in this case. |
| 1178 |
return |
| 1179 |
} |
| 1180 |
|
| 1181 |
privilegedLoopbackToken := loopback.BearerToken |
| 1182 |
var uid = uuid.New().String() |
| 1183 |
tokens := make(map[string]*user.DefaultInfo) |
| 1184 |
tokens[privilegedLoopbackToken] = &user.DefaultInfo{ |
| 1185 |
Name: user.APIServerUser, |
| 1186 |
UID: uid, |
| 1187 |
Groups: []string{user.AllAuthenticated, user.SystemPrivilegedGroup}, |
| 1188 |
} |
| 1189 |
|
| 1190 |
tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens, authn.APIAudiences) |
| 1191 |
authn.Authenticator = authenticatorunion.New(tokenAuthenticator, authn.Authenticator) |
| 1192 |
} |
| 1193 |
|
| 1194 |
// For testing purpose only |
| 1195 |
func SetHostnameFuncForTests(name string) { |
| 1196 |
hostnameFunc = func() (host string, err error) { |
| 1197 |
host = name |
| 1198 |
err = nil |
| 1199 |
return |
| 1200 |
} |
| 1201 |
} |