Debian Codegraph @

pkg/server/options/server_run_options.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package options
18
19 import (
20 "fmt"
21 "net"
22 "regexp"
23 "strings"
24 "time"
25
26 "k8s.io/apimachinery/pkg/runtime/serializer"
27 "k8s.io/apimachinery/pkg/util/errors"
28 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
29 "k8s.io/apiserver/pkg/server"
30 "k8s.io/apiserver/pkg/util/compatibility"
31 utilfeature "k8s.io/apiserver/pkg/util/feature"
32 basecompatibility "k8s.io/component-base/compatibility"
33
34 "github.com/spf13/pflag"
35 )
36
37 const (
38 corsAllowedOriginsHelpText = "List of allowed origins for CORS, comma separated. " +
39 "An allowed origin can be a regular expression to support subdomain matching. " +
40 "If this list is empty CORS will not be enabled. " +
41 "Please ensure each expression matches the entire hostname by anchoring " +
42 "to the start with '^' or including the '//' prefix, and by anchoring to the " +
43 "end with '$' or including the ':' port separator suffix. " +
44 "Examples of valid expressions are '//example\\.com(:|$)' and '^https://example\\.com(:|$)'"
45 )
46
47 // ServerRunOptions contains the options while running a generic api server.
48 type ServerRunOptions struct {
49 AdvertiseAddress net.IP
50
51 CorsAllowedOriginList []string
52 HSTSDirectives []string
53 ExternalHost string
54 MaxRequestsInFlight int
55 MaxMutatingRequestsInFlight int
56 RequestTimeout time.Duration
57 GoawayChance float64
58 LivezGracePeriod time.Duration
59 MinRequestTimeout int
60 StorageInitializationTimeout time.Duration
61 ShutdownDelayDuration time.Duration
62 // We intentionally did not add a flag for this option. Users of the
63 // apiserver library can wire it to a flag.
64 JSONPatchMaxCopyBytes int64
65 // The limit on the request body size that would be accepted and
66 // decoded in a write request. 0 means no limit.
67 // We intentionally did not add a flag for this option. Users of the
68 // apiserver library can wire it to a flag.
69 MaxRequestBodyBytes int64
70
71 // ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
72 // Server during the graceful termination of the apiserver. If true, we wait
73 // for non longrunning requests in flight to be drained and then initiate a
74 // shutdown of the HTTP Server. If false, we initiate a shutdown of the HTTP
75 // Server as soon as ShutdownDelayDuration has elapsed.
76 // If enabled, after ShutdownDelayDuration elapses, any incoming request is
77 // rejected with a 429 status code and a 'Retry-After' response.
78 ShutdownSendRetryAfter bool
79
80 // ShutdownWatchTerminationGracePeriod, if set to a positive value,
81 // is the maximum duration the apiserver will wait for all active
82 // watch request(s) to drain.
83 // Once this grace period elapses, the apiserver will no longer
84 // wait for any active watch request(s) in flight to drain, it will
85 // proceed to the next step in the graceful server shutdown process.
86 // If set to a positive value, the apiserver will keep track of the
87 // number of active watch request(s) in flight and during shutdown
88 // it will wait, at most, for the specified duration and allow these
89 // active watch requests to drain with some rate limiting in effect.
90 // The default is zero, which implies the apiserver will not keep
91 // track of active watch request(s) in flight and will not wait
92 // for them to drain, this maintains backward compatibility.
93 // This grace period is orthogonal to other grace periods, and
94 // it is not overridden by any other grace period.
95 ShutdownWatchTerminationGracePeriod time.Duration
96
97 // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
98 ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry
99 // ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
100 ComponentName string
101 // EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
102 // have higher priority than APIs of the same group resource enabled at the emulation version.
103 // If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
104 // This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
105 // Not applicable to alpha APIs.
106 EmulationForwardCompatible bool
107 // RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
108 // If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.
109 // If false, error would be thrown if any GroupVersion or GroupVersionResource explicitly enabled in the --runtime-config flag is introduced after the emulation version.
110 RuntimeConfigEmulationForwardCompatible bool
111 }
112
113 func NewServerRunOptions() *ServerRunOptions {
114 if compatibility.DefaultComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent) == nil {
115 featureGate := utilfeature.DefaultMutableFeatureGate
116 effectiveVersion := compatibility.DefaultBuildEffectiveVersion()
117 utilruntime.Must(compatibility.DefaultComponentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate))
118 }
119
120 return NewServerRunOptionsForComponent(basecompatibility.DefaultKubeComponent, compatibility.DefaultComponentGlobalsRegistry)
121 }
122
123 func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry) *ServerRunOptions {
124 defaults := server.NewConfig(serializer.CodecFactory{})
125 return &ServerRunOptions{
126 MaxRequestsInFlight: defaults.MaxRequestsInFlight,
127 MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
128 RequestTimeout: defaults.RequestTimeout,
129 LivezGracePeriod: defaults.LivezGracePeriod,
130 MinRequestTimeout: defaults.MinRequestTimeout,
131 StorageInitializationTimeout: defaults.StorageInitializationTimeout,
132 ShutdownDelayDuration: defaults.ShutdownDelayDuration,
133 ShutdownWatchTerminationGracePeriod: defaults.ShutdownWatchTerminationGracePeriod,
134 JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
135 MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
136 ShutdownSendRetryAfter: false,
137 ComponentName: componentName,
138 ComponentGlobalsRegistry: componentGlobalsRegistry,
139 }
140 }
141
142 // ApplyTo applies the run options to the method receiver and returns self
143 func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
144 if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil {
145 return err
146 }
147 c.CorsAllowedOriginList = s.CorsAllowedOriginList
148 c.HSTSDirectives = s.HSTSDirectives
149 c.ExternalAddress = s.ExternalHost
150 c.MaxRequestsInFlight = s.MaxRequestsInFlight
151 c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
152 c.LivezGracePeriod = s.LivezGracePeriod
153 c.RequestTimeout = s.RequestTimeout
154 c.GoawayChance = s.GoawayChance
155 c.MinRequestTimeout = s.MinRequestTimeout
156 c.StorageInitializationTimeout = s.StorageInitializationTimeout
157 c.ShutdownDelayDuration = s.ShutdownDelayDuration
158 c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
159 c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
160 c.PublicAddress = s.AdvertiseAddress
161 c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
162 c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
163 c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
164 c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName)
165 c.EmulationForwardCompatible = s.EmulationForwardCompatible
166 c.RuntimeConfigEmulationForwardCompatible = s.RuntimeConfigEmulationForwardCompatible
167
168 return nil
169 }
170
171 // DefaultAdvertiseAddress sets the field AdvertiseAddress if unset. The field will be set based on the SecureServingOptions.
172 func (s *ServerRunOptions) DefaultAdvertiseAddress(secure *SecureServingOptions) error {
173 if secure == nil {
174 return nil
175 }
176
177 if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
178 hostIP, err := secure.DefaultExternalAddress()
179 if err != nil {
180 return fmt.Errorf("Unable to find suitable network address.error='%v'. "+
181 "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
182 }
183 s.AdvertiseAddress = hostIP
184 }
185
186 return nil
187 }
188
189 // Validate checks validation of ServerRunOptions
190 func (s *ServerRunOptions) Validate() []error {
191 errors := []error{}
192
193 if s.LivezGracePeriod < 0 {
194 errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
195 }
196
197 if s.MaxRequestsInFlight < 0 {
198 errors = append(errors, fmt.Errorf("--max-requests-inflight can not be negative value"))
199 }
200 if s.MaxMutatingRequestsInFlight < 0 {
201 errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight can not be negative value"))
202 }
203
204 if s.RequestTimeout.Nanoseconds() < 0 {
205 errors = append(errors, fmt.Errorf("--request-timeout can not be negative value"))
206 }
207
208 if s.GoawayChance < 0 || s.GoawayChance > 0.02 {
209 errors = append(errors, fmt.Errorf("--goaway-chance can not be less than 0 or greater than 0.02"))
210 }
211
212 if s.MinRequestTimeout < 0 {
213 errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
214 }
215
216 if s.StorageInitializationTimeout < 0 {
217 errors = append(errors, fmt.Errorf("--storage-initialization-timeout can not be negative value"))
218 }
219
220 if s.ShutdownDelayDuration < 0 {
221 errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
222 }
223
224 if s.ShutdownWatchTerminationGracePeriod < 0 {
225 errors = append(errors, fmt.Errorf("shutdown-watch-termination-grace-period, if provided, can not be a negative value"))
226 }
227
228 if s.JSONPatchMaxCopyBytes < 0 {
229 errors = append(errors, fmt.Errorf("ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value"))
230 }
231
232 if s.MaxRequestBodyBytes < 0 {
233 errors = append(errors, fmt.Errorf("ServerRunOptions.MaxRequestBodyBytes can not be negative value"))
234 }
235
236 if err := validateHSTSDirectives(s.HSTSDirectives); err != nil {
237 errors = append(errors, err)
238 }
239
240 if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
241 errors = append(errors, err)
242 }
243 if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 {
244 errors = append(errors, errs...)
245 }
246 effectiveVersion := s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
247 if effectiveVersion == nil {
248 return errors
249 }
250 notEmulationMode := effectiveVersion.BinaryVersion().WithPatch(0).EqualTo(effectiveVersion.EmulationVersion())
251 if notEmulationMode && s.EmulationForwardCompatible {
252 errors = append(errors, fmt.Errorf("ServerRunOptions.EmulationForwardCompatible cannot be set to true if the emulation version is the same as the binary version"))
253 }
254 if notEmulationMode && s.RuntimeConfigEmulationForwardCompatible {
255 errors = append(errors, fmt.Errorf("ServerRunOptions.RuntimeConfigEmulationForwardCompatible cannot be set to true if the emulation version is the same as the binary version"))
256 }
257 return errors
258 }
259
260 func validateHSTSDirectives(hstsDirectives []string) error {
261 // HSTS Headers format: Strict-Transport-Security:max-age=expireTime [;includeSubDomains] [;preload]
262 // See https://tools.ietf.org/html/rfc6797#section-6.1 for more information
263 allErrors := []error{}
264 for _, hstsDirective := range hstsDirectives {
265 if len(strings.TrimSpace(hstsDirective)) == 0 {
266 allErrors = append(allErrors, fmt.Errorf("empty value in strict-transport-security-directives"))
267 continue
268 }
269 if hstsDirective != "includeSubDomains" && hstsDirective != "preload" {
270 maxAgeDirective := strings.Split(hstsDirective, "=")
271 if len(maxAgeDirective) != 2 || maxAgeDirective[0] != "max-age" {
272 allErrors = append(allErrors, fmt.Errorf("--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information"))
273 }
274 }
275 }
276 return errors.NewAggregate(allErrors)
277 }
278
279 func validateCorsAllowedOriginList(corsAllowedOriginList []string) error {
280 allErrors := []error{}
281 validateRegexFn := func(regexpStr string) error {
282 if _, err := regexp.Compile(regexpStr); err != nil {
283 return err
284 }
285
286 // the regular expression should pin to the start and end of the host
287 // in the origin header, this will prevent CVE-2022-1996.
288 // possible ways it can pin to the start of host in the origin header:
289 // - match the start of the origin with '^'
290 // - match what separates the scheme and host with '//' or '://',
291 // this pins to the start of host in the origin header.
292 // possible ways it can match the end of the host in the origin header:
293 // - match the end of the origin with '$'
294 // - with a capture group that matches the host and port separator '(:|$)'
295 // We will relax the validation to check if these regex markers
296 // are present in the user specified expression.
297 var pinStart, pinEnd bool
298 for _, prefix := range []string{"^", "//"} {
299 if strings.Contains(regexpStr, prefix) {
300 pinStart = true
301 break
302 }
303 }
304 for _, suffix := range []string{"$", ":"} {
305 if strings.Contains(regexpStr, suffix) {
306 pinEnd = true
307 break
308 }
309 }
310 if !pinStart || !pinEnd {
311 return fmt.Errorf("regular expression does not pin to start/end of host in the origin header")
312 }
313 return nil
314 }
315
316 for _, regexp := range corsAllowedOriginList {
317 if len(regexp) == 0 {
318 allErrors = append(allErrors, fmt.Errorf("empty value in --cors-allowed-origins, help: %s", corsAllowedOriginsHelpText))
319 continue
320 }
321
322 if err := validateRegexFn(regexp); err != nil {
323 err = fmt.Errorf("--cors-allowed-origins has an invalid regular expression: %v, help: %s", err, corsAllowedOriginsHelpText)
324 allErrors = append(allErrors, err)
325 }
326 }
327 return errors.NewAggregate(allErrors)
328 }
329
330 // AddUniversalFlags adds flags for a specific APIServer to the specified FlagSet
331 func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
332 // Note: the weird ""+ in below lines seems to be the only way to get gofmt to
333 // arrange these text blocks sensibly. Grrr.
334
335 fs.IPVar(&s.AdvertiseAddress, "advertise-address", s.AdvertiseAddress, ""+
336 "The IP address on which to advertise the apiserver to members of the cluster. This "+
337 "address must be reachable by the rest of the cluster. If blank, the --bind-address "+
338 "will be used. If --bind-address is unspecified, the host's default interface will "+
339 "be used.")
340
341 fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, corsAllowedOriginsHelpText)
342
343 fs.StringSliceVar(&s.HSTSDirectives, "strict-transport-security-directives", s.HSTSDirectives, ""+
344 "List of directives for HSTS, comma separated. If this list is empty, then HSTS directives will not "+
345 "be added. Example: 'max-age=31536000,includeSubDomains,preload'")
346
347 fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
348 "The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs or OpenID Discovery).")
349
350 fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
351 "This and --max-mutating-requests-inflight are summed to determine the server's total concurrency limit "+
352 "(which must be positive) if --enable-priority-and-fairness is true. "+
353 "Otherwise, this flag limits the maximum number of non-mutating requests in flight, "+
354 "or a zero value disables the limit completely.")
355
356 fs.IntVar(&s.MaxMutatingRequestsInFlight, "max-mutating-requests-inflight", s.MaxMutatingRequestsInFlight, ""+
357 "This and --max-requests-inflight are summed to determine the server's total concurrency limit "+
358 "(which must be positive) if --enable-priority-and-fairness is true. "+
359 "Otherwise, this flag limits the maximum number of mutating requests in flight, "+
360 "or a zero value disables the limit completely.")
361
362 fs.DurationVar(&s.RequestTimeout, "request-timeout", s.RequestTimeout, ""+
363 "An optional field indicating the duration a handler must keep a request open before timing "+
364 "it out. This is the default request timeout for requests but may be overridden by flags such as "+
365 "--min-request-timeout for specific types of requests.")
366
367 fs.Float64Var(&s.GoawayChance, "goaway-chance", s.GoawayChance, ""+
368 "To prevent HTTP/2 clients from getting stuck on a single apiserver, randomly close a connection (GOAWAY). "+
369 "The client's other in-flight requests won't be affected, and the client will reconnect, likely landing on a different apiserver after going through the load balancer again. "+
370 "This argument sets the fraction of requests that will be sent a GOAWAY. Clusters with single apiservers, or which don't use a load balancer, should NOT enable this. "+
371 "Min is 0 (off), Max is .02 (1/50 requests); .001 (1/1000) is a recommended starting point.")
372
373 fs.DurationVar(&s.LivezGracePeriod, "livez-grace-period", s.LivezGracePeriod, ""+
374 "This option represents the maximum amount of time it should take for apiserver to complete its startup sequence "+
375 "and become live. From apiserver's start time to when this amount of time has elapsed, /livez will assume "+
376 "that unfinished post-start hooks will complete successfully and therefore return true.")
377
378 fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+
379 "An optional field indicating the minimum number of seconds a handler must keep "+
380 "a request open before timing it out. Currently only honored by the watch request "+
381 "handler, which picks a randomized value above this number as the connection timeout, "+
382 "to spread out load.")
383
384 fs.DurationVar(&s.StorageInitializationTimeout, "storage-initialization-timeout", s.StorageInitializationTimeout,
385 "Maximum amount of time to wait for storage initialization before declaring apiserver ready. Defaults to 1m.")
386
387 fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
388 "Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
389 "will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
390 "has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")
391
392 fs.BoolVar(&s.ShutdownSendRetryAfter, "shutdown-send-retry-after", s.ShutdownSendRetryAfter, ""+
393 "If true the HTTP Server will continue listening until all non long running request(s) in flight have been drained, "+
394 "during this window all incoming requests will be rejected with a status code 429 and a 'Retry-After' response header, "+
395 "in addition 'Connection: close' response header is set in order to tear down the TCP connection when idle.")
396
397 fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+
398 "This option, if set, represents the maximum amount of grace period the apiserver will wait "+
399 "for active watch request(s) to drain during the graceful server shutdown window.")
400
401 s.ComponentGlobalsRegistry.AddFlags(fs)
402 fs.BoolVar(&s.EmulationForwardCompatible, "emulation-forward-compatible", s.EmulationForwardCompatible, ""+
403 "If true, for any beta+ APIs enabled by default or by --runtime-config at the emulation version, their future versions with higher priority/stability will be auto enabled even if they introduced after the emulation version. "+
404 "Can only be set to true if the emulation version is lower than the binary version.")
405 fs.BoolVar(&s.RuntimeConfigEmulationForwardCompatible, "runtime-config-emulation-forward-compatible", s.RuntimeConfigEmulationForwardCompatible, ""+
406 "If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. "+
407 "If false, server would fail to start if any APIs identified by group/version that are enabled in the --runtime-config flag are introduced after the emulation version. "+
408 "Can only be set to true if the emulation version is lower than the binary version.")
409 }
410
411 // Complete fills missing fields with defaults.
412 func (s *ServerRunOptions) Complete() error {
413 return s.ComponentGlobalsRegistry.SetFallback()
414 }