Debian Codegraph @

pkg/server/options/etcd.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package options
18
19 import (
20 "context"
21 "fmt"
22 "net/http"
23 "strconv"
24 "strings"
25 "time"
26
27 "github.com/spf13/pflag"
28
29 "k8s.io/apimachinery/pkg/runtime"
30 "k8s.io/apimachinery/pkg/runtime/schema"
31 "k8s.io/apimachinery/pkg/util/sets"
32 "k8s.io/apimachinery/pkg/util/wait"
33 "k8s.io/apiserver/pkg/registry/generic"
34 genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
35 "k8s.io/apiserver/pkg/server"
36 "k8s.io/apiserver/pkg/server/healthz"
37 "k8s.io/apiserver/pkg/server/options/encryptionconfig"
38 encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
39 serverstorage "k8s.io/apiserver/pkg/server/storage"
40 "k8s.io/apiserver/pkg/storage/etcd3/metrics"
41 "k8s.io/apiserver/pkg/storage/storagebackend"
42 storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
43 storagevalue "k8s.io/apiserver/pkg/storage/value"
44 "k8s.io/klog/v2"
45 )
46
47 type EtcdOptions struct {
48 StorageConfig storagebackend.Config
49 EncryptionProviderConfigFilepath string
50 EncryptionProviderConfigAutomaticReload bool
51
52 EtcdServersOverrides []string
53
54 // To enable protobuf as storage format, it is enough
55 // to set it to "application/vnd.kubernetes.protobuf".
56 DefaultStorageMediaType string
57 DeleteCollectionWorkers int
58 EnableGarbageCollection bool
59
60 // Set EnableWatchCache to false to disable all watch caches
61 EnableWatchCache bool
62 // Set DefaultWatchCacheSize to zero to disable watch caches for those resources that have no explicit cache size set
63 DefaultWatchCacheSize int
64 // WatchCacheSizes represents override to a given resource
65 WatchCacheSizes []string
66
67 // SkipHealthEndpoints, when true, causes the Apply methods to not set up health endpoints.
68 // This allows multiple invocations of the Apply methods without duplication of said endpoints.
69 SkipHealthEndpoints bool
70 }
71
72 var storageTypes = sets.NewString(
73 storagebackend.StorageTypeETCD3,
74 )
75
76 func NewEtcdOptions(backendConfig *storagebackend.Config) *EtcdOptions {
77 options := &EtcdOptions{
78 StorageConfig: *backendConfig,
79 DefaultStorageMediaType: "application/json",
80 DeleteCollectionWorkers: 1,
81 EnableGarbageCollection: true,
82 EnableWatchCache: true,
83 DefaultWatchCacheSize: 100,
84 }
85 options.StorageConfig.CountMetricPollPeriod = time.Minute
86 return options
87 }
88
89 var storageMediaTypes = sets.New(
90 runtime.ContentTypeJSON,
91 runtime.ContentTypeYAML,
92 runtime.ContentTypeProtobuf,
93 )
94
95 func (s *EtcdOptions) Validate() []error {
96 if s == nil {
97 return nil
98 }
99
100 allErrors := []error{}
101 if len(s.StorageConfig.Transport.ServerList) == 0 {
102 allErrors = append(allErrors, fmt.Errorf("--etcd-servers must be specified"))
103 }
104
105 if s.StorageConfig.Type != storagebackend.StorageTypeUnset && !storageTypes.Has(s.StorageConfig.Type) {
106 allErrors = append(allErrors, fmt.Errorf("--storage-backend invalid, allowed values: %s. If not specified, it will default to 'etcd3'", strings.Join(storageTypes.List(), ", ")))
107 }
108
109 for _, override := range s.EtcdServersOverrides {
110 tokens := strings.Split(override, "#")
111 if len(tokens) != 2 {
112 allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated"))
113 continue
114 }
115
116 apiresource := strings.Split(tokens[0], "/")
117 if len(apiresource) != 2 {
118 allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated"))
119 continue
120 }
121
122 }
123
124 if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload {
125 allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config"))
126 }
127
128 if s.DefaultStorageMediaType != "" && !storageMediaTypes.Has(s.DefaultStorageMediaType) {
129 allErrors = append(allErrors, fmt.Errorf("--storage-media-type %q invalid, allowed values: %s", s.DefaultStorageMediaType, strings.Join(sets.List(storageMediaTypes), ", ")))
130 }
131
132 return allErrors
133 }
134
135 // AddFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet
136 func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
137 if s == nil {
138 return
139 }
140
141 fs.StringSliceVar(&s.EtcdServersOverrides, "etcd-servers-overrides", s.EtcdServersOverrides, ""+
142 "Per-resource etcd servers overrides, comma separated. The individual override "+
143 "format: group/resource#servers, where servers are URLs, semicolon separated. "+
144 "Note that this applies only to resources compiled into this server binary. ")
145
146 fs.StringVar(&s.DefaultStorageMediaType, "storage-media-type", s.DefaultStorageMediaType, ""+
147 "The media type to use to store objects in storage. "+
148 "Some resources or storage backends may only support a specific media type and will ignore this setting. "+
149 "Supported media types: [application/json, application/yaml, application/vnd.kubernetes.protobuf]")
150 fs.IntVar(&s.DeleteCollectionWorkers, "delete-collection-workers", s.DeleteCollectionWorkers,
151 "Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.")
152
153 fs.BoolVar(&s.EnableGarbageCollection, "enable-garbage-collector", s.EnableGarbageCollection, ""+
154 "Enables the generic garbage collector. MUST be synced with the corresponding flag "+
155 "of the kube-controller-manager.")
156
157 fs.BoolVar(&s.EnableWatchCache, "watch-cache", s.EnableWatchCache,
158 "Enable watch caching in the apiserver")
159
160 fs.IntVar(&s.DefaultWatchCacheSize, "default-watch-cache-size", s.DefaultWatchCacheSize,
161 "Default watch cache size. If zero, watch cache will be disabled for resources that do not have a default watch size set.")
162
163 fs.MarkDeprecated("default-watch-cache-size",
164 "watch caches are sized automatically and this flag will be removed in a future version")
165
166 fs.StringSliceVar(&s.WatchCacheSizes, "watch-cache-sizes", s.WatchCacheSizes, ""+
167 "Watch cache size settings for some resources (pods, nodes, etc.), comma separated. "+
168 "The individual setting format: resource[.group]#size, where resource is lowercase plural (no version), "+
169 "group is omitted for resources of apiVersion v1 (the legacy core API) and included for others, "+
170 "and size is a number. This option is only meaningful for resources built into the apiserver, "+
171 "not ones defined by CRDs or aggregated from external servers, and is only consulted if the "+
172 "watch-cache is enabled. The only meaningful size setting to supply here is zero, which means to "+
173 "disable watch caching for the associated resource; all non-zero values are equivalent and mean "+
174 "to not disable watch caching for that resource")
175
176 fs.StringVar(&s.StorageConfig.Type, "storage-backend", s.StorageConfig.Type,
177 "The storage backend for persistence. Options: 'etcd3' (default).")
178
179 fs.StringSliceVar(&s.StorageConfig.Transport.ServerList, "etcd-servers", s.StorageConfig.Transport.ServerList,
180 "List of etcd servers to connect with (scheme://ip:port), comma separated.")
181
182 fs.StringVar(&s.StorageConfig.Prefix, "etcd-prefix", s.StorageConfig.Prefix,
183 "The prefix to prepend to all resource paths in etcd.")
184
185 fs.StringVar(&s.StorageConfig.Transport.KeyFile, "etcd-keyfile", s.StorageConfig.Transport.KeyFile,
186 "SSL key file used to secure etcd communication.")
187
188 fs.StringVar(&s.StorageConfig.Transport.CertFile, "etcd-certfile", s.StorageConfig.Transport.CertFile,
189 "SSL certification file used to secure etcd communication.")
190
191 fs.StringVar(&s.StorageConfig.Transport.TrustedCAFile, "etcd-cafile", s.StorageConfig.Transport.TrustedCAFile,
192 "SSL Certificate Authority file used to secure etcd communication.")
193
194 fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath,
195 "The file containing configuration for encryption providers to be used for storing secrets in etcd")
196
197 fs.BoolVar(&s.EncryptionProviderConfigAutomaticReload, "encryption-provider-config-automatic-reload", s.EncryptionProviderConfigAutomaticReload,
198 "Determines if the file set by --encryption-provider-config should be automatically reloaded if the disk contents change. "+
199 "Setting this to true disables the ability to uniquely identify distinct KMS plugins via the API server healthz endpoints.")
200
201 fs.DurationVar(&s.StorageConfig.CompactionInterval, "etcd-compaction-interval", s.StorageConfig.CompactionInterval,
202 "The interval of compaction requests. If 0, the compaction request from apiserver is disabled.")
203
204 fs.DurationVar(&s.StorageConfig.CountMetricPollPeriod, "etcd-count-metric-poll-period", s.StorageConfig.CountMetricPollPeriod, ""+
205 "Frequency of polling etcd for number of resources per type. 0 disables the metric collection.")
206
207 fs.DurationVar(&s.StorageConfig.DBMetricPollInterval, "etcd-db-metric-poll-interval", s.StorageConfig.DBMetricPollInterval,
208 "The interval of requests to poll etcd and update metric. 0 disables the metric collection")
209
210 fs.DurationVar(&s.StorageConfig.HealthcheckTimeout, "etcd-healthcheck-timeout", s.StorageConfig.HealthcheckTimeout,
211 "The timeout to use when checking etcd health.")
212
213 fs.DurationVar(&s.StorageConfig.ReadycheckTimeout, "etcd-readycheck-timeout", s.StorageConfig.ReadycheckTimeout,
214 "The timeout to use when checking etcd readiness")
215
216 fs.Int64Var(&s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds, "lease-reuse-duration-seconds", s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds,
217 "The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.")
218 }
219
220 // ApplyTo mutates the provided server.Config. It must never mutate the receiver (EtcdOptions).
221 func (s *EtcdOptions) ApplyTo(c *server.Config) error {
222 if s == nil {
223 return nil
224 }
225
226 storageConfigCopy := s.StorageConfig
227 if storageConfigCopy.StorageObjectCountTracker == nil {
228 storageConfigCopy.StorageObjectCountTracker = c.StorageObjectCountTracker
229 }
230
231 return s.ApplyWithStorageFactoryTo(&SimpleStorageFactory{StorageConfig: storageConfigCopy}, c)
232 }
233
234 // ApplyWithStorageFactoryTo mutates the provided server.Config. It must never mutate the receiver (EtcdOptions).
235 func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error {
236 if s == nil {
237 return nil
238 }
239
240 if !s.SkipHealthEndpoints {
241 if err := s.addEtcdHealthEndpoint(c); err != nil {
242 return err
243 }
244 }
245
246 // setup encryption
247 if err := s.maybeApplyResourceTransformers(c); err != nil {
248 return err
249 }
250
251 metrics.SetStorageMonitorGetter(monitorGetter(factory))
252
253 c.RESTOptionsGetter = s.CreateRESTOptionsGetter(factory, c.ResourceTransformers)
254 return nil
255 }
256
257 func monitorGetter(factory serverstorage.StorageFactory) func() (monitors []metrics.Monitor, err error) {
258 return func() (monitors []metrics.Monitor, err error) {
259 defer func() {
260 if err != nil {
261 for _, m := range monitors {
262 m.Close()
263 }
264 }
265 }()
266
267 var m metrics.Monitor
268 for _, cfg := range factory.Configs() {
269 m, err = storagefactory.CreateMonitor(cfg)
270 if err != nil {
271 return nil, err
272 }
273 monitors = append(monitors, m)
274 }
275 return monitors, nil
276 }
277 }
278
279 func (s *EtcdOptions) CreateRESTOptionsGetter(factory serverstorage.StorageFactory, resourceTransformers storagevalue.ResourceTransformers) generic.RESTOptionsGetter {
280 if resourceTransformers != nil {
281 factory = &transformerStorageFactory{
282 delegate: factory,
283 resourceTransformers: resourceTransformers,
284 }
285 }
286 return &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
287 }
288
289 func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err error) {
290 if c.ResourceTransformers != nil {
291 return nil
292 }
293 if len(s.EncryptionProviderConfigFilepath) == 0 {
294 return nil
295 }
296
297 ctxServer := wait.ContextForChannel(c.DrainedNotify())
298 ctxTransformers, closeTransformers := context.WithCancel(ctxServer)
299 defer func() {
300 // in case of error, we want to close partially initialized (if any) transformers
301 if err != nil {
302 closeTransformers()
303 }
304 }()
305
306 encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, c.APIServerID)
307 if err != nil {
308 return err
309 }
310
311 if s.EncryptionProviderConfigAutomaticReload {
312 // with reload=true we will always have 1 health check
313 if len(encryptionConfiguration.HealthChecks) != 1 {
314 return fmt.Errorf("failed to start kms encryption config hot reload controller. only 1 health check should be available when reload is enabled")
315 }
316
317 // Here the dynamic transformers take ownership of the transformers and their cancellation.
318 dynamicTransformers := encryptionconfig.NewDynamicTransformers(encryptionConfiguration.Transformers, encryptionConfiguration.HealthChecks[0], closeTransformers, encryptionConfiguration.KMSCloseGracePeriod)
319
320 // add post start hook to start hot reload controller
321 // adding this hook here will ensure that it gets configured exactly once
322 err = c.AddPostStartHook(
323 "start-encryption-provider-config-automatic-reload",
324 func(_ server.PostStartHookContext) error {
325 dynamicEncryptionConfigController := encryptionconfigcontroller.NewDynamicEncryptionConfiguration(
326 "encryption-provider-config-automatic-reload-controller",
327 s.EncryptionProviderConfigFilepath,
328 dynamicTransformers,
329 encryptionConfiguration.EncryptionFileContentHash,
330 c.APIServerID,
331 )
332
333 go dynamicEncryptionConfigController.Run(ctxServer)
334
335 return nil
336 },
337 )
338 if err != nil {
339 return fmt.Errorf("failed to add post start hook for kms encryption config hot reload controller: %w", err)
340 }
341
342 c.ResourceTransformers = dynamicTransformers
343 if !s.SkipHealthEndpoints {
344 addHealthChecksWithoutLivez(c, dynamicTransformers)
345 }
346 } else {
347 c.ResourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
348 if !s.SkipHealthEndpoints {
349 addHealthChecksWithoutLivez(c, encryptionConfiguration.HealthChecks...)
350 }
351 }
352
353 return nil
354 }
355
356 func addHealthChecksWithoutLivez(c *server.Config, healthChecks ...healthz.HealthChecker) {
357 c.HealthzChecks = append(c.HealthzChecks, healthChecks...)
358 c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...)
359 }
360
361 func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
362 healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig, c.DrainedNotify())
363 if err != nil {
364 return err
365 }
366 c.AddHealthChecks(healthz.NamedCheck("etcd", func(r *http.Request) error {
367 return healthCheck()
368 }))
369
370 readyCheck, err := storagefactory.CreateReadyCheck(s.StorageConfig, c.DrainedNotify())
371 if err != nil {
372 return err
373 }
374 c.AddReadyzChecks(healthz.NamedCheck("etcd-readiness", func(r *http.Request) error {
375 return readyCheck()
376 }))
377
378 return nil
379 }
380
381 type StorageFactoryRestOptionsFactory struct {
382 Options EtcdOptions
383 StorageFactory serverstorage.StorageFactory
384 }
385
386 func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
387 storageConfig, err := f.StorageFactory.NewConfig(resource, example)
388 if err != nil {
389 return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error())
390 }
391
392 ret := generic.RESTOptions{
393 StorageConfig: storageConfig,
394 Decorator: generic.UndecoratedStorage,
395 DeleteCollectionWorkers: f.Options.DeleteCollectionWorkers,
396 EnableGarbageCollection: f.Options.EnableGarbageCollection,
397 ResourcePrefix: f.StorageFactory.ResourcePrefix(resource),
398 CountMetricPollPeriod: f.Options.StorageConfig.CountMetricPollPeriod,
399 StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker,
400 }
401
402 if ret.StorageObjectCountTracker == nil {
403 ret.StorageObjectCountTracker = storageConfig.StorageObjectCountTracker
404 }
405
406 if f.Options.EnableWatchCache {
407 sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes)
408 if err != nil {
409 return generic.RESTOptions{}, err
410 }
411 size, ok := sizes[resource]
412 if ok && size > 0 {
413 klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
414 }
415 if ok && size <= 0 {
416 klog.V(3).InfoS("Not using watch cache", "resource", resource)
417 ret.Decorator = generic.UndecoratedStorage
418 } else {
419 klog.V(3).InfoS("Using watch cache", "resource", resource)
420 ret.Decorator = genericregistry.StorageWithCacher()
421 }
422 }
423
424 return ret, nil
425 }
426
427 // ParseWatchCacheSizes turns a list of cache size values into a map of group resources
428 // to requested sizes.
429 func ParseWatchCacheSizes(cacheSizes []string) (map[schema.GroupResource]int, error) {
430 watchCacheSizes := make(map[schema.GroupResource]int)
431 for _, c := range cacheSizes {
432 tokens := strings.Split(c, "#")
433 if len(tokens) != 2 {
434 return nil, fmt.Errorf("invalid value of watch cache size: %s", c)
435 }
436
437 size, err := strconv.Atoi(tokens[1])
438 if err != nil {
439 return nil, fmt.Errorf("invalid size of watch cache size: %s", c)
440 }
441 if size < 0 {
442 return nil, fmt.Errorf("watch cache size cannot be negative: %s", c)
443 }
444 watchCacheSizes[schema.ParseGroupResource(tokens[0])] = size
445 }
446 return watchCacheSizes, nil
447 }
448
449 // WriteWatchCacheSizes turns a map of cache size values into a list of string specifications.
450 func WriteWatchCacheSizes(watchCacheSizes map[schema.GroupResource]int) ([]string, error) {
451 var cacheSizes []string
452
453 for resource, size := range watchCacheSizes {
454 if size < 0 {
455 return nil, fmt.Errorf("watch cache size cannot be negative for resource %s", resource)
456 }
457 cacheSizes = append(cacheSizes, fmt.Sprintf("%s#%d", resource.String(), size))
458 }
459 return cacheSizes, nil
460 }
461
462 var _ serverstorage.StorageFactory = &SimpleStorageFactory{}
463
464 // SimpleStorageFactory provides a StorageFactory implementation that should be used when different
465 // resources essentially share the same storage config (as defined by the given storagebackend.Config).
466 // It assumes the resources are stored at a path that is purely based on the schema.GroupResource.
467 // Users that need flexibility and per resource overrides should use DefaultStorageFactory instead.
468 type SimpleStorageFactory struct {
469 StorageConfig storagebackend.Config
470 }
471
472 func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
473 return s.StorageConfig.ForResource(resource), nil
474 }
475
476 func (s *SimpleStorageFactory) ResourcePrefix(resource schema.GroupResource) string {
477 return resource.Group + "/" + resource.Resource
478 }
479
480 func (s *SimpleStorageFactory) Configs() []storagebackend.Config {
481 return serverstorage.Configs(s.StorageConfig)
482 }
483
484 func (s *SimpleStorageFactory) Backends() []serverstorage.Backend {
485 // nothing should ever call this method but we still provide a functional implementation
486 return serverstorage.Backends(s.StorageConfig)
487 }
488
489 var _ serverstorage.StorageFactory = &transformerStorageFactory{}
490
491 type transformerStorageFactory struct {
492 delegate serverstorage.StorageFactory
493 resourceTransformers storagevalue.ResourceTransformers
494 }
495
496 func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
497 config, err := t.delegate.NewConfig(resource, example)
498 if err != nil {
499 return nil, err
500 }
501
502 configCopy := *config
503 resourceConfig := configCopy.Config
504 resourceConfig.Transformer = t.resourceTransformers.TransformerForResource(resource)
505 configCopy.Config = resourceConfig
506
507 return &configCopy, nil
508 }
509
510 func (t *transformerStorageFactory) ResourcePrefix(resource schema.GroupResource) string {
511 return t.delegate.ResourcePrefix(resource)
512 }
513
514 func (t *transformerStorageFactory) Configs() []storagebackend.Config {
515 return t.delegate.Configs()
516 }
517
518 func (t *transformerStorageFactory) Backends() []serverstorage.Backend {
519 return t.delegate.Backends()
520 }