Debian Codegraph @

pkg/server/storage/storage_factory.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package storage
18
19 import (
20 "crypto/tls"
21 "crypto/x509"
22 "os"
23 "strings"
24
25 "k8s.io/apimachinery/pkg/runtime"
26 "k8s.io/apimachinery/pkg/runtime/schema"
27 "k8s.io/apimachinery/pkg/util/sets"
28 "k8s.io/apiserver/pkg/storage/storagebackend"
29 "k8s.io/klog/v2"
30 )
31
32 // Backend describes the storage servers, the information here should be enough
33 // for health validations.
34 type Backend struct {
35 // the url of storage backend like: https://etcd.domain:2379
36 Server string
37 // the required tls config
38 TLSConfig *tls.Config
39 }
40
41 // StorageFactory is the interface to locate the storage for a given GroupResource
42 type StorageFactory interface {
43 // New finds the storage destination for the given group and resource. It will
44 // return an error if the group has no storage destination configured.
45 NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error)
46
47 // ResourcePrefix returns the overridden resource prefix for the GroupResource
48 // This allows for cohabitation of resources with different native types and provides
49 // centralized control over the shape of etcd directories
50 ResourcePrefix(groupResource schema.GroupResource) string
51
52 // Configs gets configurations for all of registered storage destinations.
53 Configs() []storagebackend.Config
54
55 // Backends gets all backends for all registered storage destinations.
56 // Used for getting all instances for health validations.
57 // Deprecated: Use Configs instead
58 Backends() []Backend
59 }
60
61 // DefaultStorageFactory takes a GroupResource and returns back its storage interface. This result includes:
62 // 1. Merged etcd config, including: auth, server locations, prefixes
63 // 2. Resource encodings for storage: group,version,kind to store as
64 // 3. Cohabitating default: some resources like hpa are exposed through multiple APIs. They must agree on 1 and 2
65 type DefaultStorageFactory struct {
66 // StorageConfig describes how to create a storage backend in general.
67 // Its authentication information will be used for every storage.Interface returned.
68 StorageConfig storagebackend.Config
69
70 Overrides map[schema.GroupResource]groupResourceOverrides
71
72 DefaultResourcePrefixes map[schema.GroupResource]string
73
74 // DefaultMediaType is the media type used to store resources. If it is not set, "application/json" is used.
75 DefaultMediaType string
76
77 // DefaultSerializer is used to create encoders and decoders for the storage.Interface.
78 DefaultSerializer runtime.StorageSerializer
79
80 // ResourceEncodingConfig describes how to encode a particular GroupVersionResource
81 ResourceEncodingConfig ResourceEncodingConfig
82
83 // APIResourceConfigSource indicates whether the *storage* is enabled, NOT the API
84 // This is discrete from resource enablement because those are separate concerns. How this source is configured
85 // is left to the caller.
86 APIResourceConfigSource APIResourceConfigSource
87
88 // newStorageCodecFn exists to be overwritten for unit testing.
89 newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error)
90 }
91
92 type groupResourceOverrides struct {
93 // etcdLocation contains the list of "special" locations that are used for particular GroupResources
94 // These are merged on top of the StorageConfig when requesting the storage.Interface for a given GroupResource
95 etcdLocation []string
96 // etcdPrefix is the base location for a GroupResource.
97 etcdPrefix string
98 // etcdResourcePrefix is the location to use to store a particular type under the `etcdPrefix` location
99 // If empty, the default mapping is used. If the default mapping doesn't contain an entry, it will use
100 // the ToLowered name of the resource, not including the group.
101 etcdResourcePrefix string
102 // mediaType is the desired serializer to choose. If empty, the default is chosen.
103 mediaType string
104 // serializer contains the list of "special" serializers for a GroupResource. Resource=* means for the entire group
105 serializer runtime.StorageSerializer
106 // cohabitatingResources keeps track of which resources must be stored together. This happens when we have multiple ways
107 // of exposing one set of concepts. autoscaling.HPA and extensions.HPA as a for instance
108 // The order of the slice matters! It is the priority order of lookup for finding a storage location
109 cohabitatingResources []schema.GroupResource
110 // encoderDecoratorFn is optional and may wrap the provided encoder prior to being serialized.
111 encoderDecoratorFn func(runtime.Encoder) runtime.Encoder
112 // decoderDecoratorFn is optional and may wrap the provided decoders (can add new decoders). The order of
113 // returned decoders will be priority for attempt to decode.
114 decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder
115 }
116
117 // Apply overrides the provided config and options if the override has a value in that position
118 func (o groupResourceOverrides) Apply(config *storagebackend.Config, options *StorageCodecConfig) {
119 if len(o.etcdLocation) > 0 {
120 config.Transport.ServerList = o.etcdLocation
121 }
122 if len(o.etcdPrefix) > 0 {
123 config.Prefix = o.etcdPrefix
124 }
125
126 if len(o.mediaType) > 0 {
127 options.StorageMediaType = o.mediaType
128 }
129 if o.serializer != nil {
130 options.StorageSerializer = o.serializer
131 }
132 if o.encoderDecoratorFn != nil {
133 options.EncoderDecoratorFn = o.encoderDecoratorFn
134 }
135 if o.decoderDecoratorFn != nil {
136 options.DecoderDecoratorFn = o.decoderDecoratorFn
137 }
138 }
139
140 var _ StorageFactory = &DefaultStorageFactory{}
141
142 const AllResources = "*"
143
144 func NewDefaultStorageFactory(
145 config storagebackend.Config,
146 defaultMediaType string,
147 defaultSerializer runtime.StorageSerializer,
148 resourceEncodingConfig ResourceEncodingConfig,
149 resourceConfig APIResourceConfigSource,
150 specialDefaultResourcePrefixes map[schema.GroupResource]string,
151 ) *DefaultStorageFactory {
152 if len(defaultMediaType) == 0 {
153 defaultMediaType = runtime.ContentTypeJSON
154 }
155 return &DefaultStorageFactory{
156 StorageConfig: config,
157 Overrides: map[schema.GroupResource]groupResourceOverrides{},
158 DefaultMediaType: defaultMediaType,
159 DefaultSerializer: defaultSerializer,
160 ResourceEncodingConfig: resourceEncodingConfig,
161 APIResourceConfigSource: resourceConfig,
162 DefaultResourcePrefixes: specialDefaultResourcePrefixes,
163
164 newStorageCodecFn: NewStorageCodec,
165 }
166 }
167
168 func (s *DefaultStorageFactory) SetEtcdLocation(groupResource schema.GroupResource, location []string) {
169 overrides := s.Overrides[groupResource]
170 overrides.etcdLocation = location
171 s.Overrides[groupResource] = overrides
172 }
173
174 func (s *DefaultStorageFactory) SetEtcdPrefix(groupResource schema.GroupResource, prefix string) {
175 overrides := s.Overrides[groupResource]
176 overrides.etcdPrefix = prefix
177 s.Overrides[groupResource] = overrides
178 }
179
180 // SetResourceEtcdPrefix sets the prefix for a resource, but not the base-dir. You'll end up in `etcdPrefix/resourceEtcdPrefix`.
181 func (s *DefaultStorageFactory) SetResourceEtcdPrefix(groupResource schema.GroupResource, prefix string) {
182 overrides := s.Overrides[groupResource]
183 overrides.etcdResourcePrefix = prefix
184 s.Overrides[groupResource] = overrides
185 }
186
187 func (s *DefaultStorageFactory) SetSerializer(groupResource schema.GroupResource, mediaType string, serializer runtime.StorageSerializer) {
188 overrides := s.Overrides[groupResource]
189 overrides.mediaType = mediaType
190 overrides.serializer = serializer
191 s.Overrides[groupResource] = overrides
192 }
193
194 // AddCohabitatingResources links resources together the order of the slice matters! its the priority order of lookup for finding a storage location
195 func (s *DefaultStorageFactory) AddCohabitatingResources(groupResources ...schema.GroupResource) {
196 for _, groupResource := range groupResources {
197 overrides := s.Overrides[groupResource]
198 overrides.cohabitatingResources = groupResources
199 s.Overrides[groupResource] = overrides
200 }
201 }
202
203 func (s *DefaultStorageFactory) AddSerializationChains(encoderDecoratorFn func(runtime.Encoder) runtime.Encoder, decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder, groupResources ...schema.GroupResource) {
204 for _, groupResource := range groupResources {
205 overrides := s.Overrides[groupResource]
206 overrides.encoderDecoratorFn = encoderDecoratorFn
207 overrides.decoderDecoratorFn = decoderDecoratorFn
208 s.Overrides[groupResource] = overrides
209 }
210 }
211
212 func getAllResourcesAlias(resource schema.GroupResource) schema.GroupResource {
213 return schema.GroupResource{Group: resource.Group, Resource: AllResources}
214 }
215
216 func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.GroupResource) schema.GroupResource {
217 for _, potentialStorageResource := range s.Overrides[groupResource].cohabitatingResources {
218 // TODO deads2k or liggitt determine if have ever stored any of our cohabitating resources in a different location on new clusters
219 if s.APIResourceConfigSource.AnyResourceForGroupEnabled(potentialStorageResource.Group) {
220 return potentialStorageResource
221 }
222 }
223
224 return groupResource
225 }
226
227 // New finds the storage destination for the given group and resource. It will
228 // return an error if the group has no storage destination configured.
229 func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
230 chosenStorageResource := s.getStorageGroupResource(groupResource)
231
232 // operate on copy
233 storageConfig := s.StorageConfig
234 codecConfig := StorageCodecConfig{
235 StorageMediaType: s.DefaultMediaType,
236 StorageSerializer: s.DefaultSerializer,
237 }
238
239 if override, ok := s.Overrides[getAllResourcesAlias(chosenStorageResource)]; ok {
240 override.Apply(&storageConfig, &codecConfig)
241 }
242 if override, ok := s.Overrides[chosenStorageResource]; ok {
243 override.Apply(&storageConfig, &codecConfig)
244 }
245
246 var err error
247 if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok {
248 codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(chosenStorageResource, example)
249 if err != nil {
250 return nil, err
251 }
252 } else {
253 codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
254 if err != nil {
255 return nil, err
256 }
257 }
258
259 codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource)
260 if err != nil {
261 return nil, err
262 }
263
264 codecConfig.Config = storageConfig
265
266 storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)
267 if err != nil {
268 return nil, err
269 }
270 klog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config)
271
272 return storageConfig.ForResource(groupResource), nil
273 }
274
275 // Configs implements StorageFactory.
276 func (s *DefaultStorageFactory) Configs() []storagebackend.Config {
277 return configs(s.StorageConfig, s.Overrides)
278 }
279
280 // Configs gets configurations for all of registered storage destinations.
281 func Configs(storageConfig storagebackend.Config) []storagebackend.Config {
282 return configs(storageConfig, nil)
283 }
284
285 // Returns all storage configurations including those for group resource overrides
286 func configs(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []storagebackend.Config {
287 configs := []storagebackend.Config{storageConfig}
288
289 for _, override := range grOverrides {
290 if len(override.etcdLocation) == 0 {
291 continue
292 }
293 // copy
294 newConfig := storageConfig
295 override.Apply(&newConfig, &StorageCodecConfig{})
296 newConfig.Transport.ServerList = override.etcdLocation
297 configs = append(configs, newConfig)
298 }
299 return configs
300 }
301
302 // Backends implements StorageFactory.
303 func (s *DefaultStorageFactory) Backends() []Backend {
304 return backends(s.StorageConfig, s.Overrides)
305 }
306
307 // Backends returns all backends for all registered storage destinations.
308 // Used for getting all instances for health validations.
309 // Deprecated: Validate health by passing storagebackend.Config directly to storagefactory.CreateProber.
310 func Backends(storageConfig storagebackend.Config) []Backend {
311 return backends(storageConfig, nil)
312 }
313
314 func backends(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []Backend {
315 servers := sets.NewString(storageConfig.Transport.ServerList...)
316
317 for _, overrides := range grOverrides {
318 servers.Insert(overrides.etcdLocation...)
319 }
320
321 tlsConfig := &tls.Config{
322 InsecureSkipVerify: true,
323 }
324 if len(storageConfig.Transport.CertFile) > 0 && len(storageConfig.Transport.KeyFile) > 0 {
325 cert, err := tls.LoadX509KeyPair(storageConfig.Transport.CertFile, storageConfig.Transport.KeyFile)
326 if err != nil {
327 klog.Errorf("failed to load key pair while getting backends: %s", err)
328 } else {
329 tlsConfig.Certificates = []tls.Certificate{cert}
330 }
331 }
332 if len(storageConfig.Transport.TrustedCAFile) > 0 {
333 if caCert, err := os.ReadFile(storageConfig.Transport.TrustedCAFile); err != nil {
334 klog.Errorf("failed to read ca file while getting backends: %s", err)
335 } else {
336 caPool := x509.NewCertPool()
337 caPool.AppendCertsFromPEM(caCert)
338 tlsConfig.RootCAs = caPool
339 tlsConfig.InsecureSkipVerify = false
340 }
341 }
342
343 backends := []Backend{}
344 for server := range servers {
345 backends = append(backends, Backend{
346 Server: server,
347 // We can't share TLSConfig across different backends to avoid races.
348 // For more details see: https://pr.k8s.io/59338
349 TLSConfig: tlsConfig.Clone(),
350 })
351 }
352 return backends
353 }
354
355 func (s *DefaultStorageFactory) ResourcePrefix(groupResource schema.GroupResource) string {
356 chosenStorageResource := s.getStorageGroupResource(groupResource)
357 groupOverride := s.Overrides[getAllResourcesAlias(chosenStorageResource)]
358 exactResourceOverride := s.Overrides[chosenStorageResource]
359
360 etcdResourcePrefix := s.DefaultResourcePrefixes[chosenStorageResource]
361 if len(groupOverride.etcdResourcePrefix) > 0 {
362 etcdResourcePrefix = groupOverride.etcdResourcePrefix
363 }
364 if len(exactResourceOverride.etcdResourcePrefix) > 0 {
365 etcdResourcePrefix = exactResourceOverride.etcdResourcePrefix
366 }
367 if len(etcdResourcePrefix) == 0 {
368 etcdResourcePrefix = strings.ToLower(chosenStorageResource.Resource)
369 }
370
371 return etcdResourcePrefix
372 }