| 1 |
/* |
| 2 |
Copyright 2016 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package storage |
| 18 |
|
| 19 |
import ( |
| 20 |
"crypto/tls" |
| 21 |
"crypto/x509" |
| 22 |
"os" |
| 23 |
"strings" |
| 24 |
|
| 25 |
"k8s.io/apimachinery/pkg/runtime" |
| 26 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 27 |
"k8s.io/apimachinery/pkg/util/sets" |
| 28 |
"k8s.io/apiserver/pkg/storage/storagebackend" |
| 29 |
"k8s.io/klog/v2" |
| 30 |
) |
| 31 |
|
| 32 |
// Backend describes the storage servers, the information here should be enough |
| 33 |
// for health validations. |
| 34 |
type Backend struct { |
| 35 |
// the url of storage backend like: https://etcd.domain:2379 |
| 36 |
Server string |
| 37 |
// the required tls config |
| 38 |
TLSConfig *tls.Config |
| 39 |
} |
| 40 |
|
| 41 |
// StorageFactory is the interface to locate the storage for a given GroupResource |
| 42 |
type StorageFactory interface { |
| 43 |
// New finds the storage destination for the given group and resource. It will |
| 44 |
// return an error if the group has no storage destination configured. |
| 45 |
NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) |
| 46 |
|
| 47 |
// ResourcePrefix returns the overridden resource prefix for the GroupResource |
| 48 |
// This allows for cohabitation of resources with different native types and provides |
| 49 |
// centralized control over the shape of etcd directories |
| 50 |
ResourcePrefix(groupResource schema.GroupResource) string |
| 51 |
|
| 52 |
// Configs gets configurations for all of registered storage destinations. |
| 53 |
Configs() []storagebackend.Config |
| 54 |
|
| 55 |
// Backends gets all backends for all registered storage destinations. |
| 56 |
// Used for getting all instances for health validations. |
| 57 |
// Deprecated: Use Configs instead |
| 58 |
Backends() []Backend |
| 59 |
} |
| 60 |
|
| 61 |
// DefaultStorageFactory takes a GroupResource and returns back its storage interface. This result includes: |
| 62 |
// 1. Merged etcd config, including: auth, server locations, prefixes |
| 63 |
// 2. Resource encodings for storage: group,version,kind to store as |
| 64 |
// 3. Cohabitating default: some resources like hpa are exposed through multiple APIs. They must agree on 1 and 2 |
| 65 |
type DefaultStorageFactory struct { |
| 66 |
// StorageConfig describes how to create a storage backend in general. |
| 67 |
// Its authentication information will be used for every storage.Interface returned. |
| 68 |
StorageConfig storagebackend.Config |
| 69 |
|
| 70 |
Overrides map[schema.GroupResource]groupResourceOverrides |
| 71 |
|
| 72 |
DefaultResourcePrefixes map[schema.GroupResource]string |
| 73 |
|
| 74 |
// DefaultMediaType is the media type used to store resources. If it is not set, "application/json" is used. |
| 75 |
DefaultMediaType string |
| 76 |
|
| 77 |
// DefaultSerializer is used to create encoders and decoders for the storage.Interface. |
| 78 |
DefaultSerializer runtime.StorageSerializer |
| 79 |
|
| 80 |
// ResourceEncodingConfig describes how to encode a particular GroupVersionResource |
| 81 |
ResourceEncodingConfig ResourceEncodingConfig |
| 82 |
|
| 83 |
// APIResourceConfigSource indicates whether the *storage* is enabled, NOT the API |
| 84 |
// This is discrete from resource enablement because those are separate concerns. How this source is configured |
| 85 |
// is left to the caller. |
| 86 |
APIResourceConfigSource APIResourceConfigSource |
| 87 |
|
| 88 |
// newStorageCodecFn exists to be overwritten for unit testing. |
| 89 |
newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error) |
| 90 |
} |
| 91 |
|
| 92 |
type groupResourceOverrides struct { |
| 93 |
// etcdLocation contains the list of "special" locations that are used for particular GroupResources |
| 94 |
// These are merged on top of the StorageConfig when requesting the storage.Interface for a given GroupResource |
| 95 |
etcdLocation []string |
| 96 |
// etcdPrefix is the base location for a GroupResource. |
| 97 |
etcdPrefix string |
| 98 |
// etcdResourcePrefix is the location to use to store a particular type under the `etcdPrefix` location |
| 99 |
// If empty, the default mapping is used. If the default mapping doesn't contain an entry, it will use |
| 100 |
// the ToLowered name of the resource, not including the group. |
| 101 |
etcdResourcePrefix string |
| 102 |
// mediaType is the desired serializer to choose. If empty, the default is chosen. |
| 103 |
mediaType string |
| 104 |
// serializer contains the list of "special" serializers for a GroupResource. Resource=* means for the entire group |
| 105 |
serializer runtime.StorageSerializer |
| 106 |
// cohabitatingResources keeps track of which resources must be stored together. This happens when we have multiple ways |
| 107 |
// of exposing one set of concepts. autoscaling.HPA and extensions.HPA as a for instance |
| 108 |
// The order of the slice matters! It is the priority order of lookup for finding a storage location |
| 109 |
cohabitatingResources []schema.GroupResource |
| 110 |
// encoderDecoratorFn is optional and may wrap the provided encoder prior to being serialized. |
| 111 |
encoderDecoratorFn func(runtime.Encoder) runtime.Encoder |
| 112 |
// decoderDecoratorFn is optional and may wrap the provided decoders (can add new decoders). The order of |
| 113 |
// returned decoders will be priority for attempt to decode. |
| 114 |
decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder |
| 115 |
} |
| 116 |
|
| 117 |
// Apply overrides the provided config and options if the override has a value in that position |
| 118 |
func (o groupResourceOverrides) Apply(config *storagebackend.Config, options *StorageCodecConfig) { |
| 119 |
if len(o.etcdLocation) > 0 { |
| 120 |
config.Transport.ServerList = o.etcdLocation |
| 121 |
} |
| 122 |
if len(o.etcdPrefix) > 0 { |
| 123 |
config.Prefix = o.etcdPrefix |
| 124 |
} |
| 125 |
|
| 126 |
if len(o.mediaType) > 0 { |
| 127 |
options.StorageMediaType = o.mediaType |
| 128 |
} |
| 129 |
if o.serializer != nil { |
| 130 |
options.StorageSerializer = o.serializer |
| 131 |
} |
| 132 |
if o.encoderDecoratorFn != nil { |
| 133 |
options.EncoderDecoratorFn = o.encoderDecoratorFn |
| 134 |
} |
| 135 |
if o.decoderDecoratorFn != nil { |
| 136 |
options.DecoderDecoratorFn = o.decoderDecoratorFn |
| 137 |
} |
| 138 |
} |
| 139 |
|
| 140 |
var _ StorageFactory = &DefaultStorageFactory{} |
| 141 |
|
| 142 |
const AllResources = "*" |
| 143 |
|
| 144 |
func NewDefaultStorageFactory( |
| 145 |
config storagebackend.Config, |
| 146 |
defaultMediaType string, |
| 147 |
defaultSerializer runtime.StorageSerializer, |
| 148 |
resourceEncodingConfig ResourceEncodingConfig, |
| 149 |
resourceConfig APIResourceConfigSource, |
| 150 |
specialDefaultResourcePrefixes map[schema.GroupResource]string, |
| 151 |
) *DefaultStorageFactory { |
| 152 |
if len(defaultMediaType) == 0 { |
| 153 |
defaultMediaType = runtime.ContentTypeJSON |
| 154 |
} |
| 155 |
return &DefaultStorageFactory{ |
| 156 |
StorageConfig: config, |
| 157 |
Overrides: map[schema.GroupResource]groupResourceOverrides{}, |
| 158 |
DefaultMediaType: defaultMediaType, |
| 159 |
DefaultSerializer: defaultSerializer, |
| 160 |
ResourceEncodingConfig: resourceEncodingConfig, |
| 161 |
APIResourceConfigSource: resourceConfig, |
| 162 |
DefaultResourcePrefixes: specialDefaultResourcePrefixes, |
| 163 |
|
| 164 |
newStorageCodecFn: NewStorageCodec, |
| 165 |
} |
| 166 |
} |
| 167 |
|
| 168 |
func (s *DefaultStorageFactory) SetEtcdLocation(groupResource schema.GroupResource, location []string) { |
| 169 |
overrides := s.Overrides[groupResource] |
| 170 |
overrides.etcdLocation = location |
| 171 |
s.Overrides[groupResource] = overrides |
| 172 |
} |
| 173 |
|
| 174 |
func (s *DefaultStorageFactory) SetEtcdPrefix(groupResource schema.GroupResource, prefix string) { |
| 175 |
overrides := s.Overrides[groupResource] |
| 176 |
overrides.etcdPrefix = prefix |
| 177 |
s.Overrides[groupResource] = overrides |
| 178 |
} |
| 179 |
|
| 180 |
// SetResourceEtcdPrefix sets the prefix for a resource, but not the base-dir. You'll end up in `etcdPrefix/resourceEtcdPrefix`. |
| 181 |
func (s *DefaultStorageFactory) SetResourceEtcdPrefix(groupResource schema.GroupResource, prefix string) { |
| 182 |
overrides := s.Overrides[groupResource] |
| 183 |
overrides.etcdResourcePrefix = prefix |
| 184 |
s.Overrides[groupResource] = overrides |
| 185 |
} |
| 186 |
|
| 187 |
func (s *DefaultStorageFactory) SetSerializer(groupResource schema.GroupResource, mediaType string, serializer runtime.StorageSerializer) { |
| 188 |
overrides := s.Overrides[groupResource] |
| 189 |
overrides.mediaType = mediaType |
| 190 |
overrides.serializer = serializer |
| 191 |
s.Overrides[groupResource] = overrides |
| 192 |
} |
| 193 |
|
| 194 |
// AddCohabitatingResources links resources together the order of the slice matters! its the priority order of lookup for finding a storage location |
| 195 |
func (s *DefaultStorageFactory) AddCohabitatingResources(groupResources ...schema.GroupResource) { |
| 196 |
for _, groupResource := range groupResources { |
| 197 |
overrides := s.Overrides[groupResource] |
| 198 |
overrides.cohabitatingResources = groupResources |
| 199 |
s.Overrides[groupResource] = overrides |
| 200 |
} |
| 201 |
} |
| 202 |
|
| 203 |
func (s *DefaultStorageFactory) AddSerializationChains(encoderDecoratorFn func(runtime.Encoder) runtime.Encoder, decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder, groupResources ...schema.GroupResource) { |
| 204 |
for _, groupResource := range groupResources { |
| 205 |
overrides := s.Overrides[groupResource] |
| 206 |
overrides.encoderDecoratorFn = encoderDecoratorFn |
| 207 |
overrides.decoderDecoratorFn = decoderDecoratorFn |
| 208 |
s.Overrides[groupResource] = overrides |
| 209 |
} |
| 210 |
} |
| 211 |
|
| 212 |
func getAllResourcesAlias(resource schema.GroupResource) schema.GroupResource { |
| 213 |
return schema.GroupResource{Group: resource.Group, Resource: AllResources} |
| 214 |
} |
| 215 |
|
| 216 |
func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.GroupResource) schema.GroupResource { |
| 217 |
for _, potentialStorageResource := range s.Overrides[groupResource].cohabitatingResources { |
| 218 |
// TODO deads2k or liggitt determine if have ever stored any of our cohabitating resources in a different location on new clusters |
| 219 |
if s.APIResourceConfigSource.AnyResourceForGroupEnabled(potentialStorageResource.Group) { |
| 220 |
return potentialStorageResource |
| 221 |
} |
| 222 |
} |
| 223 |
|
| 224 |
return groupResource |
| 225 |
} |
| 226 |
|
| 227 |
// New finds the storage destination for the given group and resource. It will |
| 228 |
// return an error if the group has no storage destination configured. |
| 229 |
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { |
| 230 |
chosenStorageResource := s.getStorageGroupResource(groupResource) |
| 231 |
|
| 232 |
// operate on copy |
| 233 |
storageConfig := s.StorageConfig |
| 234 |
codecConfig := StorageCodecConfig{ |
| 235 |
StorageMediaType: s.DefaultMediaType, |
| 236 |
StorageSerializer: s.DefaultSerializer, |
| 237 |
} |
| 238 |
|
| 239 |
if override, ok := s.Overrides[getAllResourcesAlias(chosenStorageResource)]; ok { |
| 240 |
override.Apply(&storageConfig, &codecConfig) |
| 241 |
} |
| 242 |
if override, ok := s.Overrides[chosenStorageResource]; ok { |
| 243 |
override.Apply(&storageConfig, &codecConfig) |
| 244 |
} |
| 245 |
|
| 246 |
var err error |
| 247 |
if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok { |
| 248 |
codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(chosenStorageResource, example) |
| 249 |
if err != nil { |
| 250 |
return nil, err |
| 251 |
} |
| 252 |
} else { |
| 253 |
codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource) |
| 254 |
if err != nil { |
| 255 |
return nil, err |
| 256 |
} |
| 257 |
} |
| 258 |
|
| 259 |
codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource) |
| 260 |
if err != nil { |
| 261 |
return nil, err |
| 262 |
} |
| 263 |
|
| 264 |
codecConfig.Config = storageConfig |
| 265 |
|
| 266 |
storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig) |
| 267 |
if err != nil { |
| 268 |
return nil, err |
| 269 |
} |
| 270 |
klog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config) |
| 271 |
|
| 272 |
return storageConfig.ForResource(groupResource), nil |
| 273 |
} |
| 274 |
|
| 275 |
// Configs implements StorageFactory. |
| 276 |
func (s *DefaultStorageFactory) Configs() []storagebackend.Config { |
| 277 |
return configs(s.StorageConfig, s.Overrides) |
| 278 |
} |
| 279 |
|
| 280 |
// Configs gets configurations for all of registered storage destinations. |
| 281 |
func Configs(storageConfig storagebackend.Config) []storagebackend.Config { |
| 282 |
return configs(storageConfig, nil) |
| 283 |
} |
| 284 |
|
| 285 |
// Returns all storage configurations including those for group resource overrides |
| 286 |
func configs(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []storagebackend.Config { |
| 287 |
configs := []storagebackend.Config{storageConfig} |
| 288 |
|
| 289 |
for _, override := range grOverrides { |
| 290 |
if len(override.etcdLocation) == 0 { |
| 291 |
continue |
| 292 |
} |
| 293 |
// copy |
| 294 |
newConfig := storageConfig |
| 295 |
override.Apply(&newConfig, &StorageCodecConfig{}) |
| 296 |
newConfig.Transport.ServerList = override.etcdLocation |
| 297 |
configs = append(configs, newConfig) |
| 298 |
} |
| 299 |
return configs |
| 300 |
} |
| 301 |
|
| 302 |
// Backends implements StorageFactory. |
| 303 |
func (s *DefaultStorageFactory) Backends() []Backend { |
| 304 |
return backends(s.StorageConfig, s.Overrides) |
| 305 |
} |
| 306 |
|
| 307 |
// Backends returns all backends for all registered storage destinations. |
| 308 |
// Used for getting all instances for health validations. |
| 309 |
// Deprecated: Validate health by passing storagebackend.Config directly to storagefactory.CreateProber. |
| 310 |
func Backends(storageConfig storagebackend.Config) []Backend { |
| 311 |
return backends(storageConfig, nil) |
| 312 |
} |
| 313 |
|
| 314 |
func backends(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []Backend { |
| 315 |
servers := sets.NewString(storageConfig.Transport.ServerList...) |
| 316 |
|
| 317 |
for _, overrides := range grOverrides { |
| 318 |
servers.Insert(overrides.etcdLocation...) |
| 319 |
} |
| 320 |
|
| 321 |
tlsConfig := &tls.Config{ |
| 322 |
InsecureSkipVerify: true, |
| 323 |
} |
| 324 |
if len(storageConfig.Transport.CertFile) > 0 && len(storageConfig.Transport.KeyFile) > 0 { |
| 325 |
cert, err := tls.LoadX509KeyPair(storageConfig.Transport.CertFile, storageConfig.Transport.KeyFile) |
| 326 |
if err != nil { |
| 327 |
klog.Errorf("failed to load key pair while getting backends: %s", err) |
| 328 |
} else { |
| 329 |
tlsConfig.Certificates = []tls.Certificate{cert} |
| 330 |
} |
| 331 |
} |
| 332 |
if len(storageConfig.Transport.TrustedCAFile) > 0 { |
| 333 |
if caCert, err := os.ReadFile(storageConfig.Transport.TrustedCAFile); err != nil { |
| 334 |
klog.Errorf("failed to read ca file while getting backends: %s", err) |
| 335 |
} else { |
| 336 |
caPool := x509.NewCertPool() |
| 337 |
caPool.AppendCertsFromPEM(caCert) |
| 338 |
tlsConfig.RootCAs = caPool |
| 339 |
tlsConfig.InsecureSkipVerify = false |
| 340 |
} |
| 341 |
} |
| 342 |
|
| 343 |
backends := []Backend{} |
| 344 |
for server := range servers { |
| 345 |
backends = append(backends, Backend{ |
| 346 |
Server: server, |
| 347 |
// We can't share TLSConfig across different backends to avoid races. |
| 348 |
// For more details see: https://pr.k8s.io/59338 |
| 349 |
TLSConfig: tlsConfig.Clone(), |
| 350 |
}) |
| 351 |
} |
| 352 |
return backends |
| 353 |
} |
| 354 |
|
| 355 |
func (s *DefaultStorageFactory) ResourcePrefix(groupResource schema.GroupResource) string { |
| 356 |
chosenStorageResource := s.getStorageGroupResource(groupResource) |
| 357 |
groupOverride := s.Overrides[getAllResourcesAlias(chosenStorageResource)] |
| 358 |
exactResourceOverride := s.Overrides[chosenStorageResource] |
| 359 |
|
| 360 |
etcdResourcePrefix := s.DefaultResourcePrefixes[chosenStorageResource] |
| 361 |
if len(groupOverride.etcdResourcePrefix) > 0 { |
| 362 |
etcdResourcePrefix = groupOverride.etcdResourcePrefix |
| 363 |
} |
| 364 |
if len(exactResourceOverride.etcdResourcePrefix) > 0 { |
| 365 |
etcdResourcePrefix = exactResourceOverride.etcdResourcePrefix |
| 366 |
} |
| 367 |
if len(etcdResourcePrefix) == 0 { |
| 368 |
etcdResourcePrefix = strings.ToLower(chosenStorageResource.Resource) |
| 369 |
} |
| 370 |
|
| 371 |
return etcdResourcePrefix |
| 372 |
} |