| 1 |
/* |
| 2 |
Copyright 2015 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package storage |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
|
| 23 |
apierrors "k8s.io/apimachinery/pkg/api/errors" |
| 24 |
"k8s.io/apimachinery/pkg/api/meta" |
| 25 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 26 |
"k8s.io/apimachinery/pkg/fields" |
| 27 |
"k8s.io/apimachinery/pkg/labels" |
| 28 |
"k8s.io/apimachinery/pkg/runtime" |
| 29 |
"k8s.io/apimachinery/pkg/types" |
| 30 |
"k8s.io/apimachinery/pkg/watch" |
| 31 |
) |
| 32 |
|
| 33 |
// Feature is the name of each feature in storage that we check in feature_support_checker. |
| 34 |
type Feature = string |
| 35 |
|
| 36 |
// RequestWatchProgress is an etcd feature that may use to check if it supported or not. |
| 37 |
var RequestWatchProgress Feature = "RequestWatchProgress" |
| 38 |
|
| 39 |
// Versioner abstracts setting and retrieving metadata fields from database response |
| 40 |
// onto the object ot list. It is required to maintain storage invariants - updating an |
| 41 |
// object twice with the same data except for the ResourceVersion and SelfLink must be |
| 42 |
// a no-op. A resourceVersion of type uint64 is a 'raw' resourceVersion, |
| 43 |
// intended to be sent directly to or from the backend. A resourceVersion of |
| 44 |
// type string is a 'safe' resourceVersion, intended for consumption by users. |
| 45 |
type Versioner interface { |
| 46 |
// UpdateObject sets storage metadata into an API object. Returns an error if the object |
| 47 |
// cannot be updated correctly. May return nil if the requested object does not need metadata |
| 48 |
// from database. |
| 49 |
UpdateObject(obj runtime.Object, resourceVersion uint64) error |
| 50 |
// UpdateList sets the resource version into an API list object. Returns an error if the object |
| 51 |
// cannot be updated correctly. May return nil if the requested object does not need metadata from |
| 52 |
// database. continueValue is optional and indicates that more results are available if the client |
| 53 |
// passes that value to the server in a subsequent call. remainingItemCount indicates the number |
| 54 |
// of remaining objects if the list is partial. The remainingItemCount field is omitted during |
| 55 |
// serialization if it is set to nil. |
| 56 |
UpdateList(obj runtime.Object, resourceVersion uint64, continueValue string, remainingItemCount *int64) error |
| 57 |
// PrepareObjectForStorage should set SelfLink and ResourceVersion to the empty value. Should |
| 58 |
// return an error if the specified object cannot be updated. |
| 59 |
PrepareObjectForStorage(obj runtime.Object) error |
| 60 |
// ObjectResourceVersion returns the resource version (for persistence) of the specified object. |
| 61 |
// Should return an error if the specified object does not have a persistable version. |
| 62 |
ObjectResourceVersion(obj runtime.Object) (uint64, error) |
| 63 |
|
| 64 |
// ParseResourceVersion takes a resource version argument and |
| 65 |
// converts it to the storage backend. For watch we should pass to helper.Watch(). |
| 66 |
// Because resourceVersion is an opaque value, the default watch |
| 67 |
// behavior for non-zero watch is to watch the next value (if you pass |
| 68 |
// "1", you will see updates from "2" onwards). |
| 69 |
ParseResourceVersion(resourceVersion string) (uint64, error) |
| 70 |
} |
| 71 |
|
| 72 |
// ResponseMeta contains information about the database metadata that is associated with |
| 73 |
// an object. It abstracts the actual underlying objects to prevent coupling with concrete |
| 74 |
// database and to improve testability. |
| 75 |
type ResponseMeta struct { |
| 76 |
// TTL is the time to live of the node that contained the returned object. It may be |
| 77 |
// zero or negative in some cases (objects may be expired after the requested |
| 78 |
// expiration time due to server lag). |
| 79 |
TTL int64 |
| 80 |
// The resource version of the node that contained the returned object. |
| 81 |
ResourceVersion uint64 |
| 82 |
} |
| 83 |
|
| 84 |
// IndexerFunc is a function that for a given object computes |
| 85 |
// `<value of an index>` for a particular `<index>`. |
| 86 |
type IndexerFunc func(obj runtime.Object) string |
| 87 |
|
| 88 |
// IndexerFuncs is a mapping from `<index name>` to function that |
| 89 |
// for a given object computes `<value for that index>`. |
| 90 |
type IndexerFuncs map[string]IndexerFunc |
| 91 |
|
| 92 |
// Everything accepts all objects. |
| 93 |
var Everything = SelectionPredicate{ |
| 94 |
Label: labels.Everything(), |
| 95 |
Field: fields.Everything(), |
| 96 |
} |
| 97 |
|
| 98 |
// MatchValue defines a pair (`<index name>`, `<value for that index>`). |
| 99 |
type MatchValue struct { |
| 100 |
IndexName string |
| 101 |
Value string |
| 102 |
} |
| 103 |
|
| 104 |
// Pass an UpdateFunc to Interface.GuaranteedUpdate to make an update |
| 105 |
// that is guaranteed to succeed. |
| 106 |
// See the comment for GuaranteedUpdate for more details. |
| 107 |
type UpdateFunc func(input runtime.Object, res ResponseMeta) (output runtime.Object, ttl *uint64, err error) |
| 108 |
|
| 109 |
// ValidateObjectFunc is a function to act on a given object. An error may be returned |
| 110 |
// if the hook cannot be completed. The function may NOT transform the provided |
| 111 |
// object. |
| 112 |
// NOTE: the object in obj may be nil if it cannot be read from the |
| 113 |
// storage, due to transformation or decode error. |
| 114 |
type ValidateObjectFunc func(ctx context.Context, obj runtime.Object) error |
| 115 |
|
| 116 |
// ValidateAllObjectFunc is a "admit everything" instance of ValidateObjectFunc. |
| 117 |
func ValidateAllObjectFunc(ctx context.Context, obj runtime.Object) error { |
| 118 |
return nil |
| 119 |
} |
| 120 |
|
| 121 |
// Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out. |
| 122 |
type Preconditions struct { |
| 123 |
// Specifies the target UID. |
| 124 |
// +optional |
| 125 |
UID *types.UID `json:"uid,omitempty"` |
| 126 |
// Specifies the target ResourceVersion |
| 127 |
// +optional |
| 128 |
ResourceVersion *string `json:"resourceVersion,omitempty"` |
| 129 |
} |
| 130 |
|
| 131 |
// NewUIDPreconditions returns a Preconditions with UID set. |
| 132 |
func NewUIDPreconditions(uid string) *Preconditions { |
| 133 |
u := types.UID(uid) |
| 134 |
return &Preconditions{UID: &u} |
| 135 |
} |
| 136 |
|
| 137 |
func (p *Preconditions) Check(key string, obj runtime.Object) error { |
| 138 |
if p == nil { |
| 139 |
return nil |
| 140 |
} |
| 141 |
objMeta, err := meta.Accessor(obj) |
| 142 |
if err != nil { |
| 143 |
return NewInternalError( |
| 144 |
fmt.Errorf("can't enforce preconditions %v on un-introspectable object %v, got error: %w", |
| 145 |
*p, |
| 146 |
obj, |
| 147 |
err)) |
| 148 |
} |
| 149 |
if p.UID != nil && *p.UID != objMeta.GetUID() { |
| 150 |
err := fmt.Sprintf( |
| 151 |
"Precondition failed: UID in precondition: %v, UID in object meta: %v", |
| 152 |
*p.UID, |
| 153 |
objMeta.GetUID()) |
| 154 |
return NewInvalidObjError(key, err) |
| 155 |
} |
| 156 |
if p.ResourceVersion != nil && *p.ResourceVersion != objMeta.GetResourceVersion() { |
| 157 |
err := fmt.Sprintf( |
| 158 |
"Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v", |
| 159 |
*p.ResourceVersion, |
| 160 |
objMeta.GetResourceVersion()) |
| 161 |
return NewInvalidObjError(key, err) |
| 162 |
} |
| 163 |
return nil |
| 164 |
} |
| 165 |
|
| 166 |
// Interface offers a common interface for object marshaling/unmarshaling operations and |
| 167 |
// hides all the storage-related operations behind it. |
| 168 |
type Interface interface { |
| 169 |
// Returns Versioner associated with this interface. |
| 170 |
Versioner() Versioner |
| 171 |
|
| 172 |
// Create adds a new object at a key unless it already exists. 'ttl' is time-to-live |
| 173 |
// in seconds (0 means forever). If no error is returned and out is not nil, out will be |
| 174 |
// set to the read value from database. |
| 175 |
Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error |
| 176 |
|
| 177 |
// Delete removes the specified key and returns the value that existed at that spot. |
| 178 |
// If key didn't exist, it will return NotFound storage error. |
| 179 |
// If 'cachedExistingObject' is non-nil, it can be used as a suggestion about the |
| 180 |
// current version of the object to avoid read operation from storage to get it. |
| 181 |
// However, the implementations have to retry in case suggestion is stale. |
| 182 |
Delete( |
| 183 |
ctx context.Context, key string, out runtime.Object, preconditions *Preconditions, |
| 184 |
validateDeletion ValidateObjectFunc, cachedExistingObject runtime.Object, opts DeleteOptions) error |
| 185 |
|
| 186 |
// Watch begins watching the specified key. Events are decoded into API objects, |
| 187 |
// and any items selected by 'p' are sent down to returned watch.Interface. |
| 188 |
// resourceVersion may be used to specify what version to begin watching, |
| 189 |
// which should be the current resourceVersion, and no longer rv+1 |
| 190 |
// (e.g. reconnecting without missing any updates). |
| 191 |
// If resource version is "0", this interface will get current object at given key |
| 192 |
// and send it in an "ADDED" event, before watch starts. |
| 193 |
Watch(ctx context.Context, key string, opts ListOptions) (watch.Interface, error) |
| 194 |
|
| 195 |
// Get unmarshals object found at key into objPtr. On a not found error, will either |
| 196 |
// return a zero object of the requested type, or an error, depending on 'opts.ignoreNotFound'. |
| 197 |
// Treats empty responses and nil response nodes exactly like a not found error. |
| 198 |
// The returned contents may be delayed, but it is guaranteed that they will |
| 199 |
// match 'opts.ResourceVersion' according 'opts.ResourceVersionMatch'. |
| 200 |
Get(ctx context.Context, key string, opts GetOptions, objPtr runtime.Object) error |
| 201 |
|
| 202 |
// GetList unmarshalls objects found at key into a *List api object (an object |
| 203 |
// that satisfies runtime.IsList definition). |
| 204 |
// If 'opts.Recursive' is false, 'key' is used as an exact match. If `opts.Recursive' |
| 205 |
// is true, 'key' is used as a prefix. |
| 206 |
// The returned contents may be delayed, but it is guaranteed that they will |
| 207 |
// match 'opts.ResourceVersion' according 'opts.ResourceVersionMatch'. |
| 208 |
GetList(ctx context.Context, key string, opts ListOptions, listObj runtime.Object) error |
| 209 |
|
| 210 |
// GuaranteedUpdate keeps calling 'tryUpdate()' to update key 'key' (of type 'destination') |
| 211 |
// retrying the update until success if there is index conflict. |
| 212 |
// Note that object passed to tryUpdate may change across invocations of tryUpdate() if |
| 213 |
// other writers are simultaneously updating it, so tryUpdate() needs to take into account |
| 214 |
// the current contents of the object when deciding how the update object should look. |
| 215 |
// If the key doesn't exist, it will return NotFound storage error if ignoreNotFound=false |
| 216 |
// else `destination` will be set to the zero value of it's type. |
| 217 |
// If the eventual successful invocation of `tryUpdate` returns an output with the same serialized |
| 218 |
// contents as the input, it won't perform any update, but instead set `destination` to an object with those |
| 219 |
// contents. |
| 220 |
// If 'cachedExistingObject' is non-nil, it can be used as a suggestion about the |
| 221 |
// current version of the object to avoid read operation from storage to get it. |
| 222 |
// However, the implementations have to retry in case suggestion is stale. |
| 223 |
// |
| 224 |
// Example: |
| 225 |
// |
| 226 |
// s := /* implementation of Interface */ |
| 227 |
// err := s.GuaranteedUpdate( |
| 228 |
// "myKey", &MyType{}, true, preconditions, |
| 229 |
// func(input runtime.Object, res ResponseMeta) (runtime.Object, *uint64, error) { |
| 230 |
// // Before each invocation of the user defined function, "input" is reset to |
| 231 |
// // current contents for "myKey" in database. |
| 232 |
// curr := input.(*MyType) // Guaranteed to succeed. |
| 233 |
// |
| 234 |
// // Make the modification |
| 235 |
// curr.Counter++ |
| 236 |
// |
| 237 |
// // Return the modified object - return an error to stop iterating. Return |
| 238 |
// // a uint64 to alter the TTL on the object, or nil to keep it the same value. |
| 239 |
// return cur, nil, nil |
| 240 |
// }, cachedExistingObject |
| 241 |
// ) |
| 242 |
GuaranteedUpdate( |
| 243 |
ctx context.Context, key string, destination runtime.Object, ignoreNotFound bool, |
| 244 |
preconditions *Preconditions, tryUpdate UpdateFunc, cachedExistingObject runtime.Object) error |
| 245 |
|
| 246 |
// Count returns number of different entries under the key (generally being path prefix). |
| 247 |
Count(key string) (int64, error) |
| 248 |
|
| 249 |
// ReadinessCheck checks if the storage is ready for accepting requests. |
| 250 |
ReadinessCheck() error |
| 251 |
|
| 252 |
// RequestWatchProgress requests the a watch stream progress status be sent in the |
| 253 |
// watch response stream as soon as possible. |
| 254 |
// Used for monitor watch progress even if watching resources with no changes. |
| 255 |
// |
| 256 |
// If watch is lagging, progress status might: |
| 257 |
// * be pointing to stale resource version. Use etcd KV request to get linearizable resource version. |
| 258 |
// * not be delivered at all. It's recommended to poll request progress periodically. |
| 259 |
// |
| 260 |
// Note: Only watches with matching context grpc metadata will be notified. |
| 261 |
// https://github.com/kubernetes/kubernetes/blob/9325a57125e8502941d1b0c7379c4bb80a678d5c/vendor/go.etcd.io/etcd/client/v3/watch.go#L1037-L1042 |
| 262 |
// |
| 263 |
// TODO: Remove when storage.Interface will be separate from etc3.store. |
| 264 |
// Deprecated: Added temporarily to simplify exposing RequestProgress for watch cache. |
| 265 |
RequestWatchProgress(ctx context.Context) error |
| 266 |
|
| 267 |
// GetCurrentResourceVersion gets the current resource version from etcd. |
| 268 |
// This method issues an empty list request and reads only the ResourceVersion from the object metadata |
| 269 |
GetCurrentResourceVersion(ctx context.Context) (uint64, error) |
| 270 |
} |
| 271 |
|
| 272 |
// GetOptions provides the options that may be provided for storage get operations. |
| 273 |
type GetOptions struct { |
| 274 |
// IgnoreNotFound determines what is returned if the requested object is not found. If |
| 275 |
// true, a zero object is returned. If false, an error is returned. |
| 276 |
IgnoreNotFound bool |
| 277 |
// ResourceVersion provides a resource version constraint to apply to the get operation |
| 278 |
// as a "not older than" constraint: the result contains data at least as new as the provided |
| 279 |
// ResourceVersion. The newest available data is preferred, but any data not older than this |
| 280 |
// ResourceVersion may be served. |
| 281 |
ResourceVersion string |
| 282 |
} |
| 283 |
|
| 284 |
// ListOptions provides the options that may be provided for storage list operations. |
| 285 |
type ListOptions struct { |
| 286 |
// ResourceVersion provides a resource version constraint to apply to the list operation |
| 287 |
// as a "not older than" constraint: the result contains data at least as new as the provided |
| 288 |
// ResourceVersion. The newest available data is preferred, but any data not older than this |
| 289 |
// ResourceVersion may be served. |
| 290 |
ResourceVersion string |
| 291 |
// ResourceVersionMatch provides the rule for how the resource version constraint applies. If set |
| 292 |
// to the default value "" the legacy resource version semantic apply. |
| 293 |
ResourceVersionMatch metav1.ResourceVersionMatch |
| 294 |
// Predicate provides the selection rules for the list operation. |
| 295 |
Predicate SelectionPredicate |
| 296 |
// Recursive determines whether the list or watch is defined for a single object located at the |
| 297 |
// given key, or for the whole set of objects with the given key as a prefix. |
| 298 |
Recursive bool |
| 299 |
// ProgressNotify determines whether storage-originated bookmark (progress notify) events should |
| 300 |
// be delivered to the users. The option is ignored for non-watch requests. |
| 301 |
// |
| 302 |
// Firstly, note that this field is different from the Predicate.AllowWatchBookmarks field. |
| 303 |
// Secondly, this field is intended for internal clients only such as the watch cache. |
| 304 |
// |
| 305 |
// This means that external clients do not have the ability to set this field directly. |
| 306 |
// For example by setting the allowWatchBookmarks query parameter. |
| 307 |
// |
| 308 |
// The motivation for this approach is the fact that the frequency |
| 309 |
// of bookmark events from a storage like etcd might be very high. |
| 310 |
// As the number of watch requests increases, the server load would also increase. |
| 311 |
// |
| 312 |
// Furthermore, the server is not obligated to provide bookmark events at all, |
| 313 |
// as described in https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/956-watch-bookmark#proposal |
| 314 |
ProgressNotify bool |
| 315 |
// SendInitialEvents, when set together with Watch option, |
| 316 |
// begin the watch stream with synthetic init events to build the |
| 317 |
// whole state of all resources followed by a synthetic "Bookmark" |
| 318 |
// event containing a ResourceVersion after which the server |
| 319 |
// continues streaming events. |
| 320 |
SendInitialEvents *bool |
| 321 |
} |
| 322 |
|
| 323 |
// DeleteOptions provides the options that may be provided for storage delete operations. |
| 324 |
type DeleteOptions struct { |
| 325 |
// IgnoreStoreReadError, if enabled, will ignore store read error |
| 326 |
// such as transformation or decode failure and go ahead with the |
| 327 |
// deletion of the object. |
| 328 |
// NOTE: for normal deletion flow it should always be false, it may be |
| 329 |
// enabled by the caller only to facilitate unsafe deletion of corrupt |
| 330 |
// object which otherwise can not be deleted using the normal flow |
| 331 |
IgnoreStoreReadError bool |
| 332 |
} |
| 333 |
|
| 334 |
func ValidateListOptions(keyPrefix string, versioner Versioner, opts ListOptions) (withRev int64, continueKey string, err error) { |
| 335 |
if opts.Recursive && len(opts.Predicate.Continue) > 0 { |
| 336 |
continueKey, continueRV, err := DecodeContinue(opts.Predicate.Continue, keyPrefix) |
| 337 |
if err != nil { |
| 338 |
return 0, "", apierrors.NewBadRequest(fmt.Sprintf("invalid continue token: %v", err)) |
| 339 |
} |
| 340 |
if len(opts.ResourceVersion) > 0 && opts.ResourceVersion != "0" { |
| 341 |
return 0, "", apierrors.NewBadRequest("specifying resource version is not allowed when using continue") |
| 342 |
} |
| 343 |
// If continueRV > 0, the LIST request needs a specific resource version. |
| 344 |
// continueRV==0 is invalid. |
| 345 |
// If continueRV < 0, the request is for the latest resource version. |
| 346 |
if continueRV > 0 { |
| 347 |
withRev = continueRV |
| 348 |
} |
| 349 |
return withRev, continueKey, nil |
| 350 |
} |
| 351 |
if len(opts.ResourceVersion) == 0 { |
| 352 |
return withRev, "", nil |
| 353 |
} |
| 354 |
parsedRV, err := versioner.ParseResourceVersion(opts.ResourceVersion) |
| 355 |
if err != nil { |
| 356 |
return withRev, "", apierrors.NewBadRequest(fmt.Sprintf("invalid resource version: %v", err)) |
| 357 |
} |
| 358 |
switch opts.ResourceVersionMatch { |
| 359 |
case metav1.ResourceVersionMatchNotOlderThan: |
| 360 |
// The not older than constraint is checked after we get a response from etcd, |
| 361 |
// and returnedRV is then set to the revision we get from the etcd response. |
| 362 |
case metav1.ResourceVersionMatchExact: |
| 363 |
withRev = int64(parsedRV) |
| 364 |
case "": // legacy case |
| 365 |
if opts.Recursive && opts.Predicate.Limit > 0 && parsedRV > 0 { |
| 366 |
withRev = int64(parsedRV) |
| 367 |
} |
| 368 |
default: |
| 369 |
return withRev, "", fmt.Errorf("unknown ResourceVersionMatch value: %v", opts.ResourceVersionMatch) |
| 370 |
} |
| 371 |
return withRev, "", nil |
| 372 |
} |