| 1 |
/* |
| 2 |
Copyright 2024 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package etcd3 |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"errors" |
| 22 |
"fmt" |
| 23 |
"net/http" |
| 24 |
"strings" |
| 25 |
|
| 26 |
apierrors "k8s.io/apimachinery/pkg/api/errors" |
| 27 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 28 |
"k8s.io/apimachinery/pkg/runtime" |
| 29 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 30 |
utilerrors "k8s.io/apimachinery/pkg/util/errors" |
| 31 |
"k8s.io/apiserver/pkg/storage" |
| 32 |
"k8s.io/apiserver/pkg/storage/value" |
| 33 |
"k8s.io/klog/v2" |
| 34 |
) |
| 35 |
|
| 36 |
// NewStoreWithUnsafeCorruptObjectDeletion wraps the given store implementation |
| 37 |
// and adds support for unsafe deletion of corrupt objects |
| 38 |
func NewStoreWithUnsafeCorruptObjectDeletion(delegate storage.Interface, gr schema.GroupResource) storage.Interface { |
| 39 |
return &corruptObjectDeleter{ |
| 40 |
Interface: delegate, |
| 41 |
groupResource: gr, |
| 42 |
} |
| 43 |
} |
| 44 |
|
| 45 |
// WithCorruptObjErrorHandlingDecoder decorates the given decoder, it determines |
| 46 |
// if the error returned by the given decoder represents a corrupt object (the |
| 47 |
// object is undecodable), and then it wraps the error appropriately so the |
| 48 |
// unsafe deleter can determine if the object is a candidate for unsafe deletion |
| 49 |
func WithCorruptObjErrorHandlingDecoder(decoder Decoder) Decoder { |
| 50 |
return &corruptObjErrorInterpretingDecoder{Decoder: decoder} |
| 51 |
} |
| 52 |
|
| 53 |
// WithCorruptObjErrorHandlingTransformer decorates the given decoder, it |
| 54 |
// determines if the error returned by the given transformer represents a |
| 55 |
// corrupt object (the data from the storage is untransformable), and then it |
| 56 |
// wraps the error appropriately so the unsafe deleter can determine |
| 57 |
// if the object is a candidate for unsafe deletion |
| 58 |
func WithCorruptObjErrorHandlingTransformer(transformer value.Transformer) value.Transformer { |
| 59 |
return &corruptObjErrorInterpretingTransformer{Transformer: transformer} |
| 60 |
} |
| 61 |
|
| 62 |
// corruptObjErrAggregatorFactory returns an error aggregator that aggregates |
| 63 |
// corrupt object error(s) that the list operation encounters while |
| 64 |
// retrieving objects from the storage. |
| 65 |
// maxCount: it is the maximum number of error that will be aggregated |
| 66 |
func corruptObjErrAggregatorFactory(maxCount int) func() ListErrorAggregator { |
| 67 |
if maxCount <= 0 { |
| 68 |
return defaultListErrorAggregatorFactory |
| 69 |
} |
| 70 |
return func() ListErrorAggregator { |
| 71 |
return &corruptObjErrAggregator{maxCount: maxCount} |
| 72 |
} |
| 73 |
} |
| 74 |
|
| 75 |
var errTooMany = errors.New("too many errors, the list is truncated") |
| 76 |
|
| 77 |
// aggregate corrupt object errors from the LIST operation |
| 78 |
type corruptObjErrAggregator struct { |
| 79 |
errs []error |
| 80 |
abortErr error |
| 81 |
maxCount int |
| 82 |
} |
| 83 |
|
| 84 |
func (a *corruptObjErrAggregator) Aggregate(key string, err error) bool { |
| 85 |
if len(a.errs) >= a.maxCount { |
| 86 |
// add a sentinel error to indicate there are more |
| 87 |
a.errs = append(a.errs, errTooMany) |
| 88 |
return true |
| 89 |
} |
| 90 |
var corruptObjErr *corruptObjectError |
| 91 |
if errors.As(err, &corruptObjErr) { |
| 92 |
a.errs = append(a.errs, storage.NewCorruptObjError(key, corruptObjErr)) |
| 93 |
return false |
| 94 |
} |
| 95 |
|
| 96 |
// not a corrupt object error, the list operation should abort |
| 97 |
a.abortErr = err |
| 98 |
return true |
| 99 |
} |
| 100 |
|
| 101 |
func (a *corruptObjErrAggregator) Err() error { |
| 102 |
switch { |
| 103 |
case len(a.errs) == 0 && a.abortErr != nil: |
| 104 |
return a.abortErr |
| 105 |
case len(a.errs) > 0: |
| 106 |
err := utilerrors.NewAggregate(a.errs) |
| 107 |
return &aggregatedStorageError{errs: err, resourcePrefix: "list"} |
| 108 |
default: |
| 109 |
return nil |
| 110 |
} |
| 111 |
} |
| 112 |
|
| 113 |
// corruptObjectDeleter facilitates unsafe deletion of corrupt objects for etcd |
| 114 |
type corruptObjectDeleter struct { |
| 115 |
storage.Interface |
| 116 |
groupResource schema.GroupResource |
| 117 |
} |
| 118 |
|
| 119 |
func (s *corruptObjectDeleter) Get(ctx context.Context, key string, opts storage.GetOptions, out runtime.Object) error { |
| 120 |
if err := s.Interface.Get(ctx, key, opts, out); err != nil { |
| 121 |
var corruptObjErr *corruptObjectError |
| 122 |
if !errors.As(err, &corruptObjErr) { |
| 123 |
// this error does not represent a corrupt object |
| 124 |
return err |
| 125 |
} |
| 126 |
// the unsafe deleter at the registry layer will check whether |
| 127 |
// the given err represents a corrupt object in order to |
| 128 |
// initiate the unsafe deletion flow. |
| 129 |
return storage.NewCorruptObjError(key, corruptObjErr) |
| 130 |
} |
| 131 |
return nil |
| 132 |
} |
| 133 |
|
| 134 |
func (s *corruptObjectDeleter) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { |
| 135 |
err := s.Interface.GetList(ctx, key, opts, listObj) |
| 136 |
if err == nil { |
| 137 |
return nil |
| 138 |
} |
| 139 |
|
| 140 |
var aggregatedErr *aggregatedStorageError |
| 141 |
if errors.As(err, &aggregatedErr) { |
| 142 |
// we have aggregated a list of corrupt objects |
| 143 |
klog.V(5).ErrorS(aggregatedErr, "corrupt objects") |
| 144 |
return aggregatedErr.NewAPIStatusError(s.groupResource) |
| 145 |
} |
| 146 |
return err |
| 147 |
} |
| 148 |
|
| 149 |
// corruptObjErrorInterpretingDecoder wraps the error returned by the decorated decoder |
| 150 |
type corruptObjErrorInterpretingDecoder struct { |
| 151 |
Decoder |
| 152 |
} |
| 153 |
|
| 154 |
func (d *corruptObjErrorInterpretingDecoder) Decode(value []byte, objPtr runtime.Object, rev int64) error { |
| 155 |
// TODO: right now any error is deemed as undecodable, in |
| 156 |
// the future, we can apply some filter, if need be. |
| 157 |
if err := d.Decoder.Decode(value, objPtr, rev); err != nil { |
| 158 |
return &corruptObjectError{err: err, errType: undecodable, revision: rev} |
| 159 |
} |
| 160 |
return nil |
| 161 |
} |
| 162 |
|
| 163 |
// decodeListItem decodes bytes value in array into object. |
| 164 |
func (d *corruptObjErrorInterpretingDecoder) DecodeListItem(ctx context.Context, data []byte, rev uint64, newItemFunc func() runtime.Object) (runtime.Object, error) { |
| 165 |
// TODO: right now any error is deemed as undecodable, in |
| 166 |
// the future, we can apply some filter, if need be. |
| 167 |
obj, err := d.Decoder.DecodeListItem(ctx, data, rev, newItemFunc) |
| 168 |
if err != nil { |
| 169 |
err = &corruptObjectError{err: err, errType: undecodable, revision: int64(rev)} |
| 170 |
} |
| 171 |
return obj, err |
| 172 |
} |
| 173 |
|
| 174 |
// corruptObjErrorInterpretingTransformer wraps the error returned by the transformer |
| 175 |
type corruptObjErrorInterpretingTransformer struct { |
| 176 |
value.Transformer |
| 177 |
} |
| 178 |
|
| 179 |
func (t *corruptObjErrorInterpretingTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) { |
| 180 |
// TODO: right now any error is deemed as undecodable, in the future, we |
| 181 |
// can apply some filter, if need be. For example, any network error |
| 182 |
out, stale, err := t.Transformer.TransformFromStorage(ctx, data, dataCtx) |
| 183 |
if err != nil { |
| 184 |
err = &corruptObjectError{err: err, errType: untransformable} |
| 185 |
} |
| 186 |
return out, stale, err |
| 187 |
} |
| 188 |
|
| 189 |
// corruptObjectError is used internally, only by the corrupt object |
| 190 |
// deleter, this error represents a corrup object: |
| 191 |
// a) the data from the storage failed to transform, or |
| 192 |
// b) the data failed to decode into an object |
| 193 |
// NOTE: this error does not have any information to identify the object |
| 194 |
// that is corrupt, for example the storage key associated with the object |
| 195 |
type corruptObjectError struct { |
| 196 |
err error |
| 197 |
errType int |
| 198 |
revision int64 |
| 199 |
} |
| 200 |
|
| 201 |
const ( |
| 202 |
untransformable int = iota + 1 |
| 203 |
undecodable |
| 204 |
) |
| 205 |
|
| 206 |
var typeToMessage = map[int]string{ |
| 207 |
untransformable: "data from the storage is not transformable", |
| 208 |
undecodable: "object not decodable", |
| 209 |
} |
| 210 |
|
| 211 |
func (e *corruptObjectError) Unwrap() error { return e.err } |
| 212 |
func (e *corruptObjectError) Error() string { |
| 213 |
return fmt.Sprintf("%s revision=%d: %v", typeToMessage[e.errType], e.revision, e.err) |
| 214 |
} |
| 215 |
|
| 216 |
// aggregatedStorageError holds an aggregated list of storage.StorageError |
| 217 |
type aggregatedStorageError struct { |
| 218 |
resourcePrefix string |
| 219 |
errs utilerrors.Aggregate |
| 220 |
} |
| 221 |
|
| 222 |
func (e *aggregatedStorageError) Error() string { |
| 223 |
errs := e.errs.Errors() |
| 224 |
var b strings.Builder |
| 225 |
fmt.Fprintf(&b, "unable to transform or decode %d objects: {\n", len(errs)) |
| 226 |
for _, err := range errs { |
| 227 |
fmt.Fprintf(&b, "\t%s\n", err.Error()) |
| 228 |
} |
| 229 |
b.WriteString("}") |
| 230 |
return b.String() |
| 231 |
} |
| 232 |
|
| 233 |
// NewAPIStatusError creates a new APIStatus object from the |
| 234 |
// aggregated list of StorageError |
| 235 |
func (e *aggregatedStorageError) NewAPIStatusError(qualifiedResource schema.GroupResource) *apierrors.StatusError { |
| 236 |
var causes []metav1.StatusCause |
| 237 |
for _, err := range e.errs.Errors() { |
| 238 |
var storageErr *storage.StorageError |
| 239 |
if errors.As(err, &storageErr) { |
| 240 |
causes = append(causes, metav1.StatusCause{ |
| 241 |
Type: metav1.CauseTypeUnexpectedServerResponse, |
| 242 |
Field: storageErr.Key, |
| 243 |
// TODO: do we need to expose the internal error message here? |
| 244 |
Message: err.Error(), |
| 245 |
}) |
| 246 |
continue |
| 247 |
} |
| 248 |
if errors.Is(err, errTooMany) { |
| 249 |
causes = append(causes, metav1.StatusCause{ |
| 250 |
Type: metav1.CauseTypeTooMany, |
| 251 |
Message: errTooMany.Error(), |
| 252 |
}) |
| 253 |
} |
| 254 |
} |
| 255 |
|
| 256 |
return &apierrors.StatusError{ |
| 257 |
ErrStatus: metav1.Status{ |
| 258 |
Status: metav1.StatusFailure, |
| 259 |
Code: http.StatusInternalServerError, |
| 260 |
Reason: metav1.StatusReasonStoreReadError, |
| 261 |
Details: &metav1.StatusDetails{ |
| 262 |
Group: qualifiedResource.Group, |
| 263 |
Kind: qualifiedResource.Resource, |
| 264 |
Name: e.resourcePrefix, |
| 265 |
Causes: causes, |
| 266 |
}, |
| 267 |
Message: fmt.Sprintf("failed to read one or more %s from the storage", qualifiedResource.String()), |
| 268 |
}, |
| 269 |
} |
| 270 |
} |