Debian Codegraph @

pkg/storage/etcd3/corrupt_obj_deleter.go

raw
1 /*
2 Copyright 2024 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package etcd3
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "net/http"
24 "strings"
25
26 apierrors "k8s.io/apimachinery/pkg/api/errors"
27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28 "k8s.io/apimachinery/pkg/runtime"
29 "k8s.io/apimachinery/pkg/runtime/schema"
30 utilerrors "k8s.io/apimachinery/pkg/util/errors"
31 "k8s.io/apiserver/pkg/storage"
32 "k8s.io/apiserver/pkg/storage/value"
33 "k8s.io/klog/v2"
34 )
35
36 // NewStoreWithUnsafeCorruptObjectDeletion wraps the given store implementation
37 // and adds support for unsafe deletion of corrupt objects
38 func NewStoreWithUnsafeCorruptObjectDeletion(delegate storage.Interface, gr schema.GroupResource) storage.Interface {
39 return &corruptObjectDeleter{
40 Interface: delegate,
41 groupResource: gr,
42 }
43 }
44
45 // WithCorruptObjErrorHandlingDecoder decorates the given decoder, it determines
46 // if the error returned by the given decoder represents a corrupt object (the
47 // object is undecodable), and then it wraps the error appropriately so the
48 // unsafe deleter can determine if the object is a candidate for unsafe deletion
49 func WithCorruptObjErrorHandlingDecoder(decoder Decoder) Decoder {
50 return &corruptObjErrorInterpretingDecoder{Decoder: decoder}
51 }
52
53 // WithCorruptObjErrorHandlingTransformer decorates the given decoder, it
54 // determines if the error returned by the given transformer represents a
55 // corrupt object (the data from the storage is untransformable), and then it
56 // wraps the error appropriately so the unsafe deleter can determine
57 // if the object is a candidate for unsafe deletion
58 func WithCorruptObjErrorHandlingTransformer(transformer value.Transformer) value.Transformer {
59 return &corruptObjErrorInterpretingTransformer{Transformer: transformer}
60 }
61
62 // corruptObjErrAggregatorFactory returns an error aggregator that aggregates
63 // corrupt object error(s) that the list operation encounters while
64 // retrieving objects from the storage.
65 // maxCount: it is the maximum number of error that will be aggregated
66 func corruptObjErrAggregatorFactory(maxCount int) func() ListErrorAggregator {
67 if maxCount <= 0 {
68 return defaultListErrorAggregatorFactory
69 }
70 return func() ListErrorAggregator {
71 return &corruptObjErrAggregator{maxCount: maxCount}
72 }
73 }
74
75 var errTooMany = errors.New("too many errors, the list is truncated")
76
77 // aggregate corrupt object errors from the LIST operation
78 type corruptObjErrAggregator struct {
79 errs []error
80 abortErr error
81 maxCount int
82 }
83
84 func (a *corruptObjErrAggregator) Aggregate(key string, err error) bool {
85 if len(a.errs) >= a.maxCount {
86 // add a sentinel error to indicate there are more
87 a.errs = append(a.errs, errTooMany)
88 return true
89 }
90 var corruptObjErr *corruptObjectError
91 if errors.As(err, &corruptObjErr) {
92 a.errs = append(a.errs, storage.NewCorruptObjError(key, corruptObjErr))
93 return false
94 }
95
96 // not a corrupt object error, the list operation should abort
97 a.abortErr = err
98 return true
99 }
100
101 func (a *corruptObjErrAggregator) Err() error {
102 switch {
103 case len(a.errs) == 0 && a.abortErr != nil:
104 return a.abortErr
105 case len(a.errs) > 0:
106 err := utilerrors.NewAggregate(a.errs)
107 return &aggregatedStorageError{errs: err, resourcePrefix: "list"}
108 default:
109 return nil
110 }
111 }
112
113 // corruptObjectDeleter facilitates unsafe deletion of corrupt objects for etcd
114 type corruptObjectDeleter struct {
115 storage.Interface
116 groupResource schema.GroupResource
117 }
118
119 func (s *corruptObjectDeleter) Get(ctx context.Context, key string, opts storage.GetOptions, out runtime.Object) error {
120 if err := s.Interface.Get(ctx, key, opts, out); err != nil {
121 var corruptObjErr *corruptObjectError
122 if !errors.As(err, &corruptObjErr) {
123 // this error does not represent a corrupt object
124 return err
125 }
126 // the unsafe deleter at the registry layer will check whether
127 // the given err represents a corrupt object in order to
128 // initiate the unsafe deletion flow.
129 return storage.NewCorruptObjError(key, corruptObjErr)
130 }
131 return nil
132 }
133
134 func (s *corruptObjectDeleter) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
135 err := s.Interface.GetList(ctx, key, opts, listObj)
136 if err == nil {
137 return nil
138 }
139
140 var aggregatedErr *aggregatedStorageError
141 if errors.As(err, &aggregatedErr) {
142 // we have aggregated a list of corrupt objects
143 klog.V(5).ErrorS(aggregatedErr, "corrupt objects")
144 return aggregatedErr.NewAPIStatusError(s.groupResource)
145 }
146 return err
147 }
148
149 // corruptObjErrorInterpretingDecoder wraps the error returned by the decorated decoder
150 type corruptObjErrorInterpretingDecoder struct {
151 Decoder
152 }
153
154 func (d *corruptObjErrorInterpretingDecoder) Decode(value []byte, objPtr runtime.Object, rev int64) error {
155 // TODO: right now any error is deemed as undecodable, in
156 // the future, we can apply some filter, if need be.
157 if err := d.Decoder.Decode(value, objPtr, rev); err != nil {
158 return &corruptObjectError{err: err, errType: undecodable, revision: rev}
159 }
160 return nil
161 }
162
163 // decodeListItem decodes bytes value in array into object.
164 func (d *corruptObjErrorInterpretingDecoder) DecodeListItem(ctx context.Context, data []byte, rev uint64, newItemFunc func() runtime.Object) (runtime.Object, error) {
165 // TODO: right now any error is deemed as undecodable, in
166 // the future, we can apply some filter, if need be.
167 obj, err := d.Decoder.DecodeListItem(ctx, data, rev, newItemFunc)
168 if err != nil {
169 err = &corruptObjectError{err: err, errType: undecodable, revision: int64(rev)}
170 }
171 return obj, err
172 }
173
174 // corruptObjErrorInterpretingTransformer wraps the error returned by the transformer
175 type corruptObjErrorInterpretingTransformer struct {
176 value.Transformer
177 }
178
179 func (t *corruptObjErrorInterpretingTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
180 // TODO: right now any error is deemed as undecodable, in the future, we
181 // can apply some filter, if need be. For example, any network error
182 out, stale, err := t.Transformer.TransformFromStorage(ctx, data, dataCtx)
183 if err != nil {
184 err = &corruptObjectError{err: err, errType: untransformable}
185 }
186 return out, stale, err
187 }
188
189 // corruptObjectError is used internally, only by the corrupt object
190 // deleter, this error represents a corrup object:
191 // a) the data from the storage failed to transform, or
192 // b) the data failed to decode into an object
193 // NOTE: this error does not have any information to identify the object
194 // that is corrupt, for example the storage key associated with the object
195 type corruptObjectError struct {
196 err error
197 errType int
198 revision int64
199 }
200
201 const (
202 untransformable int = iota + 1
203 undecodable
204 )
205
206 var typeToMessage = map[int]string{
207 untransformable: "data from the storage is not transformable",
208 undecodable: "object not decodable",
209 }
210
211 func (e *corruptObjectError) Unwrap() error { return e.err }
212 func (e *corruptObjectError) Error() string {
213 return fmt.Sprintf("%s revision=%d: %v", typeToMessage[e.errType], e.revision, e.err)
214 }
215
216 // aggregatedStorageError holds an aggregated list of storage.StorageError
217 type aggregatedStorageError struct {
218 resourcePrefix string
219 errs utilerrors.Aggregate
220 }
221
222 func (e *aggregatedStorageError) Error() string {
223 errs := e.errs.Errors()
224 var b strings.Builder
225 fmt.Fprintf(&b, "unable to transform or decode %d objects: {\n", len(errs))
226 for _, err := range errs {
227 fmt.Fprintf(&b, "\t%s\n", err.Error())
228 }
229 b.WriteString("}")
230 return b.String()
231 }
232
233 // NewAPIStatusError creates a new APIStatus object from the
234 // aggregated list of StorageError
235 func (e *aggregatedStorageError) NewAPIStatusError(qualifiedResource schema.GroupResource) *apierrors.StatusError {
236 var causes []metav1.StatusCause
237 for _, err := range e.errs.Errors() {
238 var storageErr *storage.StorageError
239 if errors.As(err, &storageErr) {
240 causes = append(causes, metav1.StatusCause{
241 Type: metav1.CauseTypeUnexpectedServerResponse,
242 Field: storageErr.Key,
243 // TODO: do we need to expose the internal error message here?
244 Message: err.Error(),
245 })
246 continue
247 }
248 if errors.Is(err, errTooMany) {
249 causes = append(causes, metav1.StatusCause{
250 Type: metav1.CauseTypeTooMany,
251 Message: errTooMany.Error(),
252 })
253 }
254 }
255
256 return &apierrors.StatusError{
257 ErrStatus: metav1.Status{
258 Status: metav1.StatusFailure,
259 Code: http.StatusInternalServerError,
260 Reason: metav1.StatusReasonStoreReadError,
261 Details: &metav1.StatusDetails{
262 Group: qualifiedResource.Group,
263 Kind: qualifiedResource.Resource,
264 Name: e.resourcePrefix,
265 Causes: causes,
266 },
267 Message: fmt.Sprintf("failed to read one or more %s from the storage", qualifiedResource.String()),
268 },
269 }
270 }