Debian Codegraph @

pkg/storageversion/manager.go

raw
1 /*
2 Copyright 2020 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package storageversion
18
19 import (
20 "fmt"
21 "sort"
22 "sync"
23 "sync/atomic"
24
25 "k8s.io/apimachinery/pkg/runtime"
26 "k8s.io/apimachinery/pkg/runtime/schema"
27 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
28 "k8s.io/client-go/kubernetes"
29 "k8s.io/client-go/rest"
30 _ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration
31 "k8s.io/klog/v2"
32 )
33
34 // ResourceInfo contains the information to register the resource to the
35 // storage version API.
36 type ResourceInfo struct {
37 GroupResource schema.GroupResource
38
39 EncodingVersion string
40 // Used to calculate decodable versions. Can only be used after all
41 // equivalent versions are registered by InstallREST.
42 EquivalentResourceMapper runtime.EquivalentResourceRegistry
43
44 // DirectlyDecodableVersions is a list of versions that the converter for REST storage knows how to convert. This
45 // contains items like apiextensions.k8s.io/v1beta1 even if we don't serve that version.
46 DirectlyDecodableVersions []schema.GroupVersion
47
48 // ServedVersions holds a list of all versions of GroupResource that are served. Note that a server may be able to
49 // decode a particular version, but still not serve it.
50 ServedVersions []string
51 }
52
53 // Manager records the resources whose StorageVersions need updates, and provides a method to update those StorageVersions.
54 type Manager interface {
55 // AddResourceInfo records resources whose StorageVersions need updates
56 AddResourceInfo(resources ...*ResourceInfo)
57 // UpdateStorageVersions tries to update the StorageVersions of the recorded resources
58 UpdateStorageVersions(kubeAPIServerClientConfig *rest.Config, apiserverID string)
59 // PendingUpdate returns true if the StorageVersion of the given resource is still pending update.
60 PendingUpdate(gr schema.GroupResource) bool
61 // LastUpdateError returns the last error hit when updating the storage version of the given resource.
62 LastUpdateError(gr schema.GroupResource) error
63 // Completed returns true if updating StorageVersions of all recorded resources has completed.
64 Completed() bool
65 }
66
67 var _ Manager = &defaultManager{}
68
69 // defaultManager indicates if an apiserver has completed reporting its storage versions.
70 type defaultManager struct {
71 completed atomic.Bool
72
73 mu sync.RWMutex
74 // managedResourceInfos records the ResourceInfos whose StorageVersions will get updated in the next
75 // UpdateStorageVersions call
76 managedResourceInfos map[*ResourceInfo]struct{}
77 // managedStatus records the update status of StorageVersion for each GroupResource. Since one
78 // ResourceInfo may expand into multiple GroupResource (e.g. ingresses.networking.k8s.io and ingresses.extensions),
79 // this map allows quick status lookup for a GroupResource, during API request handling.
80 managedStatus map[schema.GroupResource]*updateStatus
81 }
82
83 type updateStatus struct {
84 done bool
85 lastErr error
86 }
87
88 // NewDefaultManager creates a new defaultManager.
89 func NewDefaultManager() Manager {
90 s := &defaultManager{}
91 s.completed.Store(false)
92 s.managedResourceInfos = make(map[*ResourceInfo]struct{})
93 s.managedStatus = make(map[schema.GroupResource]*updateStatus)
94 return s
95 }
96
97 // AddResourceInfo adds ResourceInfo to the manager.
98 func (s *defaultManager) AddResourceInfo(resources ...*ResourceInfo) {
99 s.mu.Lock()
100 defer s.mu.Unlock()
101 for _, r := range resources {
102 s.managedResourceInfos[r] = struct{}{}
103 s.addPendingManagedStatusLocked(r)
104 }
105 }
106
107 func (s *defaultManager) addPendingManagedStatusLocked(r *ResourceInfo) {
108 gvrs := r.EquivalentResourceMapper.EquivalentResourcesFor(r.GroupResource.WithVersion(""), "")
109 for _, gvr := range gvrs {
110 gr := gvr.GroupResource()
111 if _, ok := s.managedStatus[gr]; !ok {
112 s.managedStatus[gr] = &updateStatus{}
113 }
114 }
115 }
116
117 // UpdateStorageVersions tries to update the StorageVersions of the recorded resources
118 func (s *defaultManager) UpdateStorageVersions(kubeAPIServerClientConfig *rest.Config, serverID string) {
119 clientset, err := kubernetes.NewForConfig(kubeAPIServerClientConfig)
120 if err != nil {
121 utilruntime.HandleError(fmt.Errorf("failed to get clientset: %v", err))
122 return
123 }
124 sc := clientset.InternalV1alpha1().StorageVersions()
125
126 s.mu.RLock()
127 resources := []ResourceInfo{}
128 for resource := range s.managedResourceInfos {
129 resources = append(resources, *resource)
130 }
131 s.mu.RUnlock()
132 hasFailure := false
133 // Sorting the list to make sure we have a consistent dedup result, and
134 // therefore avoid creating unnecessarily duplicated StorageVersion objects.
135 // For example, extensions.ingresses and networking.k8s.io.ingresses share
136 // the same underlying storage. Without sorting, in an HA cluster, one
137 // apiserver may dedup and update StorageVersion for extensions.ingresses,
138 // while another apiserver may dedup and update StorageVersion for
139 // networking.k8s.io.ingresses. The storage migrator (which migrates objects
140 // per GroupResource) will migrate these resources twice, since both
141 // StorageVersion objects have CommonEncodingVersion (each with one server registered).
142 sortResourceInfosByGroupResource(resources)
143 for _, r := range dedupResourceInfos(resources) {
144 decodableVersions := decodableVersions(r.DirectlyDecodableVersions, r.EquivalentResourceMapper, r.GroupResource)
145 gr := r.GroupResource
146 // Group must be a valid subdomain in DNS (RFC 1123)
147 if len(gr.Group) == 0 {
148 gr.Group = "core"
149 }
150
151 servedVersions := r.ServedVersions
152
153 if err := updateStorageVersionFor(sc, serverID, gr, r.EncodingVersion, decodableVersions, servedVersions); err != nil {
154 utilruntime.HandleError(fmt.Errorf("failed to update storage version for %v: %v", r.GroupResource, err))
155 s.recordStatusFailure(&r, err)
156 hasFailure = true
157 continue
158 }
159 klog.V(2).Infof("successfully updated storage version for %v", r.GroupResource)
160 s.recordStatusSuccess(&r)
161 }
162 if hasFailure {
163 return
164 }
165 klog.V(2).Infof("storage version updates complete")
166 s.setComplete()
167 }
168
169 // dedupResourceInfos dedups ResourceInfos with the same underlying storage.
170 // ResourceInfos from the same Group with different Versions share the same underlying storage.
171 // ResourceInfos from different Groups may share the same underlying storage, e.g.
172 // networking.k8s.io ingresses and extensions ingresses. The StorageVersion manager
173 // only needs to update one StorageVersion for the equivalent Groups.
174 func dedupResourceInfos(infos []ResourceInfo) []ResourceInfo {
175 var ret []ResourceInfo
176 seen := make(map[schema.GroupResource]struct{})
177 for _, info := range infos {
178 gr := info.GroupResource
179 if _, ok := seen[gr]; ok {
180 continue
181 }
182 gvrs := info.EquivalentResourceMapper.EquivalentResourcesFor(gr.WithVersion(""), "")
183 for _, gvr := range gvrs {
184 seen[gvr.GroupResource()] = struct{}{}
185 }
186 ret = append(ret, info)
187 }
188 return ret
189 }
190
191 func sortResourceInfosByGroupResource(infos []ResourceInfo) {
192 sort.Sort(byGroupResource(infos))
193 }
194
195 type byGroupResource []ResourceInfo
196
197 func (s byGroupResource) Len() int { return len(s) }
198
199 func (s byGroupResource) Less(i, j int) bool {
200 if s[i].GroupResource.Group == s[j].GroupResource.Group {
201 return s[i].GroupResource.Resource < s[j].GroupResource.Resource
202 }
203 return s[i].GroupResource.Group < s[j].GroupResource.Group
204 }
205
206 func (s byGroupResource) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
207
208 // recordStatusSuccess marks updated ResourceInfo as completed.
209 func (s *defaultManager) recordStatusSuccess(r *ResourceInfo) {
210 s.mu.Lock()
211 defer s.mu.Unlock()
212 s.recordStatusSuccessLocked(r)
213 }
214
215 func (s *defaultManager) recordStatusSuccessLocked(r *ResourceInfo) {
216 gvrs := r.EquivalentResourceMapper.EquivalentResourcesFor(r.GroupResource.WithVersion(""), "")
217 for _, gvr := range gvrs {
218 s.recordSuccessGroupResourceLocked(gvr.GroupResource())
219 }
220 }
221
222 func (s *defaultManager) recordSuccessGroupResourceLocked(gr schema.GroupResource) {
223 if _, ok := s.managedStatus[gr]; !ok {
224 return
225 }
226 s.managedStatus[gr].done = true
227 s.managedStatus[gr].lastErr = nil
228 }
229
230 // recordStatusFailure records latest error updating ResourceInfo.
231 func (s *defaultManager) recordStatusFailure(r *ResourceInfo, err error) {
232 s.mu.Lock()
233 defer s.mu.Unlock()
234 s.recordStatusFailureLocked(r, err)
235 }
236
237 func (s *defaultManager) recordStatusFailureLocked(r *ResourceInfo, err error) {
238 gvrs := r.EquivalentResourceMapper.EquivalentResourcesFor(r.GroupResource.WithVersion(""), "")
239 for _, gvr := range gvrs {
240 s.recordErrorGroupResourceLocked(gvr.GroupResource(), err)
241 }
242 }
243
244 func (s *defaultManager) recordErrorGroupResourceLocked(gr schema.GroupResource, err error) {
245 if _, ok := s.managedStatus[gr]; !ok {
246 return
247 }
248 s.managedStatus[gr].lastErr = err
249 }
250
251 // PendingUpdate returns if the StorageVersion of a resource is still wait to be updated.
252 func (s *defaultManager) PendingUpdate(gr schema.GroupResource) bool {
253 s.mu.RLock()
254 defer s.mu.RUnlock()
255 if _, ok := s.managedStatus[gr]; !ok {
256 return false
257 }
258 return !s.managedStatus[gr].done
259 }
260
261 // LastUpdateError returns the last error hit when updating the storage version of the given resource.
262 func (s *defaultManager) LastUpdateError(gr schema.GroupResource) error {
263 s.mu.RLock()
264 defer s.mu.RUnlock()
265 if _, ok := s.managedStatus[gr]; !ok {
266 return fmt.Errorf("couldn't find managed status for %v", gr)
267 }
268 return s.managedStatus[gr].lastErr
269 }
270
271 // setComplete marks the completion of updating StorageVersions. No write requests need to be blocked anymore.
272 func (s *defaultManager) setComplete() {
273 s.completed.Store(true)
274 }
275
276 // Completed returns if updating StorageVersions has completed.
277 func (s *defaultManager) Completed() bool {
278 return s.completed.Load()
279 }
280
281 func decodableVersions(directlyDecodableVersions []schema.GroupVersion, e runtime.EquivalentResourceRegistry, gr schema.GroupResource) []string {
282 var versions []string
283 for _, decodableVersions := range directlyDecodableVersions {
284 versions = append(versions, decodableVersions.String())
285 }
286
287 decodingGVRs := e.EquivalentResourcesFor(gr.WithVersion(""), "")
288 for _, v := range decodingGVRs {
289 found := false
290 for _, existingVersion := range versions {
291 if existingVersion == v.GroupVersion().String() {
292 found = true
293 }
294 }
295 if found {
296 continue
297 }
298 versions = append(versions, v.GroupVersion().String())
299 }
300 return versions
301 }