Debian Codegraph @

pkg/reconcilers/peer_endpoint_lease.go

raw
1 /*
2 Copyright 2023 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package reconcilers
18
19 import (
20 "context"
21 "fmt"
22 "net"
23 "net/http"
24 "path"
25 "strconv"
26 "sync"
27 "sync/atomic"
28 "time"
29
30 "k8s.io/client-go/kubernetes"
31 "k8s.io/klog/v2"
32
33 corev1 "k8s.io/api/core/v1"
34 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
35 kruntime "k8s.io/apimachinery/pkg/runtime"
36 "k8s.io/apimachinery/pkg/util/runtime"
37 "k8s.io/apimachinery/pkg/util/wait"
38 apirequest "k8s.io/apiserver/pkg/endpoints/request"
39 "k8s.io/apiserver/pkg/registry/rest"
40 "k8s.io/apiserver/pkg/storage"
41 "k8s.io/apiserver/pkg/storage/storagebackend"
42 storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
43 )
44
45 const (
46 APIServerIdentityLabel = "apiserverIdentity"
47 )
48
49 type PeerAdvertiseAddress struct {
50 PeerAdvertiseIP string
51 PeerAdvertisePort string
52 }
53
54 type peerEndpointLeases struct {
55 storage storage.Interface
56 destroyFn func()
57 baseKey string
58 leaseTime time.Duration
59 }
60
61 type PeerEndpointLeaseReconciler interface {
62 // GetEndpoint retrieves the endpoint for a given apiserverId
63 GetEndpoint(serverId string) (string, error)
64 // UpdateLease updates the ip and port of peer servers
65 UpdateLease(serverId string, ip string, endpointPorts []corev1.EndpointPort) error
66 // RemoveEndpoints removes this apiserver's peer endpoint lease.
67 RemoveLease(serverId string) error
68 // Destroy cleans up everything on shutdown.
69 Destroy()
70 // StopReconciling turns any later ReconcileEndpoints call into a noop.
71 StopReconciling()
72 }
73
74 type peerEndpointLeaseReconciler struct {
75 serverLeases *peerEndpointLeases
76 stopReconcilingCalled atomic.Bool
77 }
78
79 // NewPeerEndpointLeaseReconciler creates a new peer endpoint lease reconciler
80 func NewPeerEndpointLeaseReconciler(config *storagebackend.ConfigForResource, baseKey string, leaseTime time.Duration) (PeerEndpointLeaseReconciler, error) {
81 // note that newFunc, newListFunc and resourcePrefix
82 // can be left blank unless the storage.Watch method is used
83 leaseStorage, destroyFn, err := storagefactory.Create(*config, nil, nil, "")
84 if err != nil {
85 return nil, fmt.Errorf("error creating storage factory: %v", err)
86 }
87 var once sync.Once
88 return &peerEndpointLeaseReconciler{
89 serverLeases: &peerEndpointLeases{
90 storage: leaseStorage,
91 destroyFn: func() { once.Do(destroyFn) },
92 baseKey: baseKey,
93 leaseTime: leaseTime,
94 },
95 }, nil
96 }
97
98 // PeerEndpointController is the controller manager for updating the peer endpoint leases.
99 // This provides a separate independent reconciliation loop for peer endpoint leases
100 // which ensures that the peer kube-apiservers are fetching the updated endpoint info for a given apiserver
101 // in the case when the peer wants to proxy the request to the given apiserver because it can not serve the
102 // request itself due to version mismatch.
103 type PeerEndpointLeaseController struct {
104 reconciler PeerEndpointLeaseReconciler
105 endpointInterval time.Duration
106 serverId string
107 // peeraddress stores the IP and port of this kube-apiserver. Used by peer kube-apiservers to
108 // route request to this apiserver in case of a version skew.
109 peeraddress string
110
111 client kubernetes.Interface
112
113 lock sync.Mutex
114 stopCh chan struct{} // closed by Stop()
115 }
116
117 func New(serverId string, peeraddress string,
118 reconciler PeerEndpointLeaseReconciler, endpointInterval time.Duration, client kubernetes.Interface) *PeerEndpointLeaseController {
119 return &PeerEndpointLeaseController{
120 reconciler: reconciler,
121 serverId: serverId,
122 // peeraddress stores the IP and port of this kube-apiserver. Used by peer kube-apiservers to
123 // route request to this apiserver in case of a version skew.
124 peeraddress: peeraddress,
125 endpointInterval: endpointInterval,
126 client: client,
127 stopCh: make(chan struct{}),
128 }
129 }
130
131 // Start begins the peer endpoint lease reconciler loop that must exist for bootstrapping
132 // a cluster.
133 func (c *PeerEndpointLeaseController) Start(stopCh <-chan struct{}) {
134 localStopCh := make(chan struct{})
135 go func() {
136 defer close(localStopCh)
137 select {
138 case <-stopCh: // from Start
139 case <-c.stopCh: // from Stop
140 }
141 }()
142 go c.Run(localStopCh)
143 }
144
145 // RunPeerEndpointReconciler periodically updates the peer endpoint leases
146 func (c *PeerEndpointLeaseController) Run(stopCh <-chan struct{}) {
147 // wait until process is ready
148 wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) {
149 var code int
150 c.client.CoreV1().RESTClient().Get().AbsPath("/readyz").Do(context.TODO()).StatusCode(&code)
151 return code == http.StatusOK, nil
152 }, stopCh)
153
154 wait.NonSlidingUntil(func() {
155 if err := c.UpdatePeerEndpointLeases(); err != nil {
156 runtime.HandleError(fmt.Errorf("unable to update peer endpoint leases: %v", err))
157 }
158 }, c.endpointInterval, stopCh)
159 }
160
161 // Stop cleans up this apiserver's peer endpoint leases.
162 func (c *PeerEndpointLeaseController) Stop() {
163 c.lock.Lock()
164 defer c.lock.Unlock()
165
166 select {
167 case <-c.stopCh:
168 return // only close once
169 default:
170 close(c.stopCh)
171 }
172 finishedReconciling := make(chan struct{})
173 go func() {
174 defer close(finishedReconciling)
175 klog.Infof("Shutting down peer endpoint lease reconciler")
176 // stop reconciliation
177 c.reconciler.StopReconciling()
178
179 // Ensure that there will be no race condition with the ReconcileEndpointLeases.
180 if err := c.reconciler.RemoveLease(c.serverId); err != nil {
181 klog.Errorf("Unable to remove peer endpoint leases: %v", err)
182 }
183 c.reconciler.Destroy()
184 }()
185
186 select {
187 case <-finishedReconciling:
188 // done
189 case <-time.After(2 * c.endpointInterval):
190 // don't block server shutdown forever if we can't reach etcd to remove ourselves
191 klog.Warning("peer_endpoint_controller's RemoveEndpoints() timed out")
192 }
193 }
194
195 // UpdatePeerEndpointLeases attempts to update the peer endpoint leases.
196 func (c *PeerEndpointLeaseController) UpdatePeerEndpointLeases() error {
197 host, port, err := net.SplitHostPort(c.peeraddress)
198 if err != nil {
199 return err
200 }
201
202 p, err := strconv.Atoi(port)
203 if err != nil {
204 return err
205 }
206 endpointPorts := createEndpointPortSpec(p, "https")
207
208 // Ensure that there will be no race condition with the RemoveEndpointLeases.
209 c.lock.Lock()
210 defer c.lock.Unlock()
211
212 // Refresh the TTL on our key, independently of whether any error or
213 // update conflict happens below. This makes sure that at least some of
214 // the servers will add our endpoint lease.
215 if err := c.reconciler.UpdateLease(c.serverId, host, endpointPorts); err != nil {
216 return err
217 }
218 return nil
219 }
220
221 // UpdateLease resets the TTL on a server IP in storage
222 // UpdateLease will create a new key if it doesn't exist.
223 // We use the first element in endpointPorts as a part of the lease's base key
224 // This is done to support out tests that simulate 2 apiservers running on the same ip but
225 // different ports
226
227 // It will also do the following if UnknownVersionInteroperabilityProxy feature is enabled
228 // 1. store the apiserverId as a label
229 // 2. store the values passed to --peer-advertise-ip and --peer-advertise-port flags to kube-apiserver as an annotation
230 // with value of format <ip:port>
231 func (r *peerEndpointLeaseReconciler) UpdateLease(serverId string, ip string, endpointPorts []corev1.EndpointPort) error {
232 // reconcile endpoints only if apiserver was not shutdown
233 if r.stopReconcilingCalled.Load() {
234 return nil
235 }
236
237 // we use the serverID as the key to avoid using the server IP, port as the key.
238 // note: this means that this lease doesn't enforce mutual exclusion of ip/port usage between apiserver.
239 key := path.Join(r.serverLeases.baseKey, serverId)
240 return r.serverLeases.storage.GuaranteedUpdate(apirequest.NewDefaultContext(), key, &corev1.Endpoints{}, true, nil, func(input kruntime.Object, respMeta storage.ResponseMeta) (kruntime.Object, *uint64, error) {
241 existing := input.(*corev1.Endpoints)
242 existing.Subsets = []corev1.EndpointSubset{
243 {
244 Addresses: []corev1.EndpointAddress{{IP: ip}},
245 Ports: endpointPorts,
246 },
247 }
248
249 // store this server's identity (serverId) as a label. This will be used by
250 // peers to find the IP of this server when the peer can not serve a request
251 // due to version skew.
252 if existing.Labels == nil {
253 existing.Labels = map[string]string{}
254 }
255 existing.Labels[APIServerIdentityLabel] = serverId
256
257 // leaseTime needs to be in seconds
258 leaseTime := uint64(r.serverLeases.leaseTime / time.Second)
259
260 // NB: GuaranteedUpdate does not perform the store operation unless
261 // something changed between load and store (not including resource
262 // version), meaning we can't refresh the TTL without actually
263 // changing a field.
264 existing.Generation++
265
266 klog.V(6).Infof("Resetting TTL on server IP %q listed in storage to %v", ip, leaseTime)
267 return existing, &leaseTime, nil
268 }, nil)
269 }
270
271 // ListLeases retrieves a list of the current server IPs from storage
272 func (r *peerEndpointLeaseReconciler) ListLeases() ([]string, error) {
273 storageOpts := storage.ListOptions{
274 ResourceVersion: "0",
275 ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan,
276 Predicate: storage.Everything,
277 Recursive: true,
278 }
279 ipInfoList, err := r.getIpInfoList(storageOpts)
280 if err != nil {
281 return nil, err
282 }
283 ipList := make([]string, 0, len(ipInfoList.Items))
284 for _, ip := range ipInfoList.Items {
285 if len(ip.Subsets) > 0 && len(ip.Subsets[0].Addresses) > 0 && len(ip.Subsets[0].Addresses[0].IP) > 0 {
286 ipList = append(ipList, ip.Subsets[0].Addresses[0].IP)
287 }
288 }
289 klog.V(6).Infof("Current server IPs listed in storage are %v", ipList)
290 return ipList, nil
291 }
292
293 // GetLease retrieves the server IP and port for a specific server id
294 func (r *peerEndpointLeaseReconciler) GetLease(serverId string) (string, error) {
295 var fullAddr string
296 if serverId == "" {
297 return "", fmt.Errorf("error getting endpoint for serverId: empty serverId")
298 }
299 storageOpts := storage.ListOptions{
300 ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan,
301 Predicate: storage.Everything,
302 Recursive: true,
303 }
304 ipInfoList, err := r.getIpInfoList(storageOpts)
305 if err != nil {
306 return "", err
307 }
308
309 for _, ip := range ipInfoList.Items {
310 if ip.Labels[APIServerIdentityLabel] == serverId {
311 if len(ip.Subsets) > 0 {
312 var ipStr, portStr string
313 if len(ip.Subsets[0].Addresses) > 0 {
314 if len(ip.Subsets[0].Addresses[0].IP) > 0 {
315 ipStr = ip.Subsets[0].Addresses[0].IP
316 }
317 }
318 if len(ip.Subsets[0].Ports) > 0 {
319 portStr = fmt.Sprint(ip.Subsets[0].Ports[0].Port)
320 }
321 fullAddr = net.JoinHostPort(ipStr, portStr)
322 break
323 }
324 }
325 }
326 klog.V(6).Infof("Fetched this server IP for the specified apiserverId %v, %v", serverId, fullAddr)
327 return fullAddr, nil
328 }
329
330 func (r *peerEndpointLeaseReconciler) StopReconciling() {
331 r.stopReconcilingCalled.Store(true)
332 }
333
334 // RemoveLease removes the lease on a server IP in storage
335 // We use the first element in endpointPorts as a part of the lease's base key
336 // This is done to support out tests that simulate 2 apiservers running on the same ip but
337 // different ports
338 func (r *peerEndpointLeaseReconciler) RemoveLease(serverId string) error {
339 key := path.Join(r.serverLeases.baseKey, serverId)
340 return r.serverLeases.storage.Delete(apirequest.NewDefaultContext(), key, &corev1.Endpoints{}, nil, rest.ValidateAllObjectFunc, nil, storage.DeleteOptions{})
341 }
342
343 func (r *peerEndpointLeaseReconciler) Destroy() {
344 r.serverLeases.destroyFn()
345 }
346
347 func (r *peerEndpointLeaseReconciler) GetEndpoint(serverId string) (string, error) {
348 return r.GetLease(serverId)
349 }
350
351 func (r *peerEndpointLeaseReconciler) getIpInfoList(storageOpts storage.ListOptions) (*corev1.EndpointsList, error) {
352 ipInfoList := &corev1.EndpointsList{}
353 if err := r.serverLeases.storage.GetList(apirequest.NewDefaultContext(), r.serverLeases.baseKey, storageOpts, ipInfoList); err != nil {
354 return nil, err
355 }
356 return ipInfoList, nil
357 }
358
359 // createEndpointPortSpec creates the endpoint ports
360 func createEndpointPortSpec(endpointPort int, endpointPortName string) []corev1.EndpointPort {
361 return []corev1.EndpointPort{{
362 Protocol: corev1.ProtocolTCP,
363 Port: int32(endpointPort),
364 Name: endpointPortName,
365 }}
366 }