| 1 |
/* |
| 2 |
Copyright 2023 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package reconcilers |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"net" |
| 23 |
"net/http" |
| 24 |
"path" |
| 25 |
"strconv" |
| 26 |
"sync" |
| 27 |
"sync/atomic" |
| 28 |
"time" |
| 29 |
|
| 30 |
"k8s.io/client-go/kubernetes" |
| 31 |
"k8s.io/klog/v2" |
| 32 |
|
| 33 |
corev1 "k8s.io/api/core/v1" |
| 34 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 35 |
kruntime "k8s.io/apimachinery/pkg/runtime" |
| 36 |
"k8s.io/apimachinery/pkg/util/runtime" |
| 37 |
"k8s.io/apimachinery/pkg/util/wait" |
| 38 |
apirequest "k8s.io/apiserver/pkg/endpoints/request" |
| 39 |
"k8s.io/apiserver/pkg/registry/rest" |
| 40 |
"k8s.io/apiserver/pkg/storage" |
| 41 |
"k8s.io/apiserver/pkg/storage/storagebackend" |
| 42 |
storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory" |
| 43 |
) |
| 44 |
|
| 45 |
const ( |
| 46 |
APIServerIdentityLabel = "apiserverIdentity" |
| 47 |
) |
| 48 |
|
| 49 |
type PeerAdvertiseAddress struct { |
| 50 |
PeerAdvertiseIP string |
| 51 |
PeerAdvertisePort string |
| 52 |
} |
| 53 |
|
| 54 |
type peerEndpointLeases struct { |
| 55 |
storage storage.Interface |
| 56 |
destroyFn func() |
| 57 |
baseKey string |
| 58 |
leaseTime time.Duration |
| 59 |
} |
| 60 |
|
| 61 |
type PeerEndpointLeaseReconciler interface { |
| 62 |
// GetEndpoint retrieves the endpoint for a given apiserverId |
| 63 |
GetEndpoint(serverId string) (string, error) |
| 64 |
// UpdateLease updates the ip and port of peer servers |
| 65 |
UpdateLease(serverId string, ip string, endpointPorts []corev1.EndpointPort) error |
| 66 |
// RemoveEndpoints removes this apiserver's peer endpoint lease. |
| 67 |
RemoveLease(serverId string) error |
| 68 |
// Destroy cleans up everything on shutdown. |
| 69 |
Destroy() |
| 70 |
// StopReconciling turns any later ReconcileEndpoints call into a noop. |
| 71 |
StopReconciling() |
| 72 |
} |
| 73 |
|
| 74 |
type peerEndpointLeaseReconciler struct { |
| 75 |
serverLeases *peerEndpointLeases |
| 76 |
stopReconcilingCalled atomic.Bool |
| 77 |
} |
| 78 |
|
| 79 |
// NewPeerEndpointLeaseReconciler creates a new peer endpoint lease reconciler |
| 80 |
func NewPeerEndpointLeaseReconciler(config *storagebackend.ConfigForResource, baseKey string, leaseTime time.Duration) (PeerEndpointLeaseReconciler, error) { |
| 81 |
// note that newFunc, newListFunc and resourcePrefix |
| 82 |
// can be left blank unless the storage.Watch method is used |
| 83 |
leaseStorage, destroyFn, err := storagefactory.Create(*config, nil, nil, "") |
| 84 |
if err != nil { |
| 85 |
return nil, fmt.Errorf("error creating storage factory: %v", err) |
| 86 |
} |
| 87 |
var once sync.Once |
| 88 |
return &peerEndpointLeaseReconciler{ |
| 89 |
serverLeases: &peerEndpointLeases{ |
| 90 |
storage: leaseStorage, |
| 91 |
destroyFn: func() { once.Do(destroyFn) }, |
| 92 |
baseKey: baseKey, |
| 93 |
leaseTime: leaseTime, |
| 94 |
}, |
| 95 |
}, nil |
| 96 |
} |
| 97 |
|
| 98 |
// PeerEndpointController is the controller manager for updating the peer endpoint leases. |
| 99 |
// This provides a separate independent reconciliation loop for peer endpoint leases |
| 100 |
// which ensures that the peer kube-apiservers are fetching the updated endpoint info for a given apiserver |
| 101 |
// in the case when the peer wants to proxy the request to the given apiserver because it can not serve the |
| 102 |
// request itself due to version mismatch. |
| 103 |
type PeerEndpointLeaseController struct { |
| 104 |
reconciler PeerEndpointLeaseReconciler |
| 105 |
endpointInterval time.Duration |
| 106 |
serverId string |
| 107 |
// peeraddress stores the IP and port of this kube-apiserver. Used by peer kube-apiservers to |
| 108 |
// route request to this apiserver in case of a version skew. |
| 109 |
peeraddress string |
| 110 |
|
| 111 |
client kubernetes.Interface |
| 112 |
|
| 113 |
lock sync.Mutex |
| 114 |
stopCh chan struct{} // closed by Stop() |
| 115 |
} |
| 116 |
|
| 117 |
func New(serverId string, peeraddress string, |
| 118 |
reconciler PeerEndpointLeaseReconciler, endpointInterval time.Duration, client kubernetes.Interface) *PeerEndpointLeaseController { |
| 119 |
return &PeerEndpointLeaseController{ |
| 120 |
reconciler: reconciler, |
| 121 |
serverId: serverId, |
| 122 |
// peeraddress stores the IP and port of this kube-apiserver. Used by peer kube-apiservers to |
| 123 |
// route request to this apiserver in case of a version skew. |
| 124 |
peeraddress: peeraddress, |
| 125 |
endpointInterval: endpointInterval, |
| 126 |
client: client, |
| 127 |
stopCh: make(chan struct{}), |
| 128 |
} |
| 129 |
} |
| 130 |
|
| 131 |
// Start begins the peer endpoint lease reconciler loop that must exist for bootstrapping |
| 132 |
// a cluster. |
| 133 |
func (c *PeerEndpointLeaseController) Start(stopCh <-chan struct{}) { |
| 134 |
localStopCh := make(chan struct{}) |
| 135 |
go func() { |
| 136 |
defer close(localStopCh) |
| 137 |
select { |
| 138 |
case <-stopCh: // from Start |
| 139 |
case <-c.stopCh: // from Stop |
| 140 |
} |
| 141 |
}() |
| 142 |
go c.Run(localStopCh) |
| 143 |
} |
| 144 |
|
| 145 |
// RunPeerEndpointReconciler periodically updates the peer endpoint leases |
| 146 |
func (c *PeerEndpointLeaseController) Run(stopCh <-chan struct{}) { |
| 147 |
// wait until process is ready |
| 148 |
wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) { |
| 149 |
var code int |
| 150 |
c.client.CoreV1().RESTClient().Get().AbsPath("/readyz").Do(context.TODO()).StatusCode(&code) |
| 151 |
return code == http.StatusOK, nil |
| 152 |
}, stopCh) |
| 153 |
|
| 154 |
wait.NonSlidingUntil(func() { |
| 155 |
if err := c.UpdatePeerEndpointLeases(); err != nil { |
| 156 |
runtime.HandleError(fmt.Errorf("unable to update peer endpoint leases: %v", err)) |
| 157 |
} |
| 158 |
}, c.endpointInterval, stopCh) |
| 159 |
} |
| 160 |
|
| 161 |
// Stop cleans up this apiserver's peer endpoint leases. |
| 162 |
func (c *PeerEndpointLeaseController) Stop() { |
| 163 |
c.lock.Lock() |
| 164 |
defer c.lock.Unlock() |
| 165 |
|
| 166 |
select { |
| 167 |
case <-c.stopCh: |
| 168 |
return // only close once |
| 169 |
default: |
| 170 |
close(c.stopCh) |
| 171 |
} |
| 172 |
finishedReconciling := make(chan struct{}) |
| 173 |
go func() { |
| 174 |
defer close(finishedReconciling) |
| 175 |
klog.Infof("Shutting down peer endpoint lease reconciler") |
| 176 |
// stop reconciliation |
| 177 |
c.reconciler.StopReconciling() |
| 178 |
|
| 179 |
// Ensure that there will be no race condition with the ReconcileEndpointLeases. |
| 180 |
if err := c.reconciler.RemoveLease(c.serverId); err != nil { |
| 181 |
klog.Errorf("Unable to remove peer endpoint leases: %v", err) |
| 182 |
} |
| 183 |
c.reconciler.Destroy() |
| 184 |
}() |
| 185 |
|
| 186 |
select { |
| 187 |
case <-finishedReconciling: |
| 188 |
// done |
| 189 |
case <-time.After(2 * c.endpointInterval): |
| 190 |
// don't block server shutdown forever if we can't reach etcd to remove ourselves |
| 191 |
klog.Warning("peer_endpoint_controller's RemoveEndpoints() timed out") |
| 192 |
} |
| 193 |
} |
| 194 |
|
| 195 |
// UpdatePeerEndpointLeases attempts to update the peer endpoint leases. |
| 196 |
func (c *PeerEndpointLeaseController) UpdatePeerEndpointLeases() error { |
| 197 |
host, port, err := net.SplitHostPort(c.peeraddress) |
| 198 |
if err != nil { |
| 199 |
return err |
| 200 |
} |
| 201 |
|
| 202 |
p, err := strconv.Atoi(port) |
| 203 |
if err != nil { |
| 204 |
return err |
| 205 |
} |
| 206 |
endpointPorts := createEndpointPortSpec(p, "https") |
| 207 |
|
| 208 |
// Ensure that there will be no race condition with the RemoveEndpointLeases. |
| 209 |
c.lock.Lock() |
| 210 |
defer c.lock.Unlock() |
| 211 |
|
| 212 |
// Refresh the TTL on our key, independently of whether any error or |
| 213 |
// update conflict happens below. This makes sure that at least some of |
| 214 |
// the servers will add our endpoint lease. |
| 215 |
if err := c.reconciler.UpdateLease(c.serverId, host, endpointPorts); err != nil { |
| 216 |
return err |
| 217 |
} |
| 218 |
return nil |
| 219 |
} |
| 220 |
|
| 221 |
// UpdateLease resets the TTL on a server IP in storage |
| 222 |
// UpdateLease will create a new key if it doesn't exist. |
| 223 |
// We use the first element in endpointPorts as a part of the lease's base key |
| 224 |
// This is done to support out tests that simulate 2 apiservers running on the same ip but |
| 225 |
// different ports |
| 226 |
|
| 227 |
// It will also do the following if UnknownVersionInteroperabilityProxy feature is enabled |
| 228 |
// 1. store the apiserverId as a label |
| 229 |
// 2. store the values passed to --peer-advertise-ip and --peer-advertise-port flags to kube-apiserver as an annotation |
| 230 |
// with value of format <ip:port> |
| 231 |
func (r *peerEndpointLeaseReconciler) UpdateLease(serverId string, ip string, endpointPorts []corev1.EndpointPort) error { |
| 232 |
// reconcile endpoints only if apiserver was not shutdown |
| 233 |
if r.stopReconcilingCalled.Load() { |
| 234 |
return nil |
| 235 |
} |
| 236 |
|
| 237 |
// we use the serverID as the key to avoid using the server IP, port as the key. |
| 238 |
// note: this means that this lease doesn't enforce mutual exclusion of ip/port usage between apiserver. |
| 239 |
key := path.Join(r.serverLeases.baseKey, serverId) |
| 240 |
return r.serverLeases.storage.GuaranteedUpdate(apirequest.NewDefaultContext(), key, &corev1.Endpoints{}, true, nil, func(input kruntime.Object, respMeta storage.ResponseMeta) (kruntime.Object, *uint64, error) { |
| 241 |
existing := input.(*corev1.Endpoints) |
| 242 |
existing.Subsets = []corev1.EndpointSubset{ |
| 243 |
{ |
| 244 |
Addresses: []corev1.EndpointAddress{{IP: ip}}, |
| 245 |
Ports: endpointPorts, |
| 246 |
}, |
| 247 |
} |
| 248 |
|
| 249 |
// store this server's identity (serverId) as a label. This will be used by |
| 250 |
// peers to find the IP of this server when the peer can not serve a request |
| 251 |
// due to version skew. |
| 252 |
if existing.Labels == nil { |
| 253 |
existing.Labels = map[string]string{} |
| 254 |
} |
| 255 |
existing.Labels[APIServerIdentityLabel] = serverId |
| 256 |
|
| 257 |
// leaseTime needs to be in seconds |
| 258 |
leaseTime := uint64(r.serverLeases.leaseTime / time.Second) |
| 259 |
|
| 260 |
// NB: GuaranteedUpdate does not perform the store operation unless |
| 261 |
// something changed between load and store (not including resource |
| 262 |
// version), meaning we can't refresh the TTL without actually |
| 263 |
// changing a field. |
| 264 |
existing.Generation++ |
| 265 |
|
| 266 |
klog.V(6).Infof("Resetting TTL on server IP %q listed in storage to %v", ip, leaseTime) |
| 267 |
return existing, &leaseTime, nil |
| 268 |
}, nil) |
| 269 |
} |
| 270 |
|
| 271 |
// ListLeases retrieves a list of the current server IPs from storage |
| 272 |
func (r *peerEndpointLeaseReconciler) ListLeases() ([]string, error) { |
| 273 |
storageOpts := storage.ListOptions{ |
| 274 |
ResourceVersion: "0", |
| 275 |
ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan, |
| 276 |
Predicate: storage.Everything, |
| 277 |
Recursive: true, |
| 278 |
} |
| 279 |
ipInfoList, err := r.getIpInfoList(storageOpts) |
| 280 |
if err != nil { |
| 281 |
return nil, err |
| 282 |
} |
| 283 |
ipList := make([]string, 0, len(ipInfoList.Items)) |
| 284 |
for _, ip := range ipInfoList.Items { |
| 285 |
if len(ip.Subsets) > 0 && len(ip.Subsets[0].Addresses) > 0 && len(ip.Subsets[0].Addresses[0].IP) > 0 { |
| 286 |
ipList = append(ipList, ip.Subsets[0].Addresses[0].IP) |
| 287 |
} |
| 288 |
} |
| 289 |
klog.V(6).Infof("Current server IPs listed in storage are %v", ipList) |
| 290 |
return ipList, nil |
| 291 |
} |
| 292 |
|
| 293 |
// GetLease retrieves the server IP and port for a specific server id |
| 294 |
func (r *peerEndpointLeaseReconciler) GetLease(serverId string) (string, error) { |
| 295 |
var fullAddr string |
| 296 |
if serverId == "" { |
| 297 |
return "", fmt.Errorf("error getting endpoint for serverId: empty serverId") |
| 298 |
} |
| 299 |
storageOpts := storage.ListOptions{ |
| 300 |
ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan, |
| 301 |
Predicate: storage.Everything, |
| 302 |
Recursive: true, |
| 303 |
} |
| 304 |
ipInfoList, err := r.getIpInfoList(storageOpts) |
| 305 |
if err != nil { |
| 306 |
return "", err |
| 307 |
} |
| 308 |
|
| 309 |
for _, ip := range ipInfoList.Items { |
| 310 |
if ip.Labels[APIServerIdentityLabel] == serverId { |
| 311 |
if len(ip.Subsets) > 0 { |
| 312 |
var ipStr, portStr string |
| 313 |
if len(ip.Subsets[0].Addresses) > 0 { |
| 314 |
if len(ip.Subsets[0].Addresses[0].IP) > 0 { |
| 315 |
ipStr = ip.Subsets[0].Addresses[0].IP |
| 316 |
} |
| 317 |
} |
| 318 |
if len(ip.Subsets[0].Ports) > 0 { |
| 319 |
portStr = fmt.Sprint(ip.Subsets[0].Ports[0].Port) |
| 320 |
} |
| 321 |
fullAddr = net.JoinHostPort(ipStr, portStr) |
| 322 |
break |
| 323 |
} |
| 324 |
} |
| 325 |
} |
| 326 |
klog.V(6).Infof("Fetched this server IP for the specified apiserverId %v, %v", serverId, fullAddr) |
| 327 |
return fullAddr, nil |
| 328 |
} |
| 329 |
|
| 330 |
func (r *peerEndpointLeaseReconciler) StopReconciling() { |
| 331 |
r.stopReconcilingCalled.Store(true) |
| 332 |
} |
| 333 |
|
| 334 |
// RemoveLease removes the lease on a server IP in storage |
| 335 |
// We use the first element in endpointPorts as a part of the lease's base key |
| 336 |
// This is done to support out tests that simulate 2 apiservers running on the same ip but |
| 337 |
// different ports |
| 338 |
func (r *peerEndpointLeaseReconciler) RemoveLease(serverId string) error { |
| 339 |
key := path.Join(r.serverLeases.baseKey, serverId) |
| 340 |
return r.serverLeases.storage.Delete(apirequest.NewDefaultContext(), key, &corev1.Endpoints{}, nil, rest.ValidateAllObjectFunc, nil, storage.DeleteOptions{}) |
| 341 |
} |
| 342 |
|
| 343 |
func (r *peerEndpointLeaseReconciler) Destroy() { |
| 344 |
r.serverLeases.destroyFn() |
| 345 |
} |
| 346 |
|
| 347 |
func (r *peerEndpointLeaseReconciler) GetEndpoint(serverId string) (string, error) { |
| 348 |
return r.GetLease(serverId) |
| 349 |
} |
| 350 |
|
| 351 |
func (r *peerEndpointLeaseReconciler) getIpInfoList(storageOpts storage.ListOptions) (*corev1.EndpointsList, error) { |
| 352 |
ipInfoList := &corev1.EndpointsList{} |
| 353 |
if err := r.serverLeases.storage.GetList(apirequest.NewDefaultContext(), r.serverLeases.baseKey, storageOpts, ipInfoList); err != nil { |
| 354 |
return nil, err |
| 355 |
} |
| 356 |
return ipInfoList, nil |
| 357 |
} |
| 358 |
|
| 359 |
// createEndpointPortSpec creates the endpoint ports |
| 360 |
func createEndpointPortSpec(endpointPort int, endpointPortName string) []corev1.EndpointPort { |
| 361 |
return []corev1.EndpointPort{{ |
| 362 |
Protocol: corev1.ProtocolTCP, |
| 363 |
Port: int32(endpointPort), |
| 364 |
Name: endpointPortName, |
| 365 |
}} |
| 366 |
} |