| 1 |
/* |
| 2 |
Copyright 2025 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package peerproxy |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"net/http" |
| 23 |
"time" |
| 24 |
|
| 25 |
"k8s.io/apimachinery/pkg/runtime" |
| 26 |
"k8s.io/apimachinery/pkg/util/wait" |
| 27 |
"k8s.io/apiserver/pkg/authentication/user" |
| 28 |
"k8s.io/client-go/discovery" |
| 29 |
"k8s.io/klog/v2" |
| 30 |
|
| 31 |
apidiscoveryv2 "k8s.io/api/apidiscovery/v2" |
| 32 |
v1 "k8s.io/api/coordination/v1" |
| 33 |
schema "k8s.io/apimachinery/pkg/runtime/schema" |
| 34 |
utilruntime "k8s.io/apimachinery/pkg/util/runtime" |
| 35 |
apirequest "k8s.io/apiserver/pkg/endpoints/request" |
| 36 |
responsewriterutil "k8s.io/apiserver/pkg/util/responsewriter" |
| 37 |
) |
| 38 |
|
| 39 |
const ( |
| 40 |
controllerName = "peer-discovery-cache-sync" |
| 41 |
maxRetries = 5 |
| 42 |
) |
| 43 |
|
| 44 |
func (h *peerProxyHandler) RunPeerDiscoveryCacheSync(ctx context.Context, workers int) { |
| 45 |
defer utilruntime.HandleCrash() |
| 46 |
defer h.peerLeaseQueue.ShutDown() |
| 47 |
defer func() { |
| 48 |
err := h.apiserverIdentityInformer.Informer().RemoveEventHandler(h.leaseRegistration) |
| 49 |
if err != nil { |
| 50 |
klog.Warning("error removing leaseInformer eventhandler") |
| 51 |
} |
| 52 |
}() |
| 53 |
|
| 54 |
klog.Infof("Workers: %d", workers) |
| 55 |
for i := 0; i < workers; i++ { |
| 56 |
klog.Infof("Starting worker") |
| 57 |
go wait.UntilWithContext(ctx, h.runWorker, time.Second) |
| 58 |
} |
| 59 |
<-ctx.Done() |
| 60 |
} |
| 61 |
|
| 62 |
func (h *peerProxyHandler) enqueueLease(lease *v1.Lease) { |
| 63 |
h.peerLeaseQueue.Add(lease.Name) |
| 64 |
} |
| 65 |
|
| 66 |
func (h *peerProxyHandler) runWorker(ctx context.Context) { |
| 67 |
for h.processNextElectionItem(ctx) { |
| 68 |
} |
| 69 |
} |
| 70 |
|
| 71 |
func (h *peerProxyHandler) processNextElectionItem(ctx context.Context) bool { |
| 72 |
key, shutdown := h.peerLeaseQueue.Get() |
| 73 |
if shutdown { |
| 74 |
return false |
| 75 |
} |
| 76 |
defer h.peerLeaseQueue.Done(key) |
| 77 |
|
| 78 |
err := h.syncPeerDiscoveryCache(ctx) |
| 79 |
h.handleErr(err, key) |
| 80 |
return true |
| 81 |
} |
| 82 |
|
| 83 |
func (h *peerProxyHandler) syncPeerDiscoveryCache(ctx context.Context) error { |
| 84 |
var fetchDiscoveryErr error |
| 85 |
// Rebuild the peer discovery cache from available leases. |
| 86 |
leases, err := h.apiserverIdentityInformer.Lister().List(h.identityLeaseLabelSelector) |
| 87 |
if err != nil { |
| 88 |
utilruntime.HandleError(err) |
| 89 |
return err |
| 90 |
} |
| 91 |
|
| 92 |
newCache := map[string]map[schema.GroupVersionResource]bool{} |
| 93 |
for _, l := range leases { |
| 94 |
_, ok := h.isValidPeerIdentityLease(l) |
| 95 |
if !ok { |
| 96 |
continue |
| 97 |
} |
| 98 |
|
| 99 |
discoveryInfo, err := h.fetchNewDiscoveryFor(ctx, l.Name, *l.Spec.HolderIdentity) |
| 100 |
if err != nil { |
| 101 |
fetchDiscoveryErr = err |
| 102 |
} |
| 103 |
|
| 104 |
if discoveryInfo != nil { |
| 105 |
newCache[l.Name] = discoveryInfo |
| 106 |
} |
| 107 |
} |
| 108 |
|
| 109 |
// Overwrite cache with new contents. |
| 110 |
h.peerDiscoveryInfoCache.Store(newCache) |
| 111 |
return fetchDiscoveryErr |
| 112 |
} |
| 113 |
|
| 114 |
func (h *peerProxyHandler) fetchNewDiscoveryFor(ctx context.Context, serverID string, holderIdentity string) (map[schema.GroupVersionResource]bool, error) { |
| 115 |
hostport, err := h.hostportInfo(serverID) |
| 116 |
if err != nil { |
| 117 |
return nil, fmt.Errorf("failed to get host port info from identity lease for server %s: %w", serverID, err) |
| 118 |
} |
| 119 |
|
| 120 |
klog.V(4).Infof("Proxying an agg-discovery call from %s to %s", h.serverID, serverID) |
| 121 |
servedResources := make(map[schema.GroupVersionResource]bool) |
| 122 |
var discoveryErr error |
| 123 |
var discoveryResponse *apidiscoveryv2.APIGroupDiscoveryList |
| 124 |
discoveryPaths := []string{"/api", "/apis"} |
| 125 |
for _, path := range discoveryPaths { |
| 126 |
discoveryResponse, discoveryErr = h.aggregateDiscovery(ctx, path, hostport) |
| 127 |
if err != nil { |
| 128 |
klog.ErrorS(err, "error querying discovery endpoint for serverID", "path", path, "serverID", serverID) |
| 129 |
continue |
| 130 |
} |
| 131 |
|
| 132 |
for _, groupDiscovery := range discoveryResponse.Items { |
| 133 |
groupName := groupDiscovery.Name |
| 134 |
if groupName == "" { |
| 135 |
groupName = "core" |
| 136 |
} |
| 137 |
|
| 138 |
for _, version := range groupDiscovery.Versions { |
| 139 |
for _, resource := range version.Resources { |
| 140 |
gvr := schema.GroupVersionResource{Group: groupName, Version: version.Version, Resource: resource.Resource} |
| 141 |
servedResources[gvr] = true |
| 142 |
} |
| 143 |
} |
| 144 |
} |
| 145 |
} |
| 146 |
|
| 147 |
klog.V(4).Infof("Agg discovery done successfully by %s for %s", h.serverID, serverID) |
| 148 |
return servedResources, discoveryErr |
| 149 |
} |
| 150 |
|
| 151 |
func (h *peerProxyHandler) aggregateDiscovery(ctx context.Context, path string, hostport string) (*apidiscoveryv2.APIGroupDiscoveryList, error) { |
| 152 |
req, err := http.NewRequest(http.MethodGet, path, nil) |
| 153 |
if err != nil { |
| 154 |
return nil, err |
| 155 |
} |
| 156 |
|
| 157 |
apiServerUser := &user.DefaultInfo{ |
| 158 |
Name: user.APIServerUser, |
| 159 |
UID: user.APIServerUser, |
| 160 |
Groups: []string{user.AllAuthenticated}, |
| 161 |
} |
| 162 |
|
| 163 |
ctx = apirequest.WithUser(ctx, apiServerUser) |
| 164 |
req = req.WithContext(ctx) |
| 165 |
|
| 166 |
req.Header.Add("Accept", discovery.AcceptV2) |
| 167 |
|
| 168 |
writer := responsewriterutil.NewInMemoryResponseWriter() |
| 169 |
h.proxyRequestToDestinationAPIServer(req, writer, hostport) |
| 170 |
if writer.RespCode() != http.StatusOK { |
| 171 |
return nil, fmt.Errorf("discovery request failed with status: %d", writer.RespCode()) |
| 172 |
} |
| 173 |
|
| 174 |
parsed := &apidiscoveryv2.APIGroupDiscoveryList{} |
| 175 |
if err := runtime.DecodeInto(h.discoverySerializer.UniversalDecoder(), writer.Data(), parsed); err != nil { |
| 176 |
return nil, fmt.Errorf("error decoding discovery response: %w", err) |
| 177 |
} |
| 178 |
|
| 179 |
return parsed, nil |
| 180 |
} |
| 181 |
|
| 182 |
// handleErr checks if an error happened and makes sure we will retry later. |
| 183 |
func (h *peerProxyHandler) handleErr(err error, key string) { |
| 184 |
if err == nil { |
| 185 |
h.peerLeaseQueue.Forget(key) |
| 186 |
return |
| 187 |
} |
| 188 |
|
| 189 |
if h.peerLeaseQueue.NumRequeues(key) < maxRetries { |
| 190 |
klog.Infof("Error syncing discovery for peer lease %v: %v", key, err) |
| 191 |
h.peerLeaseQueue.AddRateLimited(key) |
| 192 |
return |
| 193 |
} |
| 194 |
|
| 195 |
h.peerLeaseQueue.Forget(key) |
| 196 |
utilruntime.HandleError(err) |
| 197 |
klog.Infof("Dropping lease %s out of the queue: %v", key, err) |
| 198 |
} |