Debian Codegraph @

pkg/util/peerproxy/peerproxy_handler.go

raw
1 /*
2 Copyright 2023 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package peerproxy
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "math/rand"
24 "net"
25 "net/http"
26 "net/url"
27 "strconv"
28 "sync"
29 "sync/atomic"
30 "time"
31
32 "k8s.io/apimachinery/pkg/labels"
33 "k8s.io/apimachinery/pkg/runtime"
34 "k8s.io/apimachinery/pkg/util/proxy"
35 "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
36 "k8s.io/apiserver/pkg/endpoints/responsewriter"
37 "k8s.io/apiserver/pkg/reconcilers"
38 "k8s.io/apiserver/pkg/util/peerproxy/metrics"
39 "k8s.io/client-go/discovery"
40 "k8s.io/client-go/tools/cache"
41 "k8s.io/client-go/transport"
42 "k8s.io/client-go/util/workqueue"
43 "k8s.io/klog/v2"
44
45 apierrors "k8s.io/apimachinery/pkg/api/errors"
46 schema "k8s.io/apimachinery/pkg/runtime/schema"
47 "k8s.io/apimachinery/pkg/runtime/serializer"
48 epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
49 apirequest "k8s.io/apiserver/pkg/endpoints/request"
50 apiserverproxyutil "k8s.io/apiserver/pkg/util/proxy"
51 coordinationv1informers "k8s.io/client-go/informers/coordination/v1"
52 )
53
54 const (
55 PeerProxiedHeader = "x-kubernetes-peer-proxied"
56 )
57
58 type peerProxyHandler struct {
59 name string
60 // Identity for this server.
61 serverID string
62 finishedSync atomic.Bool
63 // Label to check against in identity leases to make sure
64 // we are working with apiserver identity leases only.
65 identityLeaseLabelSelector labels.Selector
66 apiserverIdentityInformer coordinationv1informers.LeaseInformer
67 leaseRegistration cache.ResourceEventHandlerRegistration
68 // Reconciler that is used to fetch host port of peer apiserver when proxying request to a peer.
69 reconciler reconcilers.PeerEndpointLeaseReconciler
70 // Client to make discovery calls locally.
71 discoveryClient *discovery.DiscoveryClient
72 discoverySerializer serializer.CodecFactory
73 // Cache that stores resources served by this apiserver. Refreshed periodically.
74 // We always look up in the local discovery cache first, to check whether the
75 // request can be served by this apiserver instead of proxying it to a peer.
76 localDiscoveryInfoCache atomic.Value
77 localDiscoveryCacheTicker *time.Ticker
78 localDiscoveryInfoCachePopulated chan struct{}
79 localDiscoveryInfoCachePopulatedOnce sync.Once
80 // Cache that stores resources served by peer apiservers.
81 // Refreshed if a new apiserver identity lease is added, deleted or
82 // holderIndentity change is observed in the lease.
83 peerDiscoveryInfoCache atomic.Value
84 proxyTransport http.RoundTripper
85 // Worker queue that keeps the peerDiscoveryInfoCache up-to-date.
86 peerLeaseQueue workqueue.TypedRateLimitingInterface[string]
87 serializer runtime.NegotiatedSerializer
88 }
89
90 // responder implements rest.Responder for assisting a connector in writing objects or errors.
91 type responder struct {
92 w http.ResponseWriter
93 ctx context.Context
94 }
95
96 func (h *peerProxyHandler) HasFinishedSync() bool {
97 return h.finishedSync.Load()
98 }
99
100 func (h *peerProxyHandler) WaitForCacheSync(stopCh <-chan struct{}) error {
101 ok := cache.WaitForNamedCacheSync("mixed-version-proxy", stopCh, h.apiserverIdentityInformer.Informer().HasSynced)
102 if !ok {
103 return fmt.Errorf("error while waiting for initial cache sync")
104 }
105
106 if !cache.WaitForNamedCacheSync(controllerName, stopCh, h.leaseRegistration.HasSynced) {
107 return fmt.Errorf("error while waiting for peer-identity-lease event handler registration sync")
108 }
109
110 // Wait for localDiscoveryInfoCache to be populated.
111 select {
112 case <-h.localDiscoveryInfoCachePopulated:
113 case <-stopCh:
114 return fmt.Errorf("stop signal received while waiting for local discovery cache population")
115 }
116
117 h.finishedSync.Store(true)
118 return nil
119 }
120
121 // WrapHandler will fetch the apiservers that can serve the request and either serve it locally
122 // or route it to a peer
123 func (h *peerProxyHandler) WrapHandler(handler http.Handler) http.Handler {
124 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
125 ctx := r.Context()
126 requestInfo, ok := apirequest.RequestInfoFrom(ctx)
127 if !ok {
128 responsewriters.InternalError(w, r, errors.New("no RequestInfo found in the context"))
129 return
130 }
131
132 // Allow non-resource requests
133 if !requestInfo.IsResourceRequest {
134 klog.V(3).Infof("Not a resource request skipping proxying")
135 handler.ServeHTTP(w, r)
136 return
137 }
138
139 // Request has already been proxied once, it must be served locally
140 if r.Header.Get(PeerProxiedHeader) == "true" {
141 klog.V(3).Infof("Already rerouted once, skipping proxying to peer")
142 handler.ServeHTTP(w, r)
143 return
144 }
145
146 // Apiserver Identity Informers is not synced yet, pass request to next handler
147 // This will happen for self requests from the kube-apiserver because we have a poststarthook
148 // to ensure that external requests are not served until the ApiserverIdentity Informer has synced
149 if !h.HasFinishedSync() {
150 handler.ServeHTTP(w, r)
151 return
152 }
153
154 gvr := schema.GroupVersionResource{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion, Resource: requestInfo.Resource}
155 if requestInfo.APIGroup == "" {
156 gvr.Group = "core"
157 }
158
159 if h.shouldServeLocally(gvr) {
160 handler.ServeHTTP(w, r)
161 return
162 }
163
164 // find servers that are capable of serving this request
165 peerServerIDs := h.findServiceableByPeerFromPeerDiscoveryCache(gvr)
166 if len(peerServerIDs) == 0 {
167 klog.Errorf("gvr %v is not served by anything in this cluster", gvr)
168 handler.ServeHTTP(w, r)
169 return
170 }
171
172 peerEndpoints, err := h.resolveServingLocation(peerServerIDs)
173 if err != nil {
174 gv := schema.GroupVersion{Group: gvr.Group, Version: gvr.Version}
175 klog.ErrorS(err, "error finding serviceable-by apiservers for the requested resource", "gvr", gvr)
176 responsewriters.ErrorNegotiated(apierrors.NewServiceUnavailable("Error getting ip and port info of the remote server while proxying"), h.serializer, gv, w, r)
177 return
178 }
179
180 rand := rand.Intn(len(peerEndpoints))
181 peerEndpoint := peerEndpoints[rand]
182 h.proxyRequestToDestinationAPIServer(r, w, peerEndpoint)
183 })
184 }
185
186 // RunLocalDiscoveryCacheSync populated the localDiscoveryInfoCache and
187 // starts a goroutine to periodically refresh the local discovery cache.
188 func (h *peerProxyHandler) RunLocalDiscoveryCacheSync(stopCh <-chan struct{}) error {
189 klog.Info("localDiscoveryCacheInvalidation goroutine started")
190 // Populate the cache initially.
191 if err := h.populateLocalDiscoveryCache(); err != nil {
192 return fmt.Errorf("failed to populate initial local discovery cache: %w", err)
193 }
194
195 go func() {
196 for {
197 select {
198 case <-h.localDiscoveryCacheTicker.C:
199 klog.V(4).Infof("Invalidating local discovery cache")
200 if err := h.populateLocalDiscoveryCache(); err != nil {
201 klog.Errorf("Failed to repopulate local discovery cache: %v", err)
202 }
203 case <-stopCh:
204 klog.Info("localDiscoveryCacheInvalidation goroutine received stop signal")
205 if h.localDiscoveryCacheTicker != nil {
206 h.localDiscoveryCacheTicker.Stop()
207 klog.Info("localDiscoveryCacheTicker stopped")
208 }
209 klog.Info("localDiscoveryCacheInvalidation goroutine exiting")
210 return
211 }
212 }
213 }()
214 return nil
215 }
216
217 func (h *peerProxyHandler) populateLocalDiscoveryCache() error {
218 _, resourcesByGV, _, err := h.discoveryClient.GroupsAndMaybeResources()
219 if err != nil {
220 return fmt.Errorf("error getting API group resources from discovery: %w", err)
221 }
222
223 freshLocalDiscoveryResponse := map[schema.GroupVersionResource]bool{}
224 for gv, resources := range resourcesByGV {
225 if gv.Group == "" {
226 gv.Group = "core"
227 }
228 for _, resource := range resources.APIResources {
229 gvr := gv.WithResource(resource.Name)
230 freshLocalDiscoveryResponse[gvr] = true
231 }
232 }
233
234 h.localDiscoveryInfoCache.Store(freshLocalDiscoveryResponse)
235 // Signal that the cache has been populated.
236 h.localDiscoveryInfoCachePopulatedOnce.Do(func() {
237 close(h.localDiscoveryInfoCachePopulated)
238 })
239 return nil
240 }
241
242 // shouldServeLocally checks if the requested resource is present in the local
243 // discovery cache indicating the request can be served by this server.
244 func (h *peerProxyHandler) shouldServeLocally(gvr schema.GroupVersionResource) bool {
245 cache := h.localDiscoveryInfoCache.Load().(map[schema.GroupVersionResource]bool)
246 exists, ok := cache[gvr]
247 if !ok {
248 klog.V(4).Infof("resource not found for %v in local discovery cache\n", gvr.GroupVersion())
249 return false
250 }
251
252 if exists {
253 return true
254 }
255
256 return false
257 }
258
259 func (h *peerProxyHandler) findServiceableByPeerFromPeerDiscoveryCache(gvr schema.GroupVersionResource) []string {
260 var serviceableByIDs []string
261 cache := h.peerDiscoveryInfoCache.Load().(map[string]map[schema.GroupVersionResource]bool)
262 for peerID, servedResources := range cache {
263 // Ignore local apiserver.
264 if peerID == h.serverID {
265 continue
266 }
267
268 exists, ok := servedResources[gvr]
269 if !ok {
270 continue
271 }
272
273 if exists {
274 serviceableByIDs = append(serviceableByIDs, peerID)
275 }
276 }
277
278 return serviceableByIDs
279 }
280
281 // resolveServingLocation resolves the host:port addresses for the given peer IDs.
282 func (h *peerProxyHandler) resolveServingLocation(peerIDs []string) ([]string, error) {
283 var peerServerEndpoints []string
284 var errs []error
285
286 for _, id := range peerIDs {
287 hostPort, err := h.hostportInfo(id)
288 if err != nil {
289 errs = append(errs, err)
290 continue
291 }
292
293 peerServerEndpoints = append(peerServerEndpoints, hostPort)
294 }
295
296 // reset err if there was atleast one valid peer server found.
297 if len(peerServerEndpoints) > 0 {
298 errs = nil
299 }
300
301 return peerServerEndpoints, errors.Join(errs...)
302 }
303
304 func (h *peerProxyHandler) hostportInfo(apiserverKey string) (string, error) {
305 hostPort, err := h.reconciler.GetEndpoint(apiserverKey)
306 if err != nil {
307 return "", err
308 }
309
310 _, _, err = net.SplitHostPort(hostPort)
311 if err != nil {
312 return "", err
313 }
314
315 return hostPort, nil
316 }
317
318 func (h *peerProxyHandler) proxyRequestToDestinationAPIServer(req *http.Request, rw http.ResponseWriter, host string) {
319 // write a new location based on the existing request pointed at the target service
320 location := &url.URL{}
321 location.Scheme = "https"
322 location.Host = host
323 location.Path = req.URL.Path
324 location.RawQuery = req.URL.Query().Encode()
325
326 newReq, cancelFn := apiserverproxyutil.NewRequestForProxy(location, req)
327 newReq.Header.Add(PeerProxiedHeader, "true")
328 defer cancelFn()
329
330 proxyRoundTripper, err := h.buildProxyRoundtripper(req)
331 if err != nil {
332 klog.Errorf("failed to build proxy round tripper: %v", err)
333 return
334 }
335
336 delegate := &epmetrics.ResponseWriterDelegator{ResponseWriter: rw}
337 w := responsewriter.WrapForHTTP1Or2(delegate)
338 handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, false, &responder{w: w, ctx: req.Context()})
339 handler.ServeHTTP(w, newReq)
340 metrics.IncPeerProxiedRequest(req.Context(), strconv.Itoa(delegate.Status()))
341 }
342
343 func (h *peerProxyHandler) buildProxyRoundtripper(req *http.Request) (http.RoundTripper, error) {
344 user, ok := apirequest.UserFrom(req.Context())
345 if !ok {
346 return nil, apierrors.NewBadRequest("no user details present in request")
347 }
348
349 return transport.NewAuthProxyRoundTripper(user.GetName(), user.GetUID(), user.GetGroups(), user.GetExtra(), h.proxyTransport), nil
350 }
351
352 func (r *responder) Error(w http.ResponseWriter, req *http.Request, err error) {
353 klog.ErrorS(err, "Error while proxying request to destination apiserver")
354 http.Error(w, err.Error(), http.StatusServiceUnavailable)
355 }