Debian Codegraph @

pkg/util/peerproxy/peerproxy_handler_test.go

raw
1 /*
2 Copyright 2023 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package peerproxy
18
19 import (
20 "net/http"
21 "strings"
22 "testing"
23 "time"
24
25 "net/http/httptest"
26
27 "github.com/google/uuid"
28 "github.com/stretchr/testify/assert"
29 "k8s.io/apimachinery/pkg/api/apitesting"
30 "k8s.io/apimachinery/pkg/runtime"
31 "k8s.io/apimachinery/pkg/runtime/schema"
32 "k8s.io/apimachinery/pkg/runtime/serializer"
33 "k8s.io/apimachinery/pkg/util/sets"
34 "k8s.io/apiserver/pkg/authentication/user"
35 "k8s.io/apiserver/pkg/features"
36 "k8s.io/apiserver/pkg/reconcilers"
37 "k8s.io/apiserver/pkg/util/peerproxy/metrics"
38 "k8s.io/client-go/informers"
39 "k8s.io/client-go/kubernetes/fake"
40 "k8s.io/client-go/rest"
41 "k8s.io/client-go/transport"
42 "k8s.io/component-base/metrics/legacyregistry"
43 "k8s.io/component-base/metrics/testutil"
44
45 corev1 "k8s.io/api/core/v1"
46 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
47 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
48 apifilters "k8s.io/apiserver/pkg/endpoints/filters"
49 apirequest "k8s.io/apiserver/pkg/endpoints/request"
50 etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
51 utilfeature "k8s.io/apiserver/pkg/util/feature"
52 featuregatetesting "k8s.io/component-base/featuregate/testing"
53 )
54
55 const (
56 requestTimeout = 30 * time.Second
57 localServerID = "local-apiserver"
58 remoteServerID1 = "remote-apiserver-1"
59 remoteServerID2 = "remote-apiserver-2"
60 )
61
62 type server struct {
63 publicIP string
64 serverID string
65 }
66
67 type reconciler struct {
68 do bool
69 servers []server
70 }
71
72 func TestPeerProxy(t *testing.T) {
73 testCases := []struct {
74 desc string
75 informerFinishedSync bool
76 requestPath string
77 peerproxiedHeader string
78 reconcilerConfig reconciler
79 localCache map[schema.GroupVersionResource]bool
80 peerCache map[string]map[schema.GroupVersionResource]bool
81 wantStatus int
82 wantMetricsData string
83 }{
84 {
85 desc: "allow non resource requests",
86 requestPath: "/foo/bar/baz",
87 wantStatus: http.StatusOK,
88 },
89 {
90 desc: "allow if already proxied once",
91 requestPath: "/api/bar/baz",
92 peerproxiedHeader: "true",
93 wantStatus: http.StatusOK,
94 },
95 {
96 desc: "allow if unsynced informers",
97 requestPath: "/api/bar/baz",
98 informerFinishedSync: false,
99 wantStatus: http.StatusOK,
100 },
101 {
102 desc: "Serve locally if serviceable",
103 requestPath: "/api/foo/bar",
104 localCache: map[schema.GroupVersionResource]bool{
105 {Group: "core", Version: "foo", Resource: "bar"}: true,
106 },
107 wantStatus: http.StatusOK,
108 },
109 {
110 desc: "200 if no appropriate peers found, serve locally",
111 requestPath: "/api/foo/bar",
112 informerFinishedSync: true,
113 wantStatus: http.StatusOK,
114 },
115 {
116 desc: "503 if no endpoint fetched from lease",
117 requestPath: "/api/foo/bar",
118 informerFinishedSync: true,
119 peerCache: map[string]map[schema.GroupVersionResource]bool{
120 remoteServerID1: {
121 {Group: "core", Version: "foo", Resource: "bar"}: true,
122 },
123 },
124 wantStatus: http.StatusServiceUnavailable,
125 },
126 {
127 desc: "503 unreachable peer bind address",
128 requestPath: "/api/foo/bar",
129 informerFinishedSync: true,
130 peerCache: map[string]map[schema.GroupVersionResource]bool{
131 remoteServerID1: {
132 {Group: "core", Version: "foo", Resource: "bar"}: true,
133 },
134 },
135 reconcilerConfig: reconciler{
136 do: true,
137 servers: []server{
138 {
139 publicIP: "1.2.3.4",
140 serverID: remoteServerID1,
141 },
142 },
143 },
144 wantStatus: http.StatusServiceUnavailable,
145 wantMetricsData: `
146 # HELP apiserver_rerouted_request_total [ALPHA] Total number of requests that were proxied to a peer kube apiserver because the local apiserver was not capable of serving it
147 # TYPE apiserver_rerouted_request_total counter
148 apiserver_rerouted_request_total{code="503"} 1
149 `,
150 },
151 {
152 desc: "503 if one apiserver's endpoint lease wasnt found but another valid (unreachable) apiserver was found",
153 requestPath: "/api/foo/bar",
154 informerFinishedSync: true,
155 peerCache: map[string]map[schema.GroupVersionResource]bool{
156 remoteServerID1: {
157 {Group: "core", Version: "foo", Resource: "bar"}: true,
158 },
159 remoteServerID2: {
160 {Group: "core", Version: "foo", Resource: "bar"}: true,
161 },
162 },
163 reconcilerConfig: reconciler{
164 do: true,
165 servers: []server{
166 {
167 publicIP: "1.2.3.4",
168 serverID: remoteServerID1,
169 },
170 },
171 },
172 wantStatus: http.StatusServiceUnavailable,
173 wantMetricsData: `
174 # HELP apiserver_rerouted_request_total [ALPHA] Total number of requests that were proxied to a peer kube apiserver because the local apiserver was not capable of serving it
175 # TYPE apiserver_rerouted_request_total counter
176 apiserver_rerouted_request_total{code="503"} 1
177 `,
178 },
179 }
180
181 metrics.Register()
182 for _, tt := range testCases {
183 t.Run(tt.desc, func(t *testing.T) {
184 defer metrics.Reset()
185 lastHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
186 w.Write([]byte("OK"))
187 })
188 serverIDs := []string{localServerID}
189 for peerID := range tt.peerCache {
190 serverIDs = append(serverIDs, peerID)
191 }
192 fakeReconciler := newFakePeerEndpointReconciler(t)
193 handler := newHandlerChain(t, tt.informerFinishedSync, lastHandler, fakeReconciler, tt.localCache, tt.peerCache)
194 server, requestGetter := createHTTP2ServerWithClient(handler, requestTimeout*2)
195 defer server.Close()
196
197 if tt.reconcilerConfig.do {
198 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)
199
200 for _, s := range tt.reconcilerConfig.servers {
201 err := fakeReconciler.UpdateLease(s.serverID,
202 s.publicIP,
203 []corev1.EndpointPort{{Name: "foo",
204 Port: 8080, Protocol: "TCP"}})
205 if err != nil {
206 t.Errorf("Failed to update lease for server %s", s.serverID)
207 }
208 }
209 }
210
211 req, err := http.NewRequest(http.MethodGet, server.URL+tt.requestPath, nil)
212 if err != nil {
213 t.Fatalf("failed to create new http request - %v", err)
214 }
215 req.Header.Set(PeerProxiedHeader, tt.peerproxiedHeader)
216
217 resp, _ := requestGetter(req)
218
219 // compare response
220 assert.Equal(t, tt.wantStatus, resp.StatusCode)
221
222 // compare metric
223 if tt.wantMetricsData != "" {
224 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.wantMetricsData), []string{"apiserver_rerouted_request_total"}...); err != nil {
225 t.Fatal(err)
226 }
227 }
228 })
229 }
230
231 }
232
233 func newFakePeerEndpointReconciler(t *testing.T) reconcilers.PeerEndpointLeaseReconciler {
234 server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
235 t.Cleanup(func() { server.Terminate(t) })
236 scheme := runtime.NewScheme()
237 metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
238 //utilruntime.Must(core.AddToScheme(scheme))
239 utilruntime.Must(corev1.AddToScheme(scheme))
240 utilruntime.Must(scheme.SetVersionPriority(corev1.SchemeGroupVersion))
241 codecs := serializer.NewCodecFactory(scheme)
242 sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion)
243 config := *sc.ForResource(schema.GroupResource{Resource: "endpoints"})
244 baseKey := "/" + uuid.New().String() + "/peer-testleases/"
245 leaseTime := 1 * time.Minute
246 reconciler, err := reconcilers.NewPeerEndpointLeaseReconciler(&config, baseKey, leaseTime)
247 if err != nil {
248 t.Fatalf("Error creating storage: %v", err)
249 }
250 return reconciler
251 }
252
253 func newHandlerChain(t *testing.T, informerFinishedSync bool, handler http.Handler,
254 reconciler reconcilers.PeerEndpointLeaseReconciler,
255 localCache map[schema.GroupVersionResource]bool, peerCache map[string]map[schema.GroupVersionResource]bool) http.Handler {
256 // Add peerproxy handler
257 s := serializer.NewCodecFactory(runtime.NewScheme()).WithoutConversion()
258 peerProxyHandler, err := newFakePeerProxyHandler(informerFinishedSync, reconciler, localServerID, s, localCache, peerCache)
259 if err != nil {
260 t.Fatalf("Error creating peer proxy handler: %v", err)
261 }
262 peerProxyHandler.finishedSync.Store(informerFinishedSync)
263 handler = peerProxyHandler.WrapHandler(handler)
264
265 // Add user info
266 handler = withFakeUser(handler)
267
268 // Add requestInfo handler
269 requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")}
270 handler = apifilters.WithRequestInfo(handler, requestInfoFactory)
271 return handler
272 }
273
274 func newFakePeerProxyHandler(informerFinishedSync bool,
275 reconciler reconcilers.PeerEndpointLeaseReconciler, id string, s runtime.NegotiatedSerializer,
276 localCache map[schema.GroupVersionResource]bool, peerCache map[string]map[schema.GroupVersionResource]bool) (*peerProxyHandler, error) {
277 clientset := fake.NewSimpleClientset()
278 informerFactory := informers.NewSharedInformerFactory(clientset, 0)
279 leaseInformer := informerFactory.Coordination().V1().Leases()
280 clientConfig := &transport.Config{
281 TLS: transport.TLSConfig{
282 Insecure: false,
283 }}
284 loopbackClientConfig := &rest.Config{
285 Host: "localhost:1010",
286 }
287 ppH, err := NewPeerProxyHandler(id, "identity=testserver", leaseInformer, reconciler, s, loopbackClientConfig, clientConfig)
288 if err != nil {
289 return nil, err
290 }
291 ppH.localDiscoveryInfoCache.Store(localCache)
292 ppH.peerDiscoveryInfoCache.Store(peerCache)
293
294 ppH.finishedSync.Store(informerFinishedSync)
295 return ppH, nil
296 }
297
298 func withFakeUser(handler http.Handler) http.Handler {
299 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
300 r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{
301 Groups: r.Header["Groups"],
302 }))
303 handler.ServeHTTP(w, r)
304 })
305 }
306
307 // returns a started http2 server, with a client function to send request to the server.
308 func createHTTP2ServerWithClient(handler http.Handler, clientTimeout time.Duration) (*httptest.Server, func(req *http.Request) (*http.Response, error)) {
309 server := httptest.NewUnstartedServer(handler)
310 server.EnableHTTP2 = true
311 server.StartTLS()
312 cli := server.Client()
313 cli.Timeout = clientTimeout
314 return server, func(req *http.Request) (*http.Response, error) {
315 return cli.Do(req)
316 }
317 }