| 1 |
/* |
| 2 |
Copyright 2025 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package peerproxy |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"encoding/json" |
| 22 |
"fmt" |
| 23 |
"net/http" |
| 24 |
"net/http/httptest" |
| 25 |
"testing" |
| 26 |
"time" |
| 27 |
|
| 28 |
"github.com/stretchr/testify/assert" |
| 29 |
"google.golang.org/protobuf/proto" |
| 30 |
"k8s.io/apimachinery/pkg/runtime" |
| 31 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 32 |
"k8s.io/apimachinery/pkg/runtime/serializer" |
| 33 |
"k8s.io/apimachinery/pkg/util/wait" |
| 34 |
"k8s.io/client-go/informers" |
| 35 |
"k8s.io/client-go/kubernetes/fake" |
| 36 |
"k8s.io/client-go/rest" |
| 37 |
"k8s.io/client-go/tools/cache" |
| 38 |
"k8s.io/client-go/transport" |
| 39 |
|
| 40 |
apidiscoveryv2 "k8s.io/api/apidiscovery/v2" |
| 41 |
v1 "k8s.io/api/coordination/v1" |
| 42 |
corev1 "k8s.io/api/core/v1" |
| 43 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 44 |
) |
| 45 |
|
| 46 |
func TestRunPeerDiscoveryCacheSync(t *testing.T) { |
| 47 |
localServerID := "local-server" |
| 48 |
|
| 49 |
testCases := []struct { |
| 50 |
desc string |
| 51 |
leases []*v1.Lease |
| 52 |
labelSelectorString string |
| 53 |
updatedLease *v1.Lease |
| 54 |
deletedLeaseNames []string |
| 55 |
wantCache map[string]map[schema.GroupVersionResource]bool |
| 56 |
}{ |
| 57 |
{ |
| 58 |
desc: "single remote server", |
| 59 |
labelSelectorString: "apiserver-identity=testserver", |
| 60 |
leases: []*v1.Lease{ |
| 61 |
{ |
| 62 |
ObjectMeta: metav1.ObjectMeta{ |
| 63 |
Name: "remote-1", |
| 64 |
Labels: map[string]string{"apiserver-identity": "testserver"}, |
| 65 |
}, |
| 66 |
Spec: v1.LeaseSpec{HolderIdentity: proto.String("holder-1")}, |
| 67 |
}, |
| 68 |
}, |
| 69 |
wantCache: map[string]map[schema.GroupVersionResource]bool{ |
| 70 |
"remote-1": { |
| 71 |
{Group: "testgroup", Version: "v1", Resource: "testresources"}: true, |
| 72 |
}, |
| 73 |
}, |
| 74 |
}, |
| 75 |
{ |
| 76 |
desc: "multiple remote servers", |
| 77 |
labelSelectorString: "apiserver-identity=testserver", |
| 78 |
leases: []*v1.Lease{ |
| 79 |
{ |
| 80 |
ObjectMeta: metav1.ObjectMeta{ |
| 81 |
Name: "remote-1", |
| 82 |
Labels: map[string]string{"apiserver-identity": "testserver"}, |
| 83 |
}, |
| 84 |
Spec: v1.LeaseSpec{HolderIdentity: proto.String("holder-1")}, |
| 85 |
}, |
| 86 |
{ |
| 87 |
ObjectMeta: metav1.ObjectMeta{ |
| 88 |
Name: "remote-2", |
| 89 |
Labels: map[string]string{"apiserver-identity": "testserver"}, |
| 90 |
}, |
| 91 |
Spec: v1.LeaseSpec{HolderIdentity: proto.String("holder-2")}, |
| 92 |
}, |
| 93 |
}, |
| 94 |
wantCache: map[string]map[schema.GroupVersionResource]bool{ |
| 95 |
"remote-1": { |
| 96 |
{Group: "testgroup", Version: "v1", Resource: "testresources"}: true, |
| 97 |
}, |
| 98 |
"remote-2": { |
| 99 |
{Group: "testgroup", Version: "v1", Resource: "testresources"}: true, |
| 100 |
}, |
| 101 |
}, |
| 102 |
}, |
| 103 |
{ |
| 104 |
desc: "lease update", |
| 105 |
labelSelectorString: "apiserver-identity=testserver", |
| 106 |
leases: []*v1.Lease{ |
| 107 |
{ |
| 108 |
ObjectMeta: metav1.ObjectMeta{ |
| 109 |
Name: "remote-1", |
| 110 |
Labels: map[string]string{"apiserver-identity": "testserver"}, |
| 111 |
}, |
| 112 |
Spec: v1.LeaseSpec{HolderIdentity: proto.String("holder-1")}, |
| 113 |
}, |
| 114 |
}, |
| 115 |
updatedLease: &v1.Lease{ |
| 116 |
ObjectMeta: metav1.ObjectMeta{ |
| 117 |
Name: "remote-1", |
| 118 |
Labels: map[string]string{"apiserver-identity": "testserver"}, |
| 119 |
}, |
| 120 |
Spec: v1.LeaseSpec{HolderIdentity: proto.String("holder-2")}, |
| 121 |
}, |
| 122 |
wantCache: map[string]map[schema.GroupVersionResource]bool{ |
| 123 |
"remote-1": { |
| 124 |
{Group: "testgroup", Version: "v1", Resource: "testresources"}: true, |
| 125 |
}, |
| 126 |
}, |
| 127 |
}, |
| 128 |
{ |
| 129 |
desc: "lease deletion", |
| 130 |
labelSelectorString: "apiserver-identity=testserver", |
| 131 |
leases: []*v1.Lease{ |
| 132 |
{ |
| 133 |
ObjectMeta: metav1.ObjectMeta{ |
| 134 |
Name: "remote-1", |
| 135 |
Labels: map[string]string{"apiserver-identity": "testserver"}, |
| 136 |
}, |
| 137 |
Spec: v1.LeaseSpec{HolderIdentity: proto.String("holder-1")}, |
| 138 |
}, |
| 139 |
}, |
| 140 |
deletedLeaseNames: []string{"remote-1"}, |
| 141 |
wantCache: map[string]map[schema.GroupVersionResource]bool{}, |
| 142 |
}, |
| 143 |
} |
| 144 |
|
| 145 |
for _, tt := range testCases { |
| 146 |
t.Run(tt.desc, func(t *testing.T) { |
| 147 |
fakeClient := fake.NewSimpleClientset() |
| 148 |
fakeInformerFactory := informers.NewSharedInformerFactory(fakeClient, 0) |
| 149 |
leaseInformer := fakeInformerFactory.Coordination().V1().Leases() |
| 150 |
|
| 151 |
fakeReconciler := newFakeReconciler() |
| 152 |
|
| 153 |
negotiatedSerializer := serializer.NewCodecFactory(runtime.NewScheme()) |
| 154 |
loopbackConfig := &rest.Config{} |
| 155 |
proxyConfig := &transport.Config{ |
| 156 |
TLS: transport.TLSConfig{Insecure: true}, |
| 157 |
} |
| 158 |
|
| 159 |
h, err := NewPeerProxyHandler( |
| 160 |
localServerID, |
| 161 |
tt.labelSelectorString, |
| 162 |
leaseInformer, |
| 163 |
fakeReconciler, |
| 164 |
negotiatedSerializer, |
| 165 |
loopbackConfig, |
| 166 |
proxyConfig, |
| 167 |
) |
| 168 |
if err != nil { |
| 169 |
t.Fatalf("failed to create handler: %v", err) |
| 170 |
} |
| 171 |
|
| 172 |
ctx, cancel := context.WithCancel(context.Background()) |
| 173 |
defer cancel() |
| 174 |
|
| 175 |
// Add leases to the fake client and informer. |
| 176 |
for _, lease := range tt.leases { |
| 177 |
_, err := fakeClient.CoordinationV1().Leases("default").Create(ctx, lease, metav1.CreateOptions{}) |
| 178 |
if err != nil { |
| 179 |
t.Fatalf("failed to create lease: %v", err) |
| 180 |
} |
| 181 |
if err = leaseInformer.Informer().GetIndexer().Add(lease); err != nil { |
| 182 |
t.Fatalf("failed to create lease: %v", err) |
| 183 |
} |
| 184 |
} |
| 185 |
|
| 186 |
go fakeInformerFactory.Start(ctx.Done()) |
| 187 |
cache.WaitForCacheSync(ctx.Done(), leaseInformer.Informer().HasSynced) |
| 188 |
|
| 189 |
// Create test servers based on leases |
| 190 |
testServers := make(map[string]*httptest.Server) |
| 191 |
for _, lease := range tt.leases { |
| 192 |
testServer := newTestTLSServer(t) |
| 193 |
defer testServer.Close() |
| 194 |
testServers[lease.Name] = testServer |
| 195 |
} |
| 196 |
|
| 197 |
// Modify the reconciler to return the test server URLs |
| 198 |
for name, server := range testServers { |
| 199 |
fakeReconciler.setEndpoint(name, server.URL[8:]) |
| 200 |
} |
| 201 |
|
| 202 |
go h.RunPeerDiscoveryCacheSync(ctx, 1) |
| 203 |
|
| 204 |
// Wait for initial cache update. |
| 205 |
initialCache := map[string]map[schema.GroupVersionResource]bool{} |
| 206 |
for _, lease := range tt.leases { |
| 207 |
initialCache[lease.Name] = map[schema.GroupVersionResource]bool{ |
| 208 |
{Group: "testgroup", Version: "v1", Resource: "testresources"}: true, |
| 209 |
} |
| 210 |
} |
| 211 |
err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 5*time.Second, false, func(ctx context.Context) (bool, error) { |
| 212 |
select { |
| 213 |
case <-ctx.Done(): |
| 214 |
return false, ctx.Err() |
| 215 |
default: |
| 216 |
} |
| 217 |
gotCache := h.peerDiscoveryInfoCache.Load() |
| 218 |
return assert.ObjectsAreEqual(initialCache, gotCache), nil |
| 219 |
}) |
| 220 |
if err != nil { |
| 221 |
t.Errorf("initial cache update failed: %v", err) |
| 222 |
} |
| 223 |
|
| 224 |
// Update the lease if indicated. |
| 225 |
if tt.updatedLease != nil { |
| 226 |
updatedLease := tt.updatedLease.DeepCopy() |
| 227 |
_, err = fakeClient.CoordinationV1().Leases("default").Update(ctx, updatedLease, metav1.UpdateOptions{}) |
| 228 |
if err != nil { |
| 229 |
t.Fatalf("failed to update lease: %v", err) |
| 230 |
} |
| 231 |
if err = leaseInformer.Informer().GetIndexer().Update(updatedLease); err != nil { |
| 232 |
t.Fatalf("failed to update lease: %v", err) |
| 233 |
} |
| 234 |
} |
| 235 |
|
| 236 |
// Delete leases if indicated. |
| 237 |
if len(tt.deletedLeaseNames) > 0 { |
| 238 |
for _, leaseName := range tt.deletedLeaseNames { |
| 239 |
lease, exists, err := leaseInformer.Informer().GetIndexer().GetByKey("default/" + leaseName) |
| 240 |
if err != nil { |
| 241 |
t.Fatalf("failed to get lease from indexer: %v", err) |
| 242 |
} |
| 243 |
if !exists { |
| 244 |
t.Fatalf("lease %s not found", leaseName) |
| 245 |
} |
| 246 |
deletedLease := lease.(*v1.Lease) |
| 247 |
err = fakeClient.CoordinationV1().Leases("default").Delete(ctx, deletedLease.Name, metav1.DeleteOptions{}) |
| 248 |
if err != nil { |
| 249 |
t.Fatalf("failed to delete lease: %v", err) |
| 250 |
} |
| 251 |
if err = leaseInformer.Informer().GetIndexer().Delete(deletedLease); err != nil { |
| 252 |
t.Fatalf("failed to delete lease: %v", err) |
| 253 |
} |
| 254 |
|
| 255 |
} |
| 256 |
} |
| 257 |
|
| 258 |
err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 5*time.Second, false, func(ctx context.Context) (bool, error) { |
| 259 |
select { |
| 260 |
case <-ctx.Done(): |
| 261 |
return false, ctx.Err() |
| 262 |
default: |
| 263 |
} |
| 264 |
gotCache := h.peerDiscoveryInfoCache.Load() |
| 265 |
return assert.ObjectsAreEqual(tt.wantCache, gotCache), nil |
| 266 |
}) |
| 267 |
if err != nil { |
| 268 |
t.Errorf("cache doesnt match expectation: %v", err) |
| 269 |
} |
| 270 |
|
| 271 |
}) |
| 272 |
} |
| 273 |
} |
| 274 |
|
| 275 |
// newTestTLSServer creates a new httptest.NewTLSServer that serves discovery endpoints. |
| 276 |
func newTestTLSServer(t *testing.T) *httptest.Server { |
| 277 |
return httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 278 |
if r.URL.Path == "/apis" || r.URL.Path == "/api" { |
| 279 |
discoveryResponse := &apidiscoveryv2.APIGroupDiscoveryList{ |
| 280 |
Items: []apidiscoveryv2.APIGroupDiscovery{ |
| 281 |
{ |
| 282 |
ObjectMeta: metav1.ObjectMeta{ |
| 283 |
Name: "testgroup", |
| 284 |
}, |
| 285 |
Versions: []apidiscoveryv2.APIVersionDiscovery{ |
| 286 |
{ |
| 287 |
Version: "v1", |
| 288 |
Resources: []apidiscoveryv2.APIResourceDiscovery{ |
| 289 |
{Resource: "testresources"}, |
| 290 |
}, |
| 291 |
}, |
| 292 |
}, |
| 293 |
}, |
| 294 |
}, |
| 295 |
} |
| 296 |
w.Header().Set("Content-Type", "application/json") |
| 297 |
if err := json.NewEncoder(w).Encode(discoveryResponse); err != nil { |
| 298 |
t.Fatalf("error recording discovery response") |
| 299 |
} |
| 300 |
} else { |
| 301 |
w.WriteHeader(http.StatusNotFound) |
| 302 |
} |
| 303 |
})) |
| 304 |
} |
| 305 |
|
| 306 |
type fakeReconciler struct { |
| 307 |
endpoints map[string]string |
| 308 |
} |
| 309 |
|
| 310 |
func newFakeReconciler() *fakeReconciler { |
| 311 |
return &fakeReconciler{ |
| 312 |
endpoints: make(map[string]string), |
| 313 |
} |
| 314 |
} |
| 315 |
|
| 316 |
func (f *fakeReconciler) UpdateLease(serverID string, publicIP string, ports []corev1.EndpointPort) error { |
| 317 |
return nil |
| 318 |
} |
| 319 |
|
| 320 |
func (f *fakeReconciler) DeleteLease(serverID string) error { |
| 321 |
return nil |
| 322 |
} |
| 323 |
|
| 324 |
func (f *fakeReconciler) Destroy() { |
| 325 |
} |
| 326 |
|
| 327 |
func (f *fakeReconciler) GetEndpoint(serverID string) (string, error) { |
| 328 |
endpoint, ok := f.endpoints[serverID] |
| 329 |
if !ok { |
| 330 |
return "", fmt.Errorf("endpoint not found for serverID: %s", serverID) |
| 331 |
} |
| 332 |
return endpoint, nil |
| 333 |
} |
| 334 |
|
| 335 |
func (f *fakeReconciler) RemoveLease(serverID string) error { |
| 336 |
return nil |
| 337 |
} |
| 338 |
|
| 339 |
func (f *fakeReconciler) StopReconciling() { |
| 340 |
} |
| 341 |
|
| 342 |
func (f *fakeReconciler) setEndpoint(serverID, endpoint string) { |
| 343 |
f.endpoints[serverID] = endpoint |
| 344 |
} |