| 1 |
/* |
| 2 |
Copyright 2017 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package webhook |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"crypto/tls" |
| 22 |
"crypto/x509" |
| 23 |
"encoding/json" |
| 24 |
"errors" |
| 25 |
"fmt" |
| 26 |
"net" |
| 27 |
"net/http" |
| 28 |
"net/http/httptest" |
| 29 |
"net/url" |
| 30 |
"os" |
| 31 |
"path/filepath" |
| 32 |
"regexp" |
| 33 |
"strings" |
| 34 |
"testing" |
| 35 |
"time" |
| 36 |
|
| 37 |
apierrors "k8s.io/apimachinery/pkg/api/errors" |
| 38 |
"k8s.io/apimachinery/pkg/runtime" |
| 39 |
"k8s.io/apimachinery/pkg/runtime/schema" |
| 40 |
"k8s.io/apimachinery/pkg/util/wait" |
| 41 |
"k8s.io/client-go/kubernetes/scheme" |
| 42 |
"k8s.io/client-go/rest" |
| 43 |
v1 "k8s.io/client-go/tools/clientcmd/api/v1" |
| 44 |
"k8s.io/component-base/metrics" |
| 45 |
"k8s.io/component-base/metrics/legacyregistry" |
| 46 |
) |
| 47 |
|
| 48 |
const ( |
| 49 |
errBadCertificate = "Get .*: remote error: tls: (bad certificate|unknown certificate authority)" |
| 50 |
errNoConfiguration = "invalid configuration: no configuration has been provided" |
| 51 |
errMissingCertPath = "invalid configuration: unable to read %s %s for %s due to open %s: .*" |
| 52 |
errSignedByUnknownCA = "Get .*: x509: .*(unknown authority|not standards compliant|not trusted)" |
| 53 |
) |
| 54 |
|
| 55 |
var ( |
| 56 |
defaultCluster = v1.NamedCluster{ |
| 57 |
Cluster: v1.Cluster{ |
| 58 |
Server: "https://webhook.example.com", |
| 59 |
CertificateAuthorityData: caCert, |
| 60 |
}, |
| 61 |
} |
| 62 |
defaultUser = v1.NamedAuthInfo{ |
| 63 |
AuthInfo: v1.AuthInfo{ |
| 64 |
ClientCertificateData: clientCert, |
| 65 |
ClientKeyData: clientKey, |
| 66 |
}, |
| 67 |
} |
| 68 |
namedCluster = v1.NamedCluster{ |
| 69 |
Cluster: v1.Cluster{ |
| 70 |
Server: "https://webhook.example.com", |
| 71 |
CertificateAuthorityData: caCert, |
| 72 |
}, |
| 73 |
Name: "test-cluster", |
| 74 |
} |
| 75 |
groupVersions = []schema.GroupVersion{} |
| 76 |
retryBackoff = DefaultRetryBackoffWithInitialDelay(time.Duration(500) * time.Millisecond) |
| 77 |
) |
| 78 |
|
| 79 |
// TestKubeConfigFile ensures that a kube config file, regardless of validity, is handled properly |
| 80 |
func TestKubeConfigFile(t *testing.T) { |
| 81 |
badCAPath := "/tmp/missing/ca.pem" |
| 82 |
badClientCertPath := "/tmp/missing/client.pem" |
| 83 |
badClientKeyPath := "/tmp/missing/client-key.pem" |
| 84 |
dir := bootstrapTestDir(t) |
| 85 |
|
| 86 |
defer os.RemoveAll(dir) |
| 87 |
|
| 88 |
// These tests check for all of the ways in which a Kubernetes config file could be malformed within the context of |
| 89 |
// configuring a webhook. Configuration issues that arise while using the webhook are tested elsewhere. |
| 90 |
tests := []struct { |
| 91 |
test string |
| 92 |
cluster *v1.NamedCluster |
| 93 |
context *v1.NamedContext |
| 94 |
currentContext string |
| 95 |
user *v1.NamedAuthInfo |
| 96 |
errRegex string |
| 97 |
}{ |
| 98 |
{ |
| 99 |
test: "missing context (no default, none specified)", |
| 100 |
cluster: &namedCluster, |
| 101 |
errRegex: errNoConfiguration, |
| 102 |
}, |
| 103 |
{ |
| 104 |
test: "missing context (specified context is missing)", |
| 105 |
cluster: &namedCluster, |
| 106 |
errRegex: errNoConfiguration, |
| 107 |
}, |
| 108 |
{ |
| 109 |
test: "context without cluster", |
| 110 |
context: &v1.NamedContext{ |
| 111 |
Context: v1.Context{}, |
| 112 |
Name: "testing-context", |
| 113 |
}, |
| 114 |
currentContext: "testing-context", |
| 115 |
errRegex: errNoConfiguration, |
| 116 |
}, |
| 117 |
{ |
| 118 |
test: "context without user", |
| 119 |
cluster: &namedCluster, |
| 120 |
context: &v1.NamedContext{ |
| 121 |
Context: v1.Context{ |
| 122 |
Cluster: namedCluster.Name, |
| 123 |
}, |
| 124 |
Name: "testing-context", |
| 125 |
}, |
| 126 |
currentContext: "testing-context", |
| 127 |
errRegex: "", // Not an error at parse time, only when using the webhook |
| 128 |
}, |
| 129 |
{ |
| 130 |
test: "context with missing cluster", |
| 131 |
cluster: &namedCluster, |
| 132 |
context: &v1.NamedContext{ |
| 133 |
Context: v1.Context{ |
| 134 |
Cluster: "missing-cluster", |
| 135 |
}, |
| 136 |
Name: "fake", |
| 137 |
}, |
| 138 |
errRegex: errNoConfiguration, |
| 139 |
}, |
| 140 |
{ |
| 141 |
test: "context with missing user", |
| 142 |
cluster: &namedCluster, |
| 143 |
context: &v1.NamedContext{ |
| 144 |
Context: v1.Context{ |
| 145 |
Cluster: namedCluster.Name, |
| 146 |
AuthInfo: "missing-user", |
| 147 |
}, |
| 148 |
Name: "testing-context", |
| 149 |
}, |
| 150 |
currentContext: "testing-context", |
| 151 |
errRegex: "", // Not an error at parse time, only when using the webhook |
| 152 |
}, |
| 153 |
{ |
| 154 |
test: "cluster with invalid CA certificate path", |
| 155 |
cluster: &v1.NamedCluster{ |
| 156 |
Cluster: v1.Cluster{ |
| 157 |
Server: namedCluster.Cluster.Server, |
| 158 |
CertificateAuthority: badCAPath, |
| 159 |
}, |
| 160 |
}, |
| 161 |
user: &defaultUser, |
| 162 |
errRegex: fmt.Sprintf(errMissingCertPath, "certificate-authority", badCAPath, "", badCAPath), |
| 163 |
}, |
| 164 |
{ |
| 165 |
test: "cluster with invalid CA certificate", |
| 166 |
cluster: &v1.NamedCluster{ |
| 167 |
Cluster: v1.Cluster{ |
| 168 |
Server: namedCluster.Cluster.Server, |
| 169 |
CertificateAuthorityData: caKey, // pretend user put caKey here instead of caCert |
| 170 |
}, |
| 171 |
}, |
| 172 |
user: &defaultUser, |
| 173 |
errRegex: "unable to load root certificates: no valid certificate authority data seen", |
| 174 |
}, |
| 175 |
{ |
| 176 |
test: "cluster with invalid CA certificate - no PEM", |
| 177 |
cluster: &v1.NamedCluster{ |
| 178 |
Cluster: v1.Cluster{ |
| 179 |
Server: namedCluster.Cluster.Server, |
| 180 |
CertificateAuthorityData: []byte(`not a cert`), |
| 181 |
}, |
| 182 |
}, |
| 183 |
user: &defaultUser, |
| 184 |
errRegex: "unable to load root certificates: unable to parse bytes as PEM block", |
| 185 |
}, |
| 186 |
{ |
| 187 |
test: "cluster with invalid CA certificate - parse error", |
| 188 |
cluster: &v1.NamedCluster{ |
| 189 |
Cluster: v1.Cluster{ |
| 190 |
Server: namedCluster.Cluster.Server, |
| 191 |
CertificateAuthorityData: []byte(` |
| 192 |
-----BEGIN CERTIFICATE----- |
| 193 |
MIIDGTCCAgGgAwIBAgIUOS2M |
| 194 |
-----END CERTIFICATE----- |
| 195 |
`), |
| 196 |
}, |
| 197 |
}, |
| 198 |
user: &defaultUser, |
| 199 |
errRegex: "unable to load root certificates: failed to parse certificate: (asn1: syntax error: data truncated|x509: malformed certificate)", |
| 200 |
}, |
| 201 |
{ |
| 202 |
test: "user with invalid client certificate path", |
| 203 |
cluster: &defaultCluster, |
| 204 |
user: &v1.NamedAuthInfo{ |
| 205 |
AuthInfo: v1.AuthInfo{ |
| 206 |
ClientCertificate: badClientCertPath, |
| 207 |
ClientKeyData: defaultUser.AuthInfo.ClientKeyData, |
| 208 |
}, |
| 209 |
}, |
| 210 |
errRegex: fmt.Sprintf(errMissingCertPath, "client-cert", badClientCertPath, "", badClientCertPath), |
| 211 |
}, |
| 212 |
{ |
| 213 |
test: "user with invalid client certificate", |
| 214 |
cluster: &defaultCluster, |
| 215 |
user: &v1.NamedAuthInfo{ |
| 216 |
AuthInfo: v1.AuthInfo{ |
| 217 |
ClientCertificateData: clientKey, |
| 218 |
ClientKeyData: defaultUser.AuthInfo.ClientKeyData, |
| 219 |
}, |
| 220 |
}, |
| 221 |
errRegex: "tls: failed to find certificate PEM data in certificate input, but did find a private key; PEM inputs may have been switched", |
| 222 |
}, |
| 223 |
{ |
| 224 |
test: "user with invalid client certificate path", |
| 225 |
cluster: &defaultCluster, |
| 226 |
user: &v1.NamedAuthInfo{ |
| 227 |
AuthInfo: v1.AuthInfo{ |
| 228 |
ClientCertificateData: defaultUser.AuthInfo.ClientCertificateData, |
| 229 |
ClientKey: badClientKeyPath, |
| 230 |
}, |
| 231 |
}, |
| 232 |
errRegex: fmt.Sprintf(errMissingCertPath, "client-key", badClientKeyPath, "", badClientKeyPath), |
| 233 |
}, |
| 234 |
{ |
| 235 |
test: "user with invalid client certificate", |
| 236 |
cluster: &defaultCluster, |
| 237 |
user: &v1.NamedAuthInfo{ |
| 238 |
AuthInfo: v1.AuthInfo{ |
| 239 |
ClientCertificateData: defaultUser.AuthInfo.ClientCertificateData, |
| 240 |
ClientKeyData: clientCert, |
| 241 |
}, |
| 242 |
}, |
| 243 |
errRegex: "tls: found a certificate rather than a key in the PEM for the private key", |
| 244 |
}, |
| 245 |
{ |
| 246 |
test: "valid configuration (certificate data embedded in config)", |
| 247 |
cluster: &defaultCluster, |
| 248 |
user: &defaultUser, |
| 249 |
errRegex: "", |
| 250 |
}, |
| 251 |
{ |
| 252 |
test: "valid configuration (certificate files referenced in config)", |
| 253 |
cluster: &v1.NamedCluster{ |
| 254 |
Cluster: v1.Cluster{ |
| 255 |
Server: "https://webhook.example.com", |
| 256 |
CertificateAuthority: filepath.Join(dir, "ca.pem"), |
| 257 |
}, |
| 258 |
}, |
| 259 |
user: &v1.NamedAuthInfo{ |
| 260 |
AuthInfo: v1.AuthInfo{ |
| 261 |
ClientCertificate: filepath.Join(dir, "client.pem"), |
| 262 |
ClientKey: filepath.Join(dir, "client-key.pem"), |
| 263 |
}, |
| 264 |
}, |
| 265 |
errRegex: "", |
| 266 |
}, |
| 267 |
} |
| 268 |
|
| 269 |
for _, tt := range tests { |
| 270 |
t.Run(tt.test, func(t *testing.T) { |
| 271 |
// Use a closure so defer statements trigger between loop iterations. |
| 272 |
err := func() error { |
| 273 |
kubeConfig := v1.Config{} |
| 274 |
|
| 275 |
if tt.cluster != nil { |
| 276 |
kubeConfig.Clusters = []v1.NamedCluster{*tt.cluster} |
| 277 |
} |
| 278 |
|
| 279 |
if tt.context != nil { |
| 280 |
kubeConfig.Contexts = []v1.NamedContext{*tt.context} |
| 281 |
} |
| 282 |
|
| 283 |
if tt.user != nil { |
| 284 |
kubeConfig.AuthInfos = []v1.NamedAuthInfo{*tt.user} |
| 285 |
} |
| 286 |
|
| 287 |
kubeConfig.CurrentContext = tt.currentContext |
| 288 |
|
| 289 |
kubeConfigFile, err := newKubeConfigFile(kubeConfig) |
| 290 |
if err != nil { |
| 291 |
return err |
| 292 |
} |
| 293 |
|
| 294 |
defer os.Remove(kubeConfigFile) |
| 295 |
|
| 296 |
config, err := LoadKubeconfig(kubeConfigFile, nil) |
| 297 |
if err != nil { |
| 298 |
return err |
| 299 |
} |
| 300 |
|
| 301 |
_, err = NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, config, groupVersions, retryBackoff) |
| 302 |
return err |
| 303 |
}() |
| 304 |
|
| 305 |
if err == nil { |
| 306 |
if tt.errRegex != "" { |
| 307 |
t.Errorf("%s: expected an error", tt.test) |
| 308 |
} |
| 309 |
} else { |
| 310 |
if tt.errRegex == "" { |
| 311 |
t.Errorf("%s: unexpected error: %v", tt.test, err) |
| 312 |
} else if !regexp.MustCompile(tt.errRegex).MatchString(err.Error()) { |
| 313 |
t.Errorf("%s: unexpected error message to match:\n Expected: %s\n Actual: %s", tt.test, tt.errRegex, err.Error()) |
| 314 |
} |
| 315 |
} |
| 316 |
}) |
| 317 |
} |
| 318 |
} |
| 319 |
|
| 320 |
// TestMissingKubeConfigFile ensures that a kube config path to a missing file is handled properly |
| 321 |
func TestMissingKubeConfigFile(t *testing.T) { |
| 322 |
kubeConfigPath := "/some/missing/path" |
| 323 |
_, err := LoadKubeconfig(kubeConfigPath, nil) |
| 324 |
|
| 325 |
if err == nil { |
| 326 |
t.Errorf("creating the webhook should had failed") |
| 327 |
} else if strings.Index(err.Error(), fmt.Sprintf("stat %s", kubeConfigPath)) != 0 { |
| 328 |
t.Errorf("unexpected error: %v", err) |
| 329 |
} |
| 330 |
} |
| 331 |
|
| 332 |
// TestTLSConfig ensures that the TLS-based communication between client and server works as expected |
| 333 |
func TestTLSConfig(t *testing.T) { |
| 334 |
invalidCert := []byte("invalid") |
| 335 |
tests := []struct { |
| 336 |
test string |
| 337 |
clientCert, clientKey, clientCA []byte |
| 338 |
serverCert, serverKey, serverCA []byte |
| 339 |
errRegex string |
| 340 |
increaseSANWarnCounter bool |
| 341 |
increaseSHA1SignatureWarnCounter bool |
| 342 |
}{ |
| 343 |
{ |
| 344 |
test: "invalid server CA", |
| 345 |
clientCert: clientCert, clientKey: clientKey, clientCA: caCert, |
| 346 |
serverCert: serverCert, serverKey: serverKey, serverCA: invalidCert, |
| 347 |
errRegex: errBadCertificate, |
| 348 |
}, |
| 349 |
{ |
| 350 |
test: "invalid client certificate", |
| 351 |
clientCert: invalidCert, clientKey: clientKey, clientCA: caCert, |
| 352 |
serverCert: serverCert, serverKey: serverKey, serverCA: caCert, |
| 353 |
errRegex: "tls: failed to find any PEM data in certificate input", |
| 354 |
}, |
| 355 |
{ |
| 356 |
test: "invalid client key", |
| 357 |
clientCert: clientCert, clientKey: invalidCert, clientCA: caCert, |
| 358 |
serverCert: serverCert, serverKey: serverKey, serverCA: caCert, |
| 359 |
errRegex: "tls: failed to find any PEM data in key input", |
| 360 |
}, |
| 361 |
{ |
| 362 |
test: "client does not trust server", |
| 363 |
clientCert: clientCert, clientKey: clientKey, |
| 364 |
serverCert: serverCert, serverKey: serverKey, |
| 365 |
errRegex: errSignedByUnknownCA, |
| 366 |
}, |
| 367 |
{ |
| 368 |
test: "server does not trust client", |
| 369 |
clientCert: clientCert, clientKey: clientKey, clientCA: badCACert, |
| 370 |
serverCert: serverCert, serverKey: serverKey, serverCA: caCert, |
| 371 |
errRegex: errSignedByUnknownCA + " .*", |
| 372 |
}, |
| 373 |
{ |
| 374 |
test: "server requires auth, client provides it", |
| 375 |
clientCert: clientCert, clientKey: clientKey, clientCA: caCert, |
| 376 |
serverCert: serverCert, serverKey: serverKey, serverCA: caCert, |
| 377 |
errRegex: "", |
| 378 |
}, |
| 379 |
{ |
| 380 |
test: "server does not require client auth", |
| 381 |
clientCA: caCert, |
| 382 |
serverCert: serverCert, serverKey: serverKey, |
| 383 |
errRegex: "", |
| 384 |
}, |
| 385 |
{ |
| 386 |
test: "server does not require client auth, client provides it", |
| 387 |
clientCert: clientCert, clientKey: clientKey, clientCA: caCert, |
| 388 |
serverCert: serverCert, serverKey: serverKey, |
| 389 |
errRegex: "", |
| 390 |
}, |
| 391 |
{ |
| 392 |
test: "webhook does not support insecure servers", |
| 393 |
serverCert: serverCert, serverKey: serverKey, |
| 394 |
errRegex: errSignedByUnknownCA, |
| 395 |
}, |
| 396 |
{ |
| 397 |
// this will fail when GODEBUG is set to x509ignoreCN=0 with |
| 398 |
// expected err, but the SAN counter gets increased |
| 399 |
test: "server cert does not have SAN extension", |
| 400 |
clientCA: caCert, |
| 401 |
serverCert: serverCertNoSAN, serverKey: serverKey, |
| 402 |
errRegex: "x509: certificate relies on legacy Common Name field", |
| 403 |
increaseSANWarnCounter: true, |
| 404 |
}, |
| 405 |
{ |
| 406 |
test: "server cert with SHA1 signature", |
| 407 |
clientCA: caCert, |
| 408 |
serverCert: append(append(sha1ServerCertInter, byte('\n')), caCertInter...), serverKey: serverKey, |
| 409 |
errRegex: "x509: cannot verify signature: insecure algorithm SHA1-RSA", |
| 410 |
increaseSHA1SignatureWarnCounter: true, |
| 411 |
}, |
| 412 |
{ |
| 413 |
test: "server cert signed by an intermediate CA with SHA1 signature", |
| 414 |
clientCA: caCert, |
| 415 |
serverCert: append(append(serverCertInterSHA1, byte('\n')), caCertInterSHA1...), serverKey: serverKey, |
| 416 |
errRegex: "x509: cannot verify signature: insecure algorithm SHA1-RSA", |
| 417 |
increaseSHA1SignatureWarnCounter: true, |
| 418 |
}, |
| 419 |
} |
| 420 |
|
| 421 |
lastSHA1SigCounter := 0 |
| 422 |
for _, tt := range tests { |
| 423 |
// Use a closure so defer statements trigger between loop iterations. |
| 424 |
func() { |
| 425 |
// Create and start a simple HTTPS server |
| 426 |
server, err := newTestServer(tt.serverCert, tt.serverKey, tt.serverCA, nil) |
| 427 |
if err != nil { |
| 428 |
t.Errorf("%s: failed to create server: %v", tt.test, err) |
| 429 |
return |
| 430 |
} |
| 431 |
|
| 432 |
serverURL, err := url.Parse(server.URL) |
| 433 |
if err != nil { |
| 434 |
t.Errorf("%s: failed to parse the testserver URL: %v", tt.test, err) |
| 435 |
return |
| 436 |
} |
| 437 |
serverURL.Host = net.JoinHostPort("localhost", serverURL.Port()) |
| 438 |
|
| 439 |
defer server.Close() |
| 440 |
|
| 441 |
// Create a Kubernetes client configuration file |
| 442 |
configFile, err := newKubeConfigFile(v1.Config{ |
| 443 |
Clusters: []v1.NamedCluster{ |
| 444 |
{ |
| 445 |
Cluster: v1.Cluster{ |
| 446 |
Server: serverURL.String(), |
| 447 |
CertificateAuthorityData: tt.clientCA, |
| 448 |
}, |
| 449 |
}, |
| 450 |
}, |
| 451 |
AuthInfos: []v1.NamedAuthInfo{ |
| 452 |
{ |
| 453 |
AuthInfo: v1.AuthInfo{ |
| 454 |
ClientCertificateData: tt.clientCert, |
| 455 |
ClientKeyData: tt.clientKey, |
| 456 |
}, |
| 457 |
}, |
| 458 |
}, |
| 459 |
}) |
| 460 |
|
| 461 |
if err != nil { |
| 462 |
t.Errorf("%s: %v", tt.test, err) |
| 463 |
return |
| 464 |
} |
| 465 |
|
| 466 |
defer os.Remove(configFile) |
| 467 |
|
| 468 |
config, err := LoadKubeconfig(configFile, nil) |
| 469 |
if err != nil { |
| 470 |
t.Fatal(err) |
| 471 |
} |
| 472 |
|
| 473 |
wh, err := NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, config, groupVersions, retryBackoff) |
| 474 |
|
| 475 |
if err == nil { |
| 476 |
err = wh.RestClient.Get().Do(context.TODO()).Error() |
| 477 |
} |
| 478 |
|
| 479 |
if err == nil { |
| 480 |
if tt.errRegex != "" { |
| 481 |
t.Errorf("%s: expected an error", tt.test) |
| 482 |
} |
| 483 |
} else { |
| 484 |
if tt.errRegex == "" { |
| 485 |
t.Errorf("%s: unexpected error: %v", tt.test, err) |
| 486 |
} else if !regexp.MustCompile(tt.errRegex).MatchString(err.Error()) { |
| 487 |
t.Errorf("%s: unexpected error message mismatch:\n Expected: %s\n Actual: %s", tt.test, tt.errRegex, err.Error()) |
| 488 |
} |
| 489 |
} |
| 490 |
|
| 491 |
if tt.increaseSANWarnCounter { |
| 492 |
errorCounter := getSingleCounterValueFromRegistry(t, legacyregistry.DefaultGatherer, "apiserver_webhooks_x509_missing_san_total") |
| 493 |
|
| 494 |
if errorCounter == -1 { |
| 495 |
t.Errorf("failed to get the x509_common_name_error_count metrics: %v", err) |
| 496 |
} |
| 497 |
if int(errorCounter) != 1 { |
| 498 |
t.Errorf("expected the x509_common_name_error_count to be 1, but it's %d", errorCounter) |
| 499 |
} |
| 500 |
} |
| 501 |
|
| 502 |
if tt.increaseSHA1SignatureWarnCounter { |
| 503 |
errorCounter := getSingleCounterValueFromRegistry(t, legacyregistry.DefaultGatherer, "apiserver_webhooks_x509_insecure_sha1_total") |
| 504 |
|
| 505 |
if errorCounter == -1 { |
| 506 |
t.Errorf("failed to get the apiserver_webhooks_x509_insecure_sha1_total metrics: %v", err) |
| 507 |
} |
| 508 |
|
| 509 |
if int(errorCounter) != lastSHA1SigCounter+1 { |
| 510 |
t.Errorf("expected the apiserver_webhooks_x509_insecure_sha1_total counter to be 1, but it's %d", errorCounter) |
| 511 |
} |
| 512 |
|
| 513 |
lastSHA1SigCounter++ |
| 514 |
} |
| 515 |
}() |
| 516 |
} |
| 517 |
} |
| 518 |
|
| 519 |
func TestRequestTimeout(t *testing.T) { |
| 520 |
done := make(chan struct{}) |
| 521 |
|
| 522 |
handler := func(w http.ResponseWriter, r *http.Request) { |
| 523 |
<-done |
| 524 |
} |
| 525 |
|
| 526 |
// Create and start a simple HTTPS server |
| 527 |
server, err := newTestServer(clientCert, clientKey, caCert, handler) |
| 528 |
if err != nil { |
| 529 |
t.Errorf("failed to create server: %v", err) |
| 530 |
return |
| 531 |
} |
| 532 |
defer server.Close() |
| 533 |
defer close(done) // done channel must be closed before server is. |
| 534 |
|
| 535 |
// Create a Kubernetes client configuration file |
| 536 |
configFile, err := newKubeConfigFile(v1.Config{ |
| 537 |
Clusters: []v1.NamedCluster{ |
| 538 |
{ |
| 539 |
Cluster: v1.Cluster{ |
| 540 |
Server: server.URL, |
| 541 |
CertificateAuthorityData: caCert, |
| 542 |
}, |
| 543 |
}, |
| 544 |
}, |
| 545 |
AuthInfos: []v1.NamedAuthInfo{ |
| 546 |
{ |
| 547 |
AuthInfo: v1.AuthInfo{ |
| 548 |
ClientCertificateData: clientCert, |
| 549 |
ClientKeyData: clientKey, |
| 550 |
}, |
| 551 |
}, |
| 552 |
}, |
| 553 |
}) |
| 554 |
if err != nil { |
| 555 |
t.Errorf("failed to create the client config file: %v", err) |
| 556 |
return |
| 557 |
} |
| 558 |
defer os.Remove(configFile) |
| 559 |
|
| 560 |
var requestTimeout = 10 * time.Millisecond |
| 561 |
|
| 562 |
config, err := LoadKubeconfig(configFile, nil) |
| 563 |
if err != nil { |
| 564 |
t.Fatal(err) |
| 565 |
} |
| 566 |
|
| 567 |
config.Timeout = requestTimeout |
| 568 |
|
| 569 |
wh, err := NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, config, groupVersions, retryBackoff) |
| 570 |
if err != nil { |
| 571 |
t.Fatalf("failed to create the webhook: %v", err) |
| 572 |
} |
| 573 |
|
| 574 |
resultCh := make(chan rest.Result) |
| 575 |
|
| 576 |
go func() { resultCh <- wh.RestClient.Get().Do(context.TODO()) }() |
| 577 |
select { |
| 578 |
case <-time.After(time.Second * 5): |
| 579 |
t.Errorf("expected request to timeout after %s", requestTimeout) |
| 580 |
case <-resultCh: |
| 581 |
} |
| 582 |
} |
| 583 |
|
| 584 |
// TestWithExponentialBackoff ensures that the webhook's exponential backoff support works as expected |
| 585 |
func TestWithExponentialBackoff(t *testing.T) { |
| 586 |
count := 0 // To keep track of the requests |
| 587 |
gr := schema.GroupResource{ |
| 588 |
Group: "webhook.util.k8s.io", |
| 589 |
Resource: "test", |
| 590 |
} |
| 591 |
|
| 592 |
// Handler that will handle all backoff CONDITIONS |
| 593 |
ebHandler := func(w http.ResponseWriter, r *http.Request) { |
| 594 |
w.Header().Set("Content-Type", "application/json") |
| 595 |
|
| 596 |
switch count++; count { |
| 597 |
case 1: |
| 598 |
// Timeout error with retry supplied |
| 599 |
w.WriteHeader(http.StatusGatewayTimeout) |
| 600 |
json.NewEncoder(w).Encode(apierrors.NewServerTimeout(gr, "get", 2)) |
| 601 |
case 2: |
| 602 |
// Internal server error |
| 603 |
w.WriteHeader(http.StatusInternalServerError) |
| 604 |
json.NewEncoder(w).Encode(apierrors.NewInternalError(fmt.Errorf("nope"))) |
| 605 |
case 3: |
| 606 |
// HTTP error that is not retryable |
| 607 |
w.WriteHeader(http.StatusNotAcceptable) |
| 608 |
json.NewEncoder(w).Encode(apierrors.NewGenericServerResponse(http.StatusNotAcceptable, "get", gr, "testing", "nope", 0, false)) |
| 609 |
case 4: |
| 610 |
// Successful request |
| 611 |
w.WriteHeader(http.StatusOK) |
| 612 |
json.NewEncoder(w).Encode(map[string]string{ |
| 613 |
"status": "OK", |
| 614 |
}) |
| 615 |
} |
| 616 |
} |
| 617 |
|
| 618 |
// Create and start a simple HTTPS server |
| 619 |
server, err := newTestServer(clientCert, clientKey, caCert, ebHandler) |
| 620 |
|
| 621 |
if err != nil { |
| 622 |
t.Errorf("failed to create server: %v", err) |
| 623 |
return |
| 624 |
} |
| 625 |
|
| 626 |
defer server.Close() |
| 627 |
|
| 628 |
// Create a Kubernetes client configuration file |
| 629 |
configFile, err := newKubeConfigFile(v1.Config{ |
| 630 |
Clusters: []v1.NamedCluster{ |
| 631 |
{ |
| 632 |
Cluster: v1.Cluster{ |
| 633 |
Server: server.URL, |
| 634 |
CertificateAuthorityData: caCert, |
| 635 |
}, |
| 636 |
}, |
| 637 |
}, |
| 638 |
AuthInfos: []v1.NamedAuthInfo{ |
| 639 |
{ |
| 640 |
AuthInfo: v1.AuthInfo{ |
| 641 |
ClientCertificateData: clientCert, |
| 642 |
ClientKeyData: clientKey, |
| 643 |
}, |
| 644 |
}, |
| 645 |
}, |
| 646 |
}) |
| 647 |
|
| 648 |
if err != nil { |
| 649 |
t.Errorf("failed to create the client config file: %v", err) |
| 650 |
return |
| 651 |
} |
| 652 |
|
| 653 |
defer os.Remove(configFile) |
| 654 |
|
| 655 |
config, err := LoadKubeconfig(configFile, nil) |
| 656 |
if err != nil { |
| 657 |
t.Fatal(err) |
| 658 |
} |
| 659 |
|
| 660 |
wh, err := NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, config, groupVersions, retryBackoff) |
| 661 |
|
| 662 |
if err != nil { |
| 663 |
t.Fatalf("failed to create the webhook: %v", err) |
| 664 |
} |
| 665 |
|
| 666 |
result := wh.WithExponentialBackoff(context.Background(), func() rest.Result { |
| 667 |
return wh.RestClient.Get().Do(context.TODO()) |
| 668 |
}) |
| 669 |
|
| 670 |
var statusCode int |
| 671 |
|
| 672 |
result.StatusCode(&statusCode) |
| 673 |
|
| 674 |
if statusCode != http.StatusNotAcceptable { |
| 675 |
t.Errorf("unexpected status code: %d", statusCode) |
| 676 |
} |
| 677 |
|
| 678 |
result = wh.WithExponentialBackoff(context.Background(), func() rest.Result { |
| 679 |
return wh.RestClient.Get().Do(context.TODO()) |
| 680 |
}) |
| 681 |
|
| 682 |
result.StatusCode(&statusCode) |
| 683 |
|
| 684 |
if statusCode != http.StatusOK { |
| 685 |
t.Errorf("unexpected status code: %d", statusCode) |
| 686 |
} |
| 687 |
} |
| 688 |
|
| 689 |
func bootstrapTestDir(t *testing.T) string { |
| 690 |
dir, err := os.MkdirTemp("", "") |
| 691 |
|
| 692 |
if err != nil { |
| 693 |
t.Fatal(err) |
| 694 |
} |
| 695 |
|
| 696 |
// The certificates needed on disk for the tests |
| 697 |
files := map[string][]byte{ |
| 698 |
"ca.pem": caCert, |
| 699 |
"client.pem": clientCert, |
| 700 |
"client-key.pem": clientKey, |
| 701 |
} |
| 702 |
|
| 703 |
// Write the certificate files to disk or fail |
| 704 |
for fileName, fileData := range files { |
| 705 |
if err := os.WriteFile(filepath.Join(dir, fileName), fileData, 0400); err != nil { |
| 706 |
os.RemoveAll(dir) |
| 707 |
t.Fatal(err) |
| 708 |
} |
| 709 |
} |
| 710 |
|
| 711 |
return dir |
| 712 |
} |
| 713 |
|
| 714 |
func newKubeConfigFile(config v1.Config) (string, error) { |
| 715 |
configFile, err := os.CreateTemp("", "") |
| 716 |
if err != nil { |
| 717 |
return "", err |
| 718 |
} |
| 719 |
defer configFile.Close() |
| 720 |
|
| 721 |
if err != nil { |
| 722 |
return "", fmt.Errorf("unable to create the Kubernetes client config file: %v", err) |
| 723 |
} |
| 724 |
|
| 725 |
if err = json.NewEncoder(configFile).Encode(config); err != nil { |
| 726 |
return "", fmt.Errorf("unable to write the Kubernetes client configuration to disk: %v", err) |
| 727 |
} |
| 728 |
|
| 729 |
return configFile.Name(), nil |
| 730 |
} |
| 731 |
|
| 732 |
func newTestServer(clientCert, clientKey, caCert []byte, handler func(http.ResponseWriter, *http.Request)) (*httptest.Server, error) { |
| 733 |
var tlsConfig *tls.Config |
| 734 |
|
| 735 |
if clientCert != nil { |
| 736 |
cert, err := tls.X509KeyPair(clientCert, clientKey) |
| 737 |
|
| 738 |
if err != nil { |
| 739 |
return nil, err |
| 740 |
} |
| 741 |
|
| 742 |
tlsConfig = &tls.Config{ |
| 743 |
Certificates: []tls.Certificate{cert}, |
| 744 |
} |
| 745 |
} |
| 746 |
|
| 747 |
if caCert != nil { |
| 748 |
rootCAs := x509.NewCertPool() |
| 749 |
|
| 750 |
rootCAs.AppendCertsFromPEM(caCert) |
| 751 |
|
| 752 |
if tlsConfig == nil { |
| 753 |
tlsConfig = &tls.Config{} |
| 754 |
} |
| 755 |
|
| 756 |
tlsConfig.ClientCAs = rootCAs |
| 757 |
tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert |
| 758 |
} |
| 759 |
|
| 760 |
if handler == nil { |
| 761 |
handler = func(w http.ResponseWriter, r *http.Request) { |
| 762 |
w.Write([]byte("OK")) |
| 763 |
} |
| 764 |
} |
| 765 |
|
| 766 |
server := httptest.NewUnstartedServer(http.HandlerFunc(handler)) |
| 767 |
|
| 768 |
server.TLS = tlsConfig |
| 769 |
server.StartTLS() |
| 770 |
|
| 771 |
return server, nil |
| 772 |
} |
| 773 |
|
| 774 |
func TestWithExponentialBackoffContextIsAlreadyCanceled(t *testing.T) { |
| 775 |
alwaysRetry := func(e error) bool { |
| 776 |
return true |
| 777 |
} |
| 778 |
|
| 779 |
attemptsGot := 0 |
| 780 |
webhookFunc := func() error { |
| 781 |
attemptsGot++ |
| 782 |
return nil |
| 783 |
} |
| 784 |
|
| 785 |
ctx, cancel := context.WithCancel(context.TODO()) |
| 786 |
cancel() |
| 787 |
|
| 788 |
// We don't expect the webhook function to be called since the context is already canceled. |
| 789 |
retryBackoff := wait.Backoff{Steps: 5} |
| 790 |
err := WithExponentialBackoff(ctx, retryBackoff, webhookFunc, alwaysRetry) |
| 791 |
|
| 792 |
errExpected := fmt.Errorf("webhook call failed: %s", context.Canceled) |
| 793 |
if errExpected.Error() != err.Error() { |
| 794 |
t.Errorf("expected error: %v, but got: %v", errExpected, err) |
| 795 |
} |
| 796 |
if attemptsGot != 0 { |
| 797 |
t.Errorf("expected %d webhook attempts, but got: %d", 0, attemptsGot) |
| 798 |
} |
| 799 |
} |
| 800 |
|
| 801 |
func TestWithExponentialBackoffWebhookErrorIsMostImportant(t *testing.T) { |
| 802 |
alwaysRetry := func(e error) bool { |
| 803 |
return true |
| 804 |
} |
| 805 |
|
| 806 |
ctx, cancel := context.WithCancel(context.TODO()) |
| 807 |
attemptsGot := 0 |
| 808 |
errExpected := errors.New("webhook not available") |
| 809 |
webhookFunc := func() error { |
| 810 |
attemptsGot++ |
| 811 |
|
| 812 |
// after the first attempt, the context is canceled |
| 813 |
cancel() |
| 814 |
|
| 815 |
return errExpected |
| 816 |
} |
| 817 |
|
| 818 |
// webhook err has higher priority than ctx error. we expect the webhook error to be returned. |
| 819 |
retryBackoff := wait.Backoff{Steps: 5} |
| 820 |
err := WithExponentialBackoff(ctx, retryBackoff, webhookFunc, alwaysRetry) |
| 821 |
|
| 822 |
if attemptsGot != 1 { |
| 823 |
t.Errorf("expected %d webhook attempts, but got: %d", 1, attemptsGot) |
| 824 |
} |
| 825 |
if errExpected != err { |
| 826 |
t.Errorf("expected error: %v, but got: %v", errExpected, err) |
| 827 |
} |
| 828 |
} |
| 829 |
|
| 830 |
func TestWithExponentialBackoffWithRetryExhaustedWhileContextIsNotCanceled(t *testing.T) { |
| 831 |
alwaysRetry := func(e error) bool { |
| 832 |
return true |
| 833 |
} |
| 834 |
|
| 835 |
ctx, cancel := context.WithCancel(context.TODO()) |
| 836 |
defer cancel() |
| 837 |
|
| 838 |
attemptsGot := 0 |
| 839 |
errExpected := errors.New("webhook not available") |
| 840 |
webhookFunc := func() error { |
| 841 |
attemptsGot++ |
| 842 |
return errExpected |
| 843 |
} |
| 844 |
|
| 845 |
// webhook err has higher priority than ctx error. we expect the webhook error to be returned. |
| 846 |
retryBackoff := wait.Backoff{Steps: 5} |
| 847 |
err := WithExponentialBackoff(ctx, retryBackoff, webhookFunc, alwaysRetry) |
| 848 |
|
| 849 |
if attemptsGot != 5 { |
| 850 |
t.Errorf("expected %d webhook attempts, but got: %d", 1, attemptsGot) |
| 851 |
} |
| 852 |
if errExpected != err { |
| 853 |
t.Errorf("expected error: %v, but got: %v", errExpected, err) |
| 854 |
} |
| 855 |
} |
| 856 |
|
| 857 |
func TestWithExponentialBackoffParametersNotSet(t *testing.T) { |
| 858 |
alwaysRetry := func(e error) bool { |
| 859 |
return true |
| 860 |
} |
| 861 |
|
| 862 |
attemptsGot := 0 |
| 863 |
webhookFunc := func() error { |
| 864 |
attemptsGot++ |
| 865 |
return nil |
| 866 |
} |
| 867 |
|
| 868 |
err := WithExponentialBackoff(context.TODO(), wait.Backoff{}, webhookFunc, alwaysRetry) |
| 869 |
|
| 870 |
errExpected := fmt.Errorf("webhook call failed: %s", wait.ErrWaitTimeout) |
| 871 |
if errExpected.Error() != err.Error() { |
| 872 |
t.Errorf("expected error: %v, but got: %v", errExpected, err) |
| 873 |
} |
| 874 |
if attemptsGot != 0 { |
| 875 |
t.Errorf("expected %d webhook attempts, but got: %d", 0, attemptsGot) |
| 876 |
} |
| 877 |
} |
| 878 |
|
| 879 |
func TestGenericWebhookWithExponentialBackoff(t *testing.T) { |
| 880 |
attemptsPerCallExpected := 5 |
| 881 |
webhook := &GenericWebhook{ |
| 882 |
RetryBackoff: wait.Backoff{ |
| 883 |
Duration: time.Millisecond, |
| 884 |
Factor: 1.5, |
| 885 |
Jitter: 0.2, |
| 886 |
Steps: attemptsPerCallExpected, |
| 887 |
}, |
| 888 |
|
| 889 |
ShouldRetry: func(e error) bool { |
| 890 |
return true |
| 891 |
}, |
| 892 |
} |
| 893 |
|
| 894 |
attemptsGot := 0 |
| 895 |
webhookFunc := func() rest.Result { |
| 896 |
attemptsGot++ |
| 897 |
return rest.Result{} |
| 898 |
} |
| 899 |
|
| 900 |
// number of retries should always be local to each call. |
| 901 |
totalAttemptsExpected := attemptsPerCallExpected * 2 |
| 902 |
webhook.WithExponentialBackoff(context.TODO(), webhookFunc) |
| 903 |
webhook.WithExponentialBackoff(context.TODO(), webhookFunc) |
| 904 |
|
| 905 |
if totalAttemptsExpected != attemptsGot { |
| 906 |
t.Errorf("expected a total of %d webhook attempts but got: %d", totalAttemptsExpected, attemptsGot) |
| 907 |
} |
| 908 |
} |
| 909 |
|
| 910 |
func getSingleCounterValueFromRegistry(t *testing.T, r metrics.Gatherer, name string) int { |
| 911 |
mfs, err := r.Gather() |
| 912 |
if err != nil { |
| 913 |
t.Logf("failed to gather local registry metrics: %v", err) |
| 914 |
return -1 |
| 915 |
} |
| 916 |
|
| 917 |
for _, mf := range mfs { |
| 918 |
if mf.Name != nil && *mf.Name == name { |
| 919 |
mfMetric := mf.GetMetric() |
| 920 |
for _, m := range mfMetric { |
| 921 |
if m.GetCounter() != nil { |
| 922 |
return int(m.GetCounter().GetValue()) |
| 923 |
} |
| 924 |
} |
| 925 |
} |
| 926 |
} |
| 927 |
|
| 928 |
return -1 |
| 929 |
} |