Debian Codegraph @

pkg/server/options/serving_test.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package options
18
19 import (
20 "bytes"
21 "context"
22 cryptorand "crypto/rand"
23 "crypto/rsa"
24 "crypto/tls"
25 "crypto/x509"
26 "crypto/x509/pkix"
27 "encoding/base64"
28 "encoding/pem"
29 "errors"
30 "fmt"
31 "io/ioutil"
32 "math/big"
33 "net"
34 "os"
35 "path/filepath"
36 "reflect"
37 "strconv"
38 "strings"
39 "testing"
40 "time"
41
42 "k8s.io/apimachinery/pkg/runtime"
43 "k8s.io/apimachinery/pkg/runtime/serializer"
44 "k8s.io/apimachinery/pkg/version"
45 "k8s.io/apiserver/pkg/server"
46 "k8s.io/client-go/discovery"
47 restclient "k8s.io/client-go/rest"
48 cliflag "k8s.io/component-base/cli/flag"
49 basecompatibility "k8s.io/component-base/compatibility"
50 baseversion "k8s.io/component-base/version"
51 "k8s.io/klog/v2/ktesting"
52 netutils "k8s.io/utils/net"
53 )
54
55 func setUp(t *testing.T) server.Config {
56 scheme := runtime.NewScheme()
57 codecs := serializer.NewCodecFactory(scheme)
58
59 config := server.NewConfig(codecs)
60
61 return *config
62 }
63
64 type TestCertSpec struct {
65 host string
66 names, ips []string // in certificate
67 }
68
69 type NamedTestCertSpec struct {
70 TestCertSpec
71 explicitNames []string // as --tls-sni-cert-key explicit names
72 }
73
74 func TestServerRunWithSNI(t *testing.T) {
75 tests := map[string]struct {
76 Cert TestCertSpec
77 SNICerts []NamedTestCertSpec
78 ExpectedCertIndex int
79
80 // passed in the client hello info, "localhost" if unset
81 ServerName string
82
83 // optional ip or hostname to pass to NewLoopbackClientConfig
84 LoopbackClientBindAddressOverride string
85 }{
86 "only one cert": {
87 Cert: TestCertSpec{
88 host: "localhost",
89 ips: []string{"127.0.0.1"},
90 },
91 ExpectedCertIndex: -1,
92 },
93 "cert with multiple alternate names": {
94 Cert: TestCertSpec{
95 host: "localhost",
96 names: []string{"test.com"},
97 ips: []string{"127.0.0.1"},
98 },
99 ExpectedCertIndex: -1,
100 ServerName: "test.com",
101 },
102 "one SNI and the default cert with the same name": {
103 Cert: TestCertSpec{
104 host: "localhost",
105 ips: []string{"127.0.0.1"},
106 },
107 SNICerts: []NamedTestCertSpec{
108 {
109 TestCertSpec: TestCertSpec{
110 host: "localhost",
111 },
112 },
113 },
114 ExpectedCertIndex: 0,
115 },
116 "matching SNI cert": {
117 Cert: TestCertSpec{
118 host: "localhost",
119 ips: []string{"127.0.0.1"},
120 },
121 SNICerts: []NamedTestCertSpec{
122 {
123 TestCertSpec: TestCertSpec{
124 host: "test.com",
125 },
126 },
127 },
128 ExpectedCertIndex: 0,
129 ServerName: "test.com",
130 },
131 "matching IP in SNI cert and the server cert": {
132 // IPs must not be passed via SNI. Hence, the ServerName in the
133 // HELLO packet is empty and the server should select the non-SNI cert.
134 Cert: TestCertSpec{
135 host: "localhost",
136 ips: []string{"10.0.0.1", "127.0.0.1"},
137 },
138 SNICerts: []NamedTestCertSpec{
139 {
140 TestCertSpec: TestCertSpec{
141 host: "test.com",
142 ips: []string{"10.0.0.1"},
143 },
144 },
145 },
146 ExpectedCertIndex: -1,
147 ServerName: "10.0.0.1",
148 },
149 "wildcards": {
150 Cert: TestCertSpec{
151 host: "localhost",
152 ips: []string{"127.0.0.1"},
153 },
154 SNICerts: []NamedTestCertSpec{
155 {
156 TestCertSpec: TestCertSpec{
157 host: "test.com",
158 names: []string{"*.test.com"},
159 },
160 },
161 },
162 ExpectedCertIndex: 0,
163 ServerName: "www.test.com",
164 },
165
166 "loopback: LoopbackClientServerNameOverride not on any cert": {
167 Cert: TestCertSpec{
168 host: "test.com",
169 },
170 SNICerts: []NamedTestCertSpec{
171 {
172 TestCertSpec: TestCertSpec{
173 host: "localhost",
174 },
175 },
176 },
177 ExpectedCertIndex: 0,
178 },
179 "loopback: LoopbackClientServerNameOverride on server cert": {
180 Cert: TestCertSpec{
181 host: server.LoopbackClientServerNameOverride,
182 },
183 SNICerts: []NamedTestCertSpec{
184 {
185 TestCertSpec: TestCertSpec{
186 host: "localhost",
187 },
188 },
189 },
190 ExpectedCertIndex: 0,
191 },
192 "loopback: LoopbackClientServerNameOverride on SNI cert": {
193 Cert: TestCertSpec{
194 host: "localhost",
195 },
196 SNICerts: []NamedTestCertSpec{
197 {
198 TestCertSpec: TestCertSpec{
199 host: server.LoopbackClientServerNameOverride,
200 },
201 },
202 },
203 ExpectedCertIndex: -1,
204 },
205 "loopback: bind to 0.0.0.0 => loopback uses localhost": {
206 Cert: TestCertSpec{
207 host: "localhost",
208 },
209 ExpectedCertIndex: -1,
210 LoopbackClientBindAddressOverride: "0.0.0.0",
211 },
212 }
213
214 specToName := func(spec TestCertSpec) string {
215 name := spec.host + "_" + strings.Join(spec.names, ",") + "_" + strings.Join(spec.ips, ",")
216 return strings.Replace(name, "*", "star", -1)
217 }
218
219 for title := range tests {
220 test := tests[title]
221 t.Run(title, func(t *testing.T) {
222 t.Parallel()
223 _, ctx := ktesting.NewTestContext(t)
224 ctx, cancel := context.WithCancelCause(ctx)
225 defer cancel(errors.New("test has completed"))
226
227 // create server cert
228 certDir := "testdata/" + specToName(test.Cert)
229 serverCertBundleFile := filepath.Join(certDir, "cert")
230 serverKeyFile := filepath.Join(certDir, "key")
231 err := getOrCreateTestCertFiles(serverCertBundleFile, serverKeyFile, test.Cert)
232 if err != nil {
233 t.Fatalf("failed to create server cert: %v", err)
234 }
235 ca, err := caCertFromBundle(serverCertBundleFile)
236 if err != nil {
237 t.Fatalf("failed to extract ca cert from server cert bundle: %v", err)
238 }
239 caCerts := []*x509.Certificate{ca}
240
241 // create SNI certs
242 var namedCertKeys []cliflag.NamedCertKey
243 serverSig, err := certFileSignature(serverCertBundleFile, serverKeyFile)
244 if err != nil {
245 t.Fatalf("failed to get server cert signature: %v", err)
246 }
247 signatures := map[string]int{
248 serverSig: -1,
249 }
250 for j, c := range test.SNICerts {
251 sniDir := filepath.Join(certDir, specToName(c.TestCertSpec))
252 certBundleFile := filepath.Join(sniDir, "cert")
253 keyFile := filepath.Join(sniDir, "key")
254 err := getOrCreateTestCertFiles(certBundleFile, keyFile, c.TestCertSpec)
255 if err != nil {
256 t.Fatalf("failed to create SNI cert %d: %v", j, err)
257 }
258
259 namedCertKeys = append(namedCertKeys, cliflag.NamedCertKey{
260 KeyFile: keyFile,
261 CertFile: certBundleFile,
262 Names: c.explicitNames,
263 })
264
265 ca, err := caCertFromBundle(certBundleFile)
266 if err != nil {
267 t.Fatalf("failed to extract ca cert from SNI cert %d: %v", j, err)
268 }
269 caCerts = append(caCerts, ca)
270
271 // store index in namedCertKeys with the signature as the key
272 sig, err := certFileSignature(certBundleFile, keyFile)
273 if err != nil {
274 t.Fatalf("failed get SNI cert %d signature: %v", j, err)
275 }
276 signatures[sig] = j
277 }
278
279 // launch server
280 config := setUp(t)
281 info := fakeVersionInfo()
282 config.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString(fmt.Sprintf("%s.%s", info.Major, info.Minor), "", "")
283 config.EnableIndex = true
284 secureOptions := (&SecureServingOptions{
285 BindAddress: netutils.ParseIPSloppy("127.0.0.1"),
286 BindPort: 6443,
287 ServerCert: GeneratableKeyCert{
288 CertKey: CertKey{
289 CertFile: serverCertBundleFile,
290 KeyFile: serverKeyFile,
291 },
292 },
293 DisableHTTP2Serving: true,
294 SNICertKeys: namedCertKeys,
295 }).WithLoopback()
296 // use a random free port
297 ln, err := net.Listen("tcp", "127.0.0.1:0")
298 if err != nil {
299 t.Fatalf("failed to listen on 127.0.0.1:0")
300 }
301
302 secureOptions.Listener = ln
303 // get port
304 secureOptions.BindPort = ln.Addr().(*net.TCPAddr).Port
305 config.LoopbackClientConfig = &restclient.Config{}
306 if err := secureOptions.ApplyTo(&config.SecureServing, &config.LoopbackClientConfig); err != nil {
307 t.Fatalf("failed applying the SecureServingOptions: %v", err)
308 }
309
310 s, err := config.Complete(nil).New("test", server.NewEmptyDelegate())
311 if err != nil {
312 t.Fatalf("failed creating the server: %v", err)
313 }
314
315 // add poststart hook to know when the server is up.
316 startedCh := make(chan struct{})
317 s.AddPostStartHookOrDie("test-notifier", func(context server.PostStartHookContext) error {
318 close(startedCh)
319 return nil
320 })
321 preparedServer := s.PrepareRun()
322 preparedServerErrors := make(chan error)
323 go func() {
324 if err := preparedServer.RunWithContext(ctx); err != nil {
325 preparedServerErrors <- err
326 }
327 }()
328
329 // load ca certificates into a pool
330 roots := x509.NewCertPool()
331 for _, caCert := range caCerts {
332 roots.AddCert(caCert)
333 }
334
335 <-startedCh
336
337 // try to dial
338 addr := fmt.Sprintf("localhost:%d", secureOptions.BindPort)
339 t.Logf("Dialing %s as %q", addr, test.ServerName)
340 conn, err := tls.Dial("tcp", addr, &tls.Config{
341 RootCAs: roots,
342 ServerName: test.ServerName, // used for SNI in the client HELLO packet
343 })
344 if err != nil {
345 t.Fatalf("failed to connect: %v", err)
346 }
347 defer conn.Close()
348
349 // check returned server certificate
350 sig := x509CertSignature(conn.ConnectionState().PeerCertificates[0])
351 gotCertIndex, found := signatures[sig]
352 if !found {
353 t.Errorf("unknown signature returned from server: %s", sig)
354 }
355 if gotCertIndex != test.ExpectedCertIndex {
356 t.Errorf("expected cert index %d, got cert index %d", test.ExpectedCertIndex, gotCertIndex)
357 }
358
359 // check that the loopback client can connect
360 host := "127.0.0.1"
361 if len(test.LoopbackClientBindAddressOverride) != 0 {
362 host = test.LoopbackClientBindAddressOverride
363 }
364 s.LoopbackClientConfig.Host = net.JoinHostPort(host, strconv.Itoa(secureOptions.BindPort))
365
366 client, err := discovery.NewDiscoveryClientForConfig(s.LoopbackClientConfig)
367 if err != nil {
368 t.Fatalf("failed to create loopback client: %v", err)
369 }
370 got, err := client.ServerVersion()
371 if err != nil {
372 t.Fatalf("failed to connect with loopback client: %v", err)
373 }
374 if expected := &info; !reflect.DeepEqual(got, expected) {
375 t.Errorf("loopback client didn't get correct version info: expected=%v got=%v", *expected, *got)
376 }
377
378 select {
379 case err := <-preparedServerErrors:
380 t.Fatalf("preparedServer failed with error: %v", err)
381 default:
382 }
383 })
384 }
385 }
386
387 func parseIPList(ips []string) []net.IP {
388 var netIPs []net.IP
389 for _, ip := range ips {
390 netIPs = append(netIPs, netutils.ParseIPSloppy(ip))
391 }
392 return netIPs
393 }
394
395 func getOrCreateTestCertFiles(certFileName, keyFileName string, spec TestCertSpec) (err error) {
396 if _, err := os.Stat(certFileName); err == nil {
397 if _, err := os.Stat(keyFileName); err == nil {
398 return nil
399 }
400 }
401
402 certPem, keyPem, err := generateSelfSignedCertKey(spec.host, parseIPList(spec.ips), spec.names)
403 if err != nil {
404 return err
405 }
406
407 os.MkdirAll(filepath.Dir(certFileName), os.FileMode(0755))
408 err = ioutil.WriteFile(certFileName, certPem, os.FileMode(0755))
409 if err != nil {
410 return err
411 }
412
413 os.MkdirAll(filepath.Dir(keyFileName), os.FileMode(0755))
414 err = ioutil.WriteFile(keyFileName, keyPem, os.FileMode(0755))
415 if err != nil {
416 return err
417 }
418
419 return nil
420 }
421
422 func caCertFromBundle(bundlePath string) (*x509.Certificate, error) {
423 pemData, err := ioutil.ReadFile(bundlePath)
424 if err != nil {
425 return nil, err
426 }
427
428 // fetch last block
429 var block *pem.Block
430 for {
431 var nextBlock *pem.Block
432 nextBlock, pemData = pem.Decode(pemData)
433 if nextBlock == nil {
434 if block == nil {
435 return nil, fmt.Errorf("no certificate found in %q", bundlePath)
436
437 }
438 return x509.ParseCertificate(block.Bytes)
439 }
440 block = nextBlock
441 }
442 }
443
444 func x509CertSignature(cert *x509.Certificate) string {
445 return base64.StdEncoding.EncodeToString(cert.Signature)
446 }
447
448 func certFileSignature(certFile, keyFile string) (string, error) {
449 cert, err := tls.LoadX509KeyPair(certFile, keyFile)
450 if err != nil {
451 return "", err
452 }
453 return certSignature(cert)
454 }
455
456 func certSignature(cert tls.Certificate) (string, error) {
457 x509Certs, err := x509.ParseCertificates(cert.Certificate[0])
458 if err != nil {
459 return "", err
460 }
461 return x509CertSignature(x509Certs[0]), nil
462 }
463
464 func fakeVersionInfo() version.Info {
465 baseVer := baseversion.Get()
466 baseVer.Major = "42"
467 baseVer.Minor = "42"
468 baseVer.EmulationMajor = "42"
469 baseVer.EmulationMinor = "42"
470 baseVer.MinCompatibilityMajor = "42"
471 baseVer.MinCompatibilityMinor = "41"
472 return baseVer
473 }
474
475 // generateSelfSignedCertKey creates a self-signed certificate and key for the given host.
476 // Host may be an IP or a DNS name
477 // You may also specify additional subject alt names (either ip or dns names) for the certificate
478 func generateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS []string) ([]byte, []byte, error) {
479 priv, err := rsa.GenerateKey(cryptorand.Reader, 2048)
480 if err != nil {
481 return nil, nil, err
482 }
483
484 template := x509.Certificate{
485 SerialNumber: big.NewInt(1),
486 Subject: pkix.Name{
487 CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()),
488 },
489 NotBefore: time.Unix(0, 0),
490 NotAfter: time.Now().Add(time.Hour * 24 * 365 * 100),
491
492 KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
493 ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
494 BasicConstraintsValid: true,
495 IsCA: true,
496 }
497
498 if ip := netutils.ParseIPSloppy(host); ip != nil {
499 template.IPAddresses = append(template.IPAddresses, ip)
500 } else {
501 template.DNSNames = append(template.DNSNames, host)
502 }
503
504 template.IPAddresses = append(template.IPAddresses, alternateIPs...)
505 template.DNSNames = append(template.DNSNames, alternateDNS...)
506
507 derBytes, err := x509.CreateCertificate(cryptorand.Reader, &template, &template, &priv.PublicKey, priv)
508 if err != nil {
509 return nil, nil, err
510 }
511
512 // Generate cert
513 certBuffer := bytes.Buffer{}
514 if err := pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
515 return nil, nil, err
516 }
517
518 // Generate key
519 keyBuffer := bytes.Buffer{}
520 if err := pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
521 return nil, nil, err
522 }
523
524 return certBuffer.Bytes(), keyBuffer.Bytes(), nil
525 }