| 1 |
/* |
| 2 |
Copyright 2016 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package options |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"net" |
| 23 |
"path" |
| 24 |
"strconv" |
| 25 |
"strings" |
| 26 |
"syscall" |
| 27 |
|
| 28 |
"github.com/spf13/pflag" |
| 29 |
"k8s.io/klog/v2" |
| 30 |
netutils "k8s.io/utils/net" |
| 31 |
|
| 32 |
utilnet "k8s.io/apimachinery/pkg/util/net" |
| 33 |
"k8s.io/apiserver/pkg/server" |
| 34 |
"k8s.io/apiserver/pkg/server/dynamiccertificates" |
| 35 |
certutil "k8s.io/client-go/util/cert" |
| 36 |
"k8s.io/client-go/util/keyutil" |
| 37 |
cliflag "k8s.io/component-base/cli/flag" |
| 38 |
) |
| 39 |
|
| 40 |
type SecureServingOptions struct { |
| 41 |
BindAddress net.IP |
| 42 |
// BindPort is ignored when Listener is set, will serve https even with 0. |
| 43 |
BindPort int |
| 44 |
// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp", |
| 45 |
// "tcp4", and "tcp6". |
| 46 |
BindNetwork string |
| 47 |
// DisableHTTP2Serving indicates that http2 serving should not be enabled. |
| 48 |
DisableHTTP2Serving bool |
| 49 |
// Required set to true means that BindPort cannot be zero. |
| 50 |
Required bool |
| 51 |
// ExternalAddress is the address advertised, even if BindAddress is a loopback. By default this |
| 52 |
// is set to BindAddress if the later no loopback, or to the first host interface address. |
| 53 |
ExternalAddress net.IP |
| 54 |
|
| 55 |
// Listener is the secure server network listener. |
| 56 |
// either Listener or BindAddress/BindPort/BindNetwork is set, |
| 57 |
// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork. |
| 58 |
Listener net.Listener |
| 59 |
|
| 60 |
// ServerCert is the TLS cert info for serving secure traffic |
| 61 |
ServerCert GeneratableKeyCert |
| 62 |
// SNICertKeys are named CertKeys for serving secure traffic with SNI support. |
| 63 |
SNICertKeys []cliflag.NamedCertKey |
| 64 |
// CipherSuites is the list of allowed cipher suites for the server. |
| 65 |
// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). |
| 66 |
CipherSuites []string |
| 67 |
// MinTLSVersion is the minimum TLS version supported. |
| 68 |
// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). |
| 69 |
MinTLSVersion string |
| 70 |
|
| 71 |
// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client. |
| 72 |
// A value of zero means to use the default provided by golang's HTTP/2 support. |
| 73 |
HTTP2MaxStreamsPerConnection int |
| 74 |
|
| 75 |
// PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows |
| 76 |
// more than one instance to bind on the same address and port. |
| 77 |
PermitPortSharing bool |
| 78 |
|
| 79 |
// PermitAddressSharing controls if SO_REUSEADDR is used when binding the port. |
| 80 |
PermitAddressSharing bool |
| 81 |
} |
| 82 |
|
| 83 |
type CertKey struct { |
| 84 |
// CertFile is a file containing a PEM-encoded certificate, and possibly the complete certificate chain |
| 85 |
CertFile string |
| 86 |
// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile |
| 87 |
KeyFile string |
| 88 |
} |
| 89 |
|
| 90 |
type GeneratableKeyCert struct { |
| 91 |
// CertKey allows setting an explicit cert/key file to use. |
| 92 |
CertKey CertKey |
| 93 |
|
| 94 |
// CertDirectory specifies a directory to write generated certificates to if CertFile/KeyFile aren't explicitly set. |
| 95 |
// PairName is used to determine the filenames within CertDirectory. |
| 96 |
// If CertDirectory and PairName are not set, an in-memory certificate will be generated. |
| 97 |
CertDirectory string |
| 98 |
// PairName is the name which will be used with CertDirectory to make a cert and key filenames. |
| 99 |
// It becomes CertDirectory/PairName.crt and CertDirectory/PairName.key |
| 100 |
PairName string |
| 101 |
|
| 102 |
// GeneratedCert holds an in-memory generated certificate if CertFile/KeyFile aren't explicitly set, and CertDirectory/PairName are not set. |
| 103 |
GeneratedCert dynamiccertificates.CertKeyContentProvider |
| 104 |
|
| 105 |
// FixtureDirectory is a directory that contains test fixture used to avoid regeneration of certs during tests. |
| 106 |
// The format is: |
| 107 |
// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.crt |
| 108 |
// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.key |
| 109 |
FixtureDirectory string |
| 110 |
} |
| 111 |
|
| 112 |
func NewSecureServingOptions() *SecureServingOptions { |
| 113 |
return &SecureServingOptions{ |
| 114 |
BindAddress: netutils.ParseIPSloppy("0.0.0.0"), |
| 115 |
BindPort: 443, |
| 116 |
ServerCert: GeneratableKeyCert{ |
| 117 |
PairName: "apiserver", |
| 118 |
CertDirectory: "apiserver.local.config/certificates", |
| 119 |
}, |
| 120 |
} |
| 121 |
} |
| 122 |
|
| 123 |
func (s *SecureServingOptions) DefaultExternalAddress() (net.IP, error) { |
| 124 |
if s.ExternalAddress != nil && !s.ExternalAddress.IsUnspecified() { |
| 125 |
return s.ExternalAddress, nil |
| 126 |
} |
| 127 |
return utilnet.ResolveBindAddress(s.BindAddress) |
| 128 |
} |
| 129 |
|
| 130 |
func (s *SecureServingOptions) Validate() []error { |
| 131 |
if s == nil { |
| 132 |
return nil |
| 133 |
} |
| 134 |
|
| 135 |
errors := []error{} |
| 136 |
|
| 137 |
if s.Required && s.BindPort < 1 || s.BindPort > 65535 { |
| 138 |
errors = append(errors, fmt.Errorf("--secure-port %v must be between 1 and 65535, inclusive. It cannot be turned off with 0", s.BindPort)) |
| 139 |
} else if s.BindPort < 0 || s.BindPort > 65535 { |
| 140 |
errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port", s.BindPort)) |
| 141 |
} |
| 142 |
|
| 143 |
if (len(s.ServerCert.CertKey.CertFile) != 0 || len(s.ServerCert.CertKey.KeyFile) != 0) && s.ServerCert.GeneratedCert != nil { |
| 144 |
errors = append(errors, fmt.Errorf("cert/key file and in-memory certificate cannot both be set")) |
| 145 |
} |
| 146 |
|
| 147 |
return errors |
| 148 |
} |
| 149 |
|
| 150 |
func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { |
| 151 |
if s == nil { |
| 152 |
return |
| 153 |
} |
| 154 |
|
| 155 |
fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+ |
| 156 |
"The IP address on which to listen for the --secure-port port. The "+ |
| 157 |
"associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+ |
| 158 |
"clients. If blank or an unspecified address (0.0.0.0 or ::), all interfaces and IP address families will be used.") |
| 159 |
|
| 160 |
desc := "The port on which to serve HTTPS with authentication and authorization." |
| 161 |
if s.Required { |
| 162 |
desc += " It cannot be switched off with 0." |
| 163 |
} else { |
| 164 |
desc += " If 0, don't serve HTTPS at all." |
| 165 |
} |
| 166 |
fs.IntVar(&s.BindPort, "secure-port", s.BindPort, desc) |
| 167 |
|
| 168 |
fs.BoolVar(&s.DisableHTTP2Serving, "disable-http2-serving", s.DisableHTTP2Serving, |
| 169 |
"If true, HTTP2 serving will be disabled [default=false]") |
| 170 |
|
| 171 |
fs.StringVar(&s.ServerCert.CertDirectory, "cert-dir", s.ServerCert.CertDirectory, ""+ |
| 172 |
"The directory where the TLS certs are located. "+ |
| 173 |
"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") |
| 174 |
|
| 175 |
fs.StringVar(&s.ServerCert.CertKey.CertFile, "tls-cert-file", s.ServerCert.CertKey.CertFile, ""+ |
| 176 |
"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ |
| 177 |
"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+ |
| 178 |
"--tls-private-key-file are not provided, a self-signed certificate and key "+ |
| 179 |
"are generated for the public address and saved to the directory specified by --cert-dir.") |
| 180 |
|
| 181 |
fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile, |
| 182 |
"File containing the default x509 private key matching --tls-cert-file.") |
| 183 |
|
| 184 |
tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames() |
| 185 |
tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames() |
| 186 |
fs.StringSliceVar(&s.CipherSuites, "tls-cipher-suites", s.CipherSuites, |
| 187 |
"Comma-separated list of cipher suites for the server. "+ |
| 188 |
"If omitted, the default Go cipher suites will be used. \n"+ |
| 189 |
"Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+ |
| 190 |
"Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".") |
| 191 |
|
| 192 |
tlsPossibleVersions := cliflag.TLSPossibleVersions() |
| 193 |
fs.StringVar(&s.MinTLSVersion, "tls-min-version", s.MinTLSVersion, |
| 194 |
"Minimum TLS version supported. "+ |
| 195 |
"Possible values: "+strings.Join(tlsPossibleVersions, ", ")) |
| 196 |
|
| 197 |
fs.Var(cliflag.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+ |
| 198 |
"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+ |
| 199 |
"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+ |
| 200 |
"segments. The domain patterns also allow IP addresses, but IPs should only be used if "+ |
| 201 |
"the apiserver has visibility to the IP address requested by a client. "+ |
| 202 |
"If no domain patterns are provided, the names of the certificate are "+ |
| 203 |
"extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+ |
| 204 |
"trump over extracted names. For multiple key/certificate pairs, use the "+ |
| 205 |
"--tls-sni-cert-key multiple times. "+ |
| 206 |
"Examples: \"example.crt,example.key\" or \"foo.crt,foo.key:*.foo.com,foo.com\".") |
| 207 |
|
| 208 |
fs.IntVar(&s.HTTP2MaxStreamsPerConnection, "http2-max-streams-per-connection", s.HTTP2MaxStreamsPerConnection, ""+ |
| 209 |
"The limit that the server gives to clients for "+ |
| 210 |
"the maximum number of streams in an HTTP/2 connection. "+ |
| 211 |
"Zero means to use golang's default.") |
| 212 |
|
| 213 |
fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing, |
| 214 |
"If true, SO_REUSEPORT will be used when binding the port, which allows "+ |
| 215 |
"more than one instance to bind on the same address and port. [default=false]") |
| 216 |
|
| 217 |
fs.BoolVar(&s.PermitAddressSharing, "permit-address-sharing", s.PermitAddressSharing, |
| 218 |
"If true, SO_REUSEADDR will be used when binding the port. This allows binding "+ |
| 219 |
"to wildcard IPs like 0.0.0.0 and specific IPs in parallel, and it avoids waiting "+ |
| 220 |
"for the kernel to release sockets in TIME_WAIT state. [default=false]") |
| 221 |
} |
| 222 |
|
| 223 |
// ApplyTo fills up serving information in the server configuration. |
| 224 |
func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error { |
| 225 |
if s == nil { |
| 226 |
return nil |
| 227 |
} |
| 228 |
if s.BindPort <= 0 && s.Listener == nil { |
| 229 |
return nil |
| 230 |
} |
| 231 |
|
| 232 |
if s.Listener == nil { |
| 233 |
var err error |
| 234 |
addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort)) |
| 235 |
|
| 236 |
c := net.ListenConfig{} |
| 237 |
|
| 238 |
ctls := multipleControls{} |
| 239 |
if s.PermitPortSharing { |
| 240 |
ctls = append(ctls, permitPortReuse) |
| 241 |
} |
| 242 |
if s.PermitAddressSharing { |
| 243 |
ctls = append(ctls, permitAddressReuse) |
| 244 |
} |
| 245 |
if len(ctls) > 0 { |
| 246 |
c.Control = ctls.Control |
| 247 |
} |
| 248 |
|
| 249 |
s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c) |
| 250 |
if err != nil { |
| 251 |
return fmt.Errorf("failed to create listener: %v", err) |
| 252 |
} |
| 253 |
} else { |
| 254 |
if _, ok := s.Listener.Addr().(*net.TCPAddr); !ok { |
| 255 |
return fmt.Errorf("failed to parse ip and port from listener") |
| 256 |
} |
| 257 |
s.BindPort = s.Listener.Addr().(*net.TCPAddr).Port |
| 258 |
s.BindAddress = s.Listener.Addr().(*net.TCPAddr).IP |
| 259 |
} |
| 260 |
|
| 261 |
*config = &server.SecureServingInfo{ |
| 262 |
Listener: s.Listener, |
| 263 |
HTTP2MaxStreamsPerConnection: s.HTTP2MaxStreamsPerConnection, |
| 264 |
DisableHTTP2: s.DisableHTTP2Serving, |
| 265 |
} |
| 266 |
c := *config |
| 267 |
|
| 268 |
serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile |
| 269 |
// load main cert *original description until 2023-08-18* |
| 270 |
|
| 271 |
/* |
| 272 |
kubernetes mutual (2-way) x509 between client and apiserver: |
| 273 |
|
| 274 |
>1. apiserver sending its apiserver certificate along with its publickey to client |
| 275 |
2. client verifies the apiserver certificate sent against its cluster certificate authority data |
| 276 |
3. client sending its client certificate along with its public key to the apiserver |
| 277 |
4. apiserver verifies the client certificate sent against its cluster certificate authority data |
| 278 |
|
| 279 |
description: |
| 280 |
here, with this block, |
| 281 |
apiserver certificate and pub key data (along with priv key)get loaded into server.SecureServingInfo |
| 282 |
for client to later in the step 2 verify the apiserver certificate during the handshake |
| 283 |
when making a request |
| 284 |
|
| 285 |
normal args related to this stage: |
| 286 |
--tls-cert-file string File containing the default x509 Certificate for HTTPS. |
| 287 |
(CA cert, if any, concatenated after server cert). If HTTPS serving is enabled, and |
| 288 |
--tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate |
| 289 |
and key are generated for the public address and saved to the directory specified by |
| 290 |
--cert-dir |
| 291 |
--tls-private-key-file string File containing the default x509 private key matching --tls-cert-file. |
| 292 |
|
| 293 |
(retrievable from "kube-apiserver --help" command) |
| 294 |
(suggested by @deads2k) |
| 295 |
|
| 296 |
see also: |
| 297 |
- for the step 2, see: staging/src/k8s.io/client-go/transport/transport.go |
| 298 |
- for the step 3, see: staging/src/k8s.io/client-go/transport/transport.go |
| 299 |
- for the step 4, see: staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go |
| 300 |
*/ |
| 301 |
|
| 302 |
if len(serverCertFile) != 0 || len(serverKeyFile) != 0 { |
| 303 |
var err error |
| 304 |
c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile) |
| 305 |
if err != nil { |
| 306 |
return err |
| 307 |
} |
| 308 |
} else if s.ServerCert.GeneratedCert != nil { |
| 309 |
c.Cert = s.ServerCert.GeneratedCert |
| 310 |
} |
| 311 |
|
| 312 |
if len(s.CipherSuites) != 0 { |
| 313 |
cipherSuites, err := cliflag.TLSCipherSuites(s.CipherSuites) |
| 314 |
if err != nil { |
| 315 |
return err |
| 316 |
} |
| 317 |
c.CipherSuites = cipherSuites |
| 318 |
} |
| 319 |
|
| 320 |
var err error |
| 321 |
c.MinTLSVersion, err = cliflag.TLSVersion(s.MinTLSVersion) |
| 322 |
if err != nil { |
| 323 |
return err |
| 324 |
} |
| 325 |
|
| 326 |
// load SNI certs |
| 327 |
namedTLSCerts := make([]dynamiccertificates.SNICertKeyContentProvider, 0, len(s.SNICertKeys)) |
| 328 |
for _, nck := range s.SNICertKeys { |
| 329 |
tlsCert, err := dynamiccertificates.NewDynamicSNIContentFromFiles("sni-serving-cert", nck.CertFile, nck.KeyFile, nck.Names...) |
| 330 |
namedTLSCerts = append(namedTLSCerts, tlsCert) |
| 331 |
if err != nil { |
| 332 |
return fmt.Errorf("failed to load SNI cert and key: %v", err) |
| 333 |
} |
| 334 |
} |
| 335 |
c.SNICerts = namedTLSCerts |
| 336 |
|
| 337 |
return nil |
| 338 |
} |
| 339 |
|
| 340 |
func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress string, alternateDNS []string, alternateIPs []net.IP) error { |
| 341 |
if s == nil || (s.BindPort == 0 && s.Listener == nil) { |
| 342 |
return nil |
| 343 |
} |
| 344 |
keyCert := &s.ServerCert.CertKey |
| 345 |
if len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 { |
| 346 |
return nil |
| 347 |
} |
| 348 |
|
| 349 |
canReadCertAndKey := false |
| 350 |
if len(s.ServerCert.CertDirectory) > 0 { |
| 351 |
if len(s.ServerCert.PairName) == 0 { |
| 352 |
return fmt.Errorf("PairName is required if CertDirectory is set") |
| 353 |
} |
| 354 |
keyCert.CertFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".crt") |
| 355 |
keyCert.KeyFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".key") |
| 356 |
if canRead, err := certutil.CanReadCertAndKey(keyCert.CertFile, keyCert.KeyFile); err != nil { |
| 357 |
return err |
| 358 |
} else { |
| 359 |
canReadCertAndKey = canRead |
| 360 |
} |
| 361 |
} |
| 362 |
|
| 363 |
if !canReadCertAndKey { |
| 364 |
// add either the bind address or localhost to the valid alternates |
| 365 |
if s.BindAddress.IsUnspecified() { |
| 366 |
alternateDNS = append(alternateDNS, "localhost") |
| 367 |
} else { |
| 368 |
alternateIPs = append(alternateIPs, s.BindAddress) |
| 369 |
} |
| 370 |
|
| 371 |
if cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures(publicAddress, alternateIPs, alternateDNS, s.ServerCert.FixtureDirectory); err != nil { |
| 372 |
return fmt.Errorf("unable to generate self signed cert: %v", err) |
| 373 |
} else if len(keyCert.CertFile) > 0 && len(keyCert.KeyFile) > 0 { |
| 374 |
if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil { |
| 375 |
return err |
| 376 |
} |
| 377 |
if err := keyutil.WriteKey(keyCert.KeyFile, key); err != nil { |
| 378 |
return err |
| 379 |
} |
| 380 |
klog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile) |
| 381 |
} else { |
| 382 |
s.ServerCert.GeneratedCert, err = dynamiccertificates.NewStaticCertKeyContent("Generated self signed cert", cert, key) |
| 383 |
if err != nil { |
| 384 |
return err |
| 385 |
} |
| 386 |
klog.Infof("Generated self-signed cert in-memory") |
| 387 |
} |
| 388 |
} |
| 389 |
|
| 390 |
return nil |
| 391 |
} |
| 392 |
|
| 393 |
func CreateListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) { |
| 394 |
if len(network) == 0 { |
| 395 |
network = "tcp" |
| 396 |
} |
| 397 |
|
| 398 |
ln, err := config.Listen(context.TODO(), network, addr) |
| 399 |
if err != nil { |
| 400 |
return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err) |
| 401 |
} |
| 402 |
|
| 403 |
// get port |
| 404 |
tcpAddr, ok := ln.Addr().(*net.TCPAddr) |
| 405 |
if !ok { |
| 406 |
ln.Close() |
| 407 |
return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) |
| 408 |
} |
| 409 |
|
| 410 |
return ln, tcpAddr.Port, nil |
| 411 |
} |
| 412 |
|
| 413 |
type multipleControls []func(network, addr string, conn syscall.RawConn) error |
| 414 |
|
| 415 |
func (mcs multipleControls) Control(network, addr string, conn syscall.RawConn) error { |
| 416 |
for _, c := range mcs { |
| 417 |
if err := c(network, addr, conn); err != nil { |
| 418 |
return err |
| 419 |
} |
| 420 |
} |
| 421 |
return nil |
| 422 |
} |