| 1 |
/* |
| 2 |
Copyright 2016 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package server |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"crypto/tls" |
| 22 |
"fmt" |
| 23 |
"io" |
| 24 |
"log" |
| 25 |
"net" |
| 26 |
"net/http" |
| 27 |
"os" |
| 28 |
"strings" |
| 29 |
"time" |
| 30 |
|
| 31 |
"golang.org/x/net/http2" |
| 32 |
"k8s.io/component-base/cli/flag" |
| 33 |
"k8s.io/klog/v2" |
| 34 |
|
| 35 |
utilruntime "k8s.io/apimachinery/pkg/util/runtime" |
| 36 |
"k8s.io/apiserver/pkg/endpoints/metrics" |
| 37 |
"k8s.io/apiserver/pkg/server/dynamiccertificates" |
| 38 |
) |
| 39 |
|
| 40 |
const ( |
| 41 |
defaultKeepAlivePeriod = 3 * time.Minute |
| 42 |
) |
| 43 |
|
| 44 |
// tlsConfig produces the tls.Config to serve with. |
| 45 |
func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, error) { |
| 46 |
tlsConfig := &tls.Config{ |
| 47 |
// Can't use SSLv3 because of POODLE and BEAST |
| 48 |
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher |
| 49 |
// Can't use TLSv1.1 because of RC4 cipher usage |
| 50 |
MinVersion: tls.VersionTLS12, |
| 51 |
// enable HTTP2 for go's 1.7 HTTP Server |
| 52 |
NextProtos: []string{"h2", "http/1.1"}, |
| 53 |
} |
| 54 |
|
| 55 |
// these are static aspects of the tls.Config |
| 56 |
if s.DisableHTTP2 { |
| 57 |
klog.Info("Forcing use of http/1.1 only") |
| 58 |
tlsConfig.NextProtos = []string{"http/1.1"} |
| 59 |
} |
| 60 |
if s.MinTLSVersion > 0 { |
| 61 |
tlsConfig.MinVersion = s.MinTLSVersion |
| 62 |
} |
| 63 |
if len(s.CipherSuites) > 0 { |
| 64 |
tlsConfig.CipherSuites = s.CipherSuites |
| 65 |
insecureCiphers := flag.InsecureTLSCiphers() |
| 66 |
for i := 0; i < len(s.CipherSuites); i++ { |
| 67 |
for cipherName, cipherID := range insecureCiphers { |
| 68 |
if s.CipherSuites[i] == cipherID { |
| 69 |
klog.Warningf("Use of insecure cipher '%s' detected.", cipherName) |
| 70 |
} |
| 71 |
} |
| 72 |
} |
| 73 |
} |
| 74 |
|
| 75 |
if s.ClientCA != nil { |
| 76 |
// Populate PeerCertificates in requests, but don't reject connections without certificates |
| 77 |
// This allows certificates to be validated by authenticators, while still allowing other auth types |
| 78 |
tlsConfig.ClientAuth = tls.RequestClientCert |
| 79 |
} |
| 80 |
|
| 81 |
if s.ClientCA != nil || s.Cert != nil || len(s.SNICerts) > 0 { |
| 82 |
dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController( |
| 83 |
tlsConfig, |
| 84 |
s.ClientCA, |
| 85 |
s.Cert, |
| 86 |
s.SNICerts, |
| 87 |
nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages. |
| 88 |
) |
| 89 |
|
| 90 |
if s.ClientCA != nil { |
| 91 |
s.ClientCA.AddListener(dynamicCertificateController) |
| 92 |
} |
| 93 |
if s.Cert != nil { |
| 94 |
s.Cert.AddListener(dynamicCertificateController) |
| 95 |
} |
| 96 |
// generate a context from stopCh. This is to avoid modifying files which are relying on apiserver |
| 97 |
// TODO: See if we can pass ctx to the current method |
| 98 |
ctx, cancel := context.WithCancel(context.Background()) |
| 99 |
go func() { |
| 100 |
select { |
| 101 |
case <-stopCh: |
| 102 |
cancel() // stopCh closed, so cancel our context |
| 103 |
case <-ctx.Done(): |
| 104 |
} |
| 105 |
}() |
| 106 |
// start controllers if possible |
| 107 |
if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok { |
| 108 |
// runonce to try to prime data. If this fails, it's ok because we fail closed. |
| 109 |
// Files are required to be populated already, so this is for convenience. |
| 110 |
if err := controller.RunOnce(ctx); err != nil { |
| 111 |
klog.Warningf("Initial population of client CA failed: %v", err) |
| 112 |
} |
| 113 |
|
| 114 |
go controller.Run(ctx, 1) |
| 115 |
} |
| 116 |
if controller, ok := s.Cert.(dynamiccertificates.ControllerRunner); ok { |
| 117 |
// runonce to try to prime data. If this fails, it's ok because we fail closed. |
| 118 |
// Files are required to be populated already, so this is for convenience. |
| 119 |
if err := controller.RunOnce(ctx); err != nil { |
| 120 |
klog.Warningf("Initial population of default serving certificate failed: %v", err) |
| 121 |
} |
| 122 |
|
| 123 |
go controller.Run(ctx, 1) |
| 124 |
} |
| 125 |
for _, sniCert := range s.SNICerts { |
| 126 |
sniCert.AddListener(dynamicCertificateController) |
| 127 |
if controller, ok := sniCert.(dynamiccertificates.ControllerRunner); ok { |
| 128 |
// runonce to try to prime data. If this fails, it's ok because we fail closed. |
| 129 |
// Files are required to be populated already, so this is for convenience. |
| 130 |
if err := controller.RunOnce(ctx); err != nil { |
| 131 |
klog.Warningf("Initial population of SNI serving certificate failed: %v", err) |
| 132 |
} |
| 133 |
|
| 134 |
go controller.Run(ctx, 1) |
| 135 |
} |
| 136 |
} |
| 137 |
|
| 138 |
// runonce to try to prime data. If this fails, it's ok because we fail closed. |
| 139 |
// Files are required to be populated already, so this is for convenience. |
| 140 |
if err := dynamicCertificateController.RunOnce(); err != nil { |
| 141 |
klog.Warningf("Initial population of dynamic certificates failed: %v", err) |
| 142 |
} |
| 143 |
go dynamicCertificateController.Run(1, stopCh) |
| 144 |
|
| 145 |
tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient |
| 146 |
} |
| 147 |
|
| 148 |
return tlsConfig, nil |
| 149 |
} |
| 150 |
|
| 151 |
// Serve runs the secure http server. It fails only if certificates cannot be loaded or the initial listen call fails. |
| 152 |
// The actual server loop (stoppable by closing stopCh) runs in a go routine, i.e. Serve does not block. |
| 153 |
// It returns a stoppedCh that is closed when all non-hijacked active requests have been processed. |
| 154 |
// It returns a listenerStoppedCh that is closed when the underlying http Server has stopped listening. |
| 155 |
func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) (<-chan struct{}, <-chan struct{}, error) { |
| 156 |
if s.Listener == nil { |
| 157 |
return nil, nil, fmt.Errorf("listener must not be nil") |
| 158 |
} |
| 159 |
|
| 160 |
tlsConfig, err := s.tlsConfig(stopCh) |
| 161 |
if err != nil { |
| 162 |
return nil, nil, err |
| 163 |
} |
| 164 |
|
| 165 |
secureServer := &http.Server{ |
| 166 |
Addr: s.Listener.Addr().String(), |
| 167 |
Handler: handler, |
| 168 |
MaxHeaderBytes: 1 << 20, |
| 169 |
TLSConfig: tlsConfig, |
| 170 |
|
| 171 |
IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout |
| 172 |
ReadHeaderTimeout: 32 * time.Second, // just shy of requestTimeoutUpperBound |
| 173 |
} |
| 174 |
|
| 175 |
if !s.DisableHTTP2 { |
| 176 |
// At least 99% of serialized resources in surveyed clusters were smaller than 256kb. |
| 177 |
// This should be big enough to accommodate most API POST requests in a single frame, |
| 178 |
// and small enough to allow a per connection buffer of this size multiplied by `MaxConcurrentStreams`. |
| 179 |
const resourceBody99Percentile = 256 * 1024 |
| 180 |
|
| 181 |
http2Options := &http2.Server{ |
| 182 |
IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout |
| 183 |
// shrink the per-stream buffer and max framesize from the 1MB default while still accommodating most API POST requests in a single frame |
| 184 |
MaxUploadBufferPerStream: resourceBody99Percentile, |
| 185 |
MaxReadFrameSize: resourceBody99Percentile, |
| 186 |
} |
| 187 |
|
| 188 |
// use the overridden concurrent streams setting or make the default of 250 explicit so we can size MaxUploadBufferPerConnection appropriately |
| 189 |
if s.HTTP2MaxStreamsPerConnection > 0 { |
| 190 |
http2Options.MaxConcurrentStreams = uint32(s.HTTP2MaxStreamsPerConnection) |
| 191 |
} else { |
| 192 |
// match http2.initialMaxConcurrentStreams used by clients |
| 193 |
// this makes it so that a malicious client can only open 400 streams before we forcibly close the connection |
| 194 |
// https://github.com/golang/net/commit/b225e7ca6dde1ef5a5ae5ce922861bda011cfabd |
| 195 |
http2Options.MaxConcurrentStreams = 100 |
| 196 |
} |
| 197 |
|
| 198 |
// increase the connection buffer size from the 1MB default to handle the specified number of concurrent streams |
| 199 |
http2Options.MaxUploadBufferPerConnection = http2Options.MaxUploadBufferPerStream * int32(http2Options.MaxConcurrentStreams) |
| 200 |
// apply settings to the server |
| 201 |
if err := http2.ConfigureServer(secureServer, http2Options); err != nil { |
| 202 |
return nil, nil, fmt.Errorf("error configuring http2: %v", err) |
| 203 |
} |
| 204 |
} |
| 205 |
|
| 206 |
// use tlsHandshakeErrorWriter to handle messages of tls handshake error |
| 207 |
tlsErrorWriter := &tlsHandshakeErrorWriter{os.Stderr} |
| 208 |
tlsErrorLogger := log.New(tlsErrorWriter, "", 0) |
| 209 |
secureServer.ErrorLog = tlsErrorLogger |
| 210 |
|
| 211 |
klog.Infof("Serving securely on %s", secureServer.Addr) |
| 212 |
return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh) |
| 213 |
} |
| 214 |
|
| 215 |
// RunServer spawns a go-routine continuously serving until the stopCh is |
| 216 |
// closed. |
| 217 |
// It returns a stoppedCh that is closed when all non-hijacked active requests |
| 218 |
// have been processed. |
| 219 |
// This function does not block |
| 220 |
// TODO: make private when insecure serving is gone from the kube-apiserver |
| 221 |
func RunServer( |
| 222 |
server *http.Server, |
| 223 |
ln net.Listener, |
| 224 |
shutDownTimeout time.Duration, |
| 225 |
stopCh <-chan struct{}, |
| 226 |
) (<-chan struct{}, <-chan struct{}, error) { |
| 227 |
if ln == nil { |
| 228 |
return nil, nil, fmt.Errorf("listener must not be nil") |
| 229 |
} |
| 230 |
|
| 231 |
// Shutdown server gracefully. |
| 232 |
serverShutdownCh, listenerStoppedCh := make(chan struct{}), make(chan struct{}) |
| 233 |
go func() { |
| 234 |
defer close(serverShutdownCh) |
| 235 |
<-stopCh |
| 236 |
ctx, cancel := context.WithTimeout(context.Background(), shutDownTimeout) |
| 237 |
defer cancel() |
| 238 |
err := server.Shutdown(ctx) |
| 239 |
if err != nil { |
| 240 |
klog.Errorf("Failed to shutdown server: %v", err) |
| 241 |
} |
| 242 |
}() |
| 243 |
|
| 244 |
go func() { |
| 245 |
defer utilruntime.HandleCrash() |
| 246 |
defer close(listenerStoppedCh) |
| 247 |
|
| 248 |
var listener net.Listener |
| 249 |
listener = tcpKeepAliveListener{ln} |
| 250 |
if server.TLSConfig != nil { |
| 251 |
listener = tls.NewListener(listener, server.TLSConfig) |
| 252 |
} |
| 253 |
|
| 254 |
err := server.Serve(listener) |
| 255 |
|
| 256 |
msg := fmt.Sprintf("Stopped listening on %s", ln.Addr().String()) |
| 257 |
select { |
| 258 |
case <-stopCh: |
| 259 |
klog.Info(msg) |
| 260 |
default: |
| 261 |
panic(fmt.Sprintf("%s due to error: %v", msg, err)) |
| 262 |
} |
| 263 |
}() |
| 264 |
|
| 265 |
return serverShutdownCh, listenerStoppedCh, nil |
| 266 |
} |
| 267 |
|
| 268 |
// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted |
| 269 |
// connections. It's used by ListenAndServe and ListenAndServeTLS so |
| 270 |
// dead TCP connections (e.g. closing laptop mid-download) eventually |
| 271 |
// go away. |
| 272 |
// |
| 273 |
// Copied from Go 1.7.2 net/http/server.go |
| 274 |
type tcpKeepAliveListener struct { |
| 275 |
net.Listener |
| 276 |
} |
| 277 |
|
| 278 |
func (ln tcpKeepAliveListener) Accept() (net.Conn, error) { |
| 279 |
c, err := ln.Listener.Accept() |
| 280 |
if err != nil { |
| 281 |
return nil, err |
| 282 |
} |
| 283 |
if tc, ok := c.(*net.TCPConn); ok { |
| 284 |
tc.SetKeepAlive(true) |
| 285 |
tc.SetKeepAlivePeriod(defaultKeepAlivePeriod) |
| 286 |
} |
| 287 |
return c, nil |
| 288 |
} |
| 289 |
|
| 290 |
// tlsHandshakeErrorWriter writes TLS handshake errors to klog with |
| 291 |
// trace level - V(5), to avoid flooding of tls handshake errors. |
| 292 |
type tlsHandshakeErrorWriter struct { |
| 293 |
out io.Writer |
| 294 |
} |
| 295 |
|
| 296 |
const tlsHandshakeErrorPrefix = "http: TLS handshake error" |
| 297 |
|
| 298 |
func (w *tlsHandshakeErrorWriter) Write(p []byte) (int, error) { |
| 299 |
if strings.Contains(string(p), tlsHandshakeErrorPrefix) { |
| 300 |
klog.V(5).Info(string(p)) |
| 301 |
metrics.TLSHandshakeErrors.Inc() |
| 302 |
return len(p), nil |
| 303 |
} |
| 304 |
|
| 305 |
// for non tls handshake error, log it as usual |
| 306 |
return w.out.Write(p) |
| 307 |
} |