Debian Codegraph @

pkg/server/secure_serving.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package server
18
19 import (
20 "context"
21 "crypto/tls"
22 "fmt"
23 "io"
24 "log"
25 "net"
26 "net/http"
27 "os"
28 "strings"
29 "time"
30
31 "golang.org/x/net/http2"
32 "k8s.io/component-base/cli/flag"
33 "k8s.io/klog/v2"
34
35 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
36 "k8s.io/apiserver/pkg/endpoints/metrics"
37 "k8s.io/apiserver/pkg/server/dynamiccertificates"
38 )
39
40 const (
41 defaultKeepAlivePeriod = 3 * time.Minute
42 )
43
44 // tlsConfig produces the tls.Config to serve with.
45 func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, error) {
46 tlsConfig := &tls.Config{
47 // Can't use SSLv3 because of POODLE and BEAST
48 // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
49 // Can't use TLSv1.1 because of RC4 cipher usage
50 MinVersion: tls.VersionTLS12,
51 // enable HTTP2 for go's 1.7 HTTP Server
52 NextProtos: []string{"h2", "http/1.1"},
53 }
54
55 // these are static aspects of the tls.Config
56 if s.DisableHTTP2 {
57 klog.Info("Forcing use of http/1.1 only")
58 tlsConfig.NextProtos = []string{"http/1.1"}
59 }
60 if s.MinTLSVersion > 0 {
61 tlsConfig.MinVersion = s.MinTLSVersion
62 }
63 if len(s.CipherSuites) > 0 {
64 tlsConfig.CipherSuites = s.CipherSuites
65 insecureCiphers := flag.InsecureTLSCiphers()
66 for i := 0; i < len(s.CipherSuites); i++ {
67 for cipherName, cipherID := range insecureCiphers {
68 if s.CipherSuites[i] == cipherID {
69 klog.Warningf("Use of insecure cipher '%s' detected.", cipherName)
70 }
71 }
72 }
73 }
74
75 if s.ClientCA != nil {
76 // Populate PeerCertificates in requests, but don't reject connections without certificates
77 // This allows certificates to be validated by authenticators, while still allowing other auth types
78 tlsConfig.ClientAuth = tls.RequestClientCert
79 }
80
81 if s.ClientCA != nil || s.Cert != nil || len(s.SNICerts) > 0 {
82 dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController(
83 tlsConfig,
84 s.ClientCA,
85 s.Cert,
86 s.SNICerts,
87 nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
88 )
89
90 if s.ClientCA != nil {
91 s.ClientCA.AddListener(dynamicCertificateController)
92 }
93 if s.Cert != nil {
94 s.Cert.AddListener(dynamicCertificateController)
95 }
96 // generate a context from stopCh. This is to avoid modifying files which are relying on apiserver
97 // TODO: See if we can pass ctx to the current method
98 ctx, cancel := context.WithCancel(context.Background())
99 go func() {
100 select {
101 case <-stopCh:
102 cancel() // stopCh closed, so cancel our context
103 case <-ctx.Done():
104 }
105 }()
106 // start controllers if possible
107 if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok {
108 // runonce to try to prime data. If this fails, it's ok because we fail closed.
109 // Files are required to be populated already, so this is for convenience.
110 if err := controller.RunOnce(ctx); err != nil {
111 klog.Warningf("Initial population of client CA failed: %v", err)
112 }
113
114 go controller.Run(ctx, 1)
115 }
116 if controller, ok := s.Cert.(dynamiccertificates.ControllerRunner); ok {
117 // runonce to try to prime data. If this fails, it's ok because we fail closed.
118 // Files are required to be populated already, so this is for convenience.
119 if err := controller.RunOnce(ctx); err != nil {
120 klog.Warningf("Initial population of default serving certificate failed: %v", err)
121 }
122
123 go controller.Run(ctx, 1)
124 }
125 for _, sniCert := range s.SNICerts {
126 sniCert.AddListener(dynamicCertificateController)
127 if controller, ok := sniCert.(dynamiccertificates.ControllerRunner); ok {
128 // runonce to try to prime data. If this fails, it's ok because we fail closed.
129 // Files are required to be populated already, so this is for convenience.
130 if err := controller.RunOnce(ctx); err != nil {
131 klog.Warningf("Initial population of SNI serving certificate failed: %v", err)
132 }
133
134 go controller.Run(ctx, 1)
135 }
136 }
137
138 // runonce to try to prime data. If this fails, it's ok because we fail closed.
139 // Files are required to be populated already, so this is for convenience.
140 if err := dynamicCertificateController.RunOnce(); err != nil {
141 klog.Warningf("Initial population of dynamic certificates failed: %v", err)
142 }
143 go dynamicCertificateController.Run(1, stopCh)
144
145 tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient
146 }
147
148 return tlsConfig, nil
149 }
150
151 // Serve runs the secure http server. It fails only if certificates cannot be loaded or the initial listen call fails.
152 // The actual server loop (stoppable by closing stopCh) runs in a go routine, i.e. Serve does not block.
153 // It returns a stoppedCh that is closed when all non-hijacked active requests have been processed.
154 // It returns a listenerStoppedCh that is closed when the underlying http Server has stopped listening.
155 func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) (<-chan struct{}, <-chan struct{}, error) {
156 if s.Listener == nil {
157 return nil, nil, fmt.Errorf("listener must not be nil")
158 }
159
160 tlsConfig, err := s.tlsConfig(stopCh)
161 if err != nil {
162 return nil, nil, err
163 }
164
165 secureServer := &http.Server{
166 Addr: s.Listener.Addr().String(),
167 Handler: handler,
168 MaxHeaderBytes: 1 << 20,
169 TLSConfig: tlsConfig,
170
171 IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout
172 ReadHeaderTimeout: 32 * time.Second, // just shy of requestTimeoutUpperBound
173 }
174
175 if !s.DisableHTTP2 {
176 // At least 99% of serialized resources in surveyed clusters were smaller than 256kb.
177 // This should be big enough to accommodate most API POST requests in a single frame,
178 // and small enough to allow a per connection buffer of this size multiplied by `MaxConcurrentStreams`.
179 const resourceBody99Percentile = 256 * 1024
180
181 http2Options := &http2.Server{
182 IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout
183 // shrink the per-stream buffer and max framesize from the 1MB default while still accommodating most API POST requests in a single frame
184 MaxUploadBufferPerStream: resourceBody99Percentile,
185 MaxReadFrameSize: resourceBody99Percentile,
186 }
187
188 // use the overridden concurrent streams setting or make the default of 250 explicit so we can size MaxUploadBufferPerConnection appropriately
189 if s.HTTP2MaxStreamsPerConnection > 0 {
190 http2Options.MaxConcurrentStreams = uint32(s.HTTP2MaxStreamsPerConnection)
191 } else {
192 // match http2.initialMaxConcurrentStreams used by clients
193 // this makes it so that a malicious client can only open 400 streams before we forcibly close the connection
194 // https://github.com/golang/net/commit/b225e7ca6dde1ef5a5ae5ce922861bda011cfabd
195 http2Options.MaxConcurrentStreams = 100
196 }
197
198 // increase the connection buffer size from the 1MB default to handle the specified number of concurrent streams
199 http2Options.MaxUploadBufferPerConnection = http2Options.MaxUploadBufferPerStream * int32(http2Options.MaxConcurrentStreams)
200 // apply settings to the server
201 if err := http2.ConfigureServer(secureServer, http2Options); err != nil {
202 return nil, nil, fmt.Errorf("error configuring http2: %v", err)
203 }
204 }
205
206 // use tlsHandshakeErrorWriter to handle messages of tls handshake error
207 tlsErrorWriter := &tlsHandshakeErrorWriter{os.Stderr}
208 tlsErrorLogger := log.New(tlsErrorWriter, "", 0)
209 secureServer.ErrorLog = tlsErrorLogger
210
211 klog.Infof("Serving securely on %s", secureServer.Addr)
212 return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh)
213 }
214
215 // RunServer spawns a go-routine continuously serving until the stopCh is
216 // closed.
217 // It returns a stoppedCh that is closed when all non-hijacked active requests
218 // have been processed.
219 // This function does not block
220 // TODO: make private when insecure serving is gone from the kube-apiserver
221 func RunServer(
222 server *http.Server,
223 ln net.Listener,
224 shutDownTimeout time.Duration,
225 stopCh <-chan struct{},
226 ) (<-chan struct{}, <-chan struct{}, error) {
227 if ln == nil {
228 return nil, nil, fmt.Errorf("listener must not be nil")
229 }
230
231 // Shutdown server gracefully.
232 serverShutdownCh, listenerStoppedCh := make(chan struct{}), make(chan struct{})
233 go func() {
234 defer close(serverShutdownCh)
235 <-stopCh
236 ctx, cancel := context.WithTimeout(context.Background(), shutDownTimeout)
237 defer cancel()
238 err := server.Shutdown(ctx)
239 if err != nil {
240 klog.Errorf("Failed to shutdown server: %v", err)
241 }
242 }()
243
244 go func() {
245 defer utilruntime.HandleCrash()
246 defer close(listenerStoppedCh)
247
248 var listener net.Listener
249 listener = tcpKeepAliveListener{ln}
250 if server.TLSConfig != nil {
251 listener = tls.NewListener(listener, server.TLSConfig)
252 }
253
254 err := server.Serve(listener)
255
256 msg := fmt.Sprintf("Stopped listening on %s", ln.Addr().String())
257 select {
258 case <-stopCh:
259 klog.Info(msg)
260 default:
261 panic(fmt.Sprintf("%s due to error: %v", msg, err))
262 }
263 }()
264
265 return serverShutdownCh, listenerStoppedCh, nil
266 }
267
268 // tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
269 // connections. It's used by ListenAndServe and ListenAndServeTLS so
270 // dead TCP connections (e.g. closing laptop mid-download) eventually
271 // go away.
272 //
273 // Copied from Go 1.7.2 net/http/server.go
274 type tcpKeepAliveListener struct {
275 net.Listener
276 }
277
278 func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
279 c, err := ln.Listener.Accept()
280 if err != nil {
281 return nil, err
282 }
283 if tc, ok := c.(*net.TCPConn); ok {
284 tc.SetKeepAlive(true)
285 tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
286 }
287 return c, nil
288 }
289
290 // tlsHandshakeErrorWriter writes TLS handshake errors to klog with
291 // trace level - V(5), to avoid flooding of tls handshake errors.
292 type tlsHandshakeErrorWriter struct {
293 out io.Writer
294 }
295
296 const tlsHandshakeErrorPrefix = "http: TLS handshake error"
297
298 func (w *tlsHandshakeErrorWriter) Write(p []byte) (int, error) {
299 if strings.Contains(string(p), tlsHandshakeErrorPrefix) {
300 klog.V(5).Info(string(p))
301 metrics.TLSHandshakeErrors.Inc()
302 return len(p), nil
303 }
304
305 // for non tls handshake error, log it as usual
306 return w.out.Write(p)
307 }