| 1 |
/* |
| 2 |
Copyright 2019 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package dynamiccertificates |
| 18 |
|
| 19 |
import ( |
| 20 |
"crypto/tls" |
| 21 |
"crypto/x509" |
| 22 |
"errors" |
| 23 |
"fmt" |
| 24 |
"net" |
| 25 |
"sync/atomic" |
| 26 |
"time" |
| 27 |
|
| 28 |
corev1 "k8s.io/api/core/v1" |
| 29 |
utilruntime "k8s.io/apimachinery/pkg/util/runtime" |
| 30 |
"k8s.io/apimachinery/pkg/util/wait" |
| 31 |
"k8s.io/client-go/tools/events" |
| 32 |
"k8s.io/client-go/util/cert" |
| 33 |
"k8s.io/client-go/util/workqueue" |
| 34 |
"k8s.io/klog/v2" |
| 35 |
) |
| 36 |
|
| 37 |
const workItemKey = "key" |
| 38 |
|
| 39 |
// DynamicServingCertificateController dynamically loads certificates and provides a golang tls compatible dynamic GetCertificate func. |
| 40 |
type DynamicServingCertificateController struct { |
| 41 |
// baseTLSConfig is the static portion of the tlsConfig for serving to clients. It is copied and the copy is mutated |
| 42 |
// based on the dynamic cert state. |
| 43 |
baseTLSConfig *tls.Config |
| 44 |
|
| 45 |
// clientCA provides the very latest content of the ca bundle |
| 46 |
clientCA CAContentProvider |
| 47 |
// servingCert provides the very latest content of the default serving certificate |
| 48 |
servingCert CertKeyContentProvider |
| 49 |
// sniCerts are a list of CertKeyContentProvider with associated names used for SNI |
| 50 |
sniCerts []SNICertKeyContentProvider |
| 51 |
|
| 52 |
// currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a |
| 53 |
// new atomic value. The types used for efficient TLSConfig preclude using the processed value. |
| 54 |
currentlyServedContent *dynamicCertificateContent |
| 55 |
// currentServingTLSConfig holds a *tls.Config that will be used to serve requests |
| 56 |
currentServingTLSConfig atomic.Value |
| 57 |
|
| 58 |
// queue only ever has one item, but it has nice error handling backoff/retry semantics |
| 59 |
queue workqueue.TypedRateLimitingInterface[string] |
| 60 |
eventRecorder events.EventRecorder |
| 61 |
} |
| 62 |
|
| 63 |
var _ Listener = &DynamicServingCertificateController{} |
| 64 |
|
| 65 |
// NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date. |
| 66 |
func NewDynamicServingCertificateController( |
| 67 |
baseTLSConfig *tls.Config, |
| 68 |
clientCA CAContentProvider, |
| 69 |
servingCert CertKeyContentProvider, |
| 70 |
sniCerts []SNICertKeyContentProvider, |
| 71 |
eventRecorder events.EventRecorder, |
| 72 |
) *DynamicServingCertificateController { |
| 73 |
c := &DynamicServingCertificateController{ |
| 74 |
baseTLSConfig: baseTLSConfig, |
| 75 |
clientCA: clientCA, |
| 76 |
servingCert: servingCert, |
| 77 |
sniCerts: sniCerts, |
| 78 |
|
| 79 |
queue: workqueue.NewTypedRateLimitingQueueWithConfig( |
| 80 |
workqueue.DefaultTypedControllerRateLimiter[string](), |
| 81 |
workqueue.TypedRateLimitingQueueConfig[string]{Name: "DynamicServingCertificateController"}, |
| 82 |
), |
| 83 |
eventRecorder: eventRecorder, |
| 84 |
} |
| 85 |
|
| 86 |
return c |
| 87 |
} |
| 88 |
|
| 89 |
// GetConfigForClient is an implementation of tls.Config.GetConfigForClient |
| 90 |
func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls.Config, error) { |
| 91 |
uncastObj := c.currentServingTLSConfig.Load() |
| 92 |
if uncastObj == nil { |
| 93 |
return nil, errors.New("dynamiccertificates: configuration not ready") |
| 94 |
} |
| 95 |
tlsConfig, ok := uncastObj.(*tls.Config) |
| 96 |
if !ok { |
| 97 |
return nil, errors.New("dynamiccertificates: unexpected config type") |
| 98 |
} |
| 99 |
|
| 100 |
tlsConfigCopy := tlsConfig.Clone() |
| 101 |
|
| 102 |
// if the client set SNI information, just use our "normal" SNI flow |
| 103 |
if len(clientHello.ServerName) > 0 { |
| 104 |
return tlsConfigCopy, nil |
| 105 |
} |
| 106 |
|
| 107 |
// if the client didn't set SNI, then we need to inspect the requested IP so that we can choose |
| 108 |
// a certificate from our list if we specifically handle that IP. This can happen when an IP is specifically mapped by name. |
| 109 |
host, _, err := net.SplitHostPort(clientHello.Conn.LocalAddr().String()) |
| 110 |
if err != nil { |
| 111 |
return tlsConfigCopy, nil |
| 112 |
} |
| 113 |
|
| 114 |
ipCert, ok := tlsConfigCopy.NameToCertificate[host] |
| 115 |
if !ok { |
| 116 |
return tlsConfigCopy, nil |
| 117 |
} |
| 118 |
tlsConfigCopy.Certificates = []tls.Certificate{*ipCert} |
| 119 |
tlsConfigCopy.NameToCertificate = nil |
| 120 |
|
| 121 |
return tlsConfigCopy, nil |
| 122 |
} |
| 123 |
|
| 124 |
// newTLSContent determines the next set of content for overriding the baseTLSConfig. |
| 125 |
func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) { |
| 126 |
newContent := &dynamicCertificateContent{} |
| 127 |
|
| 128 |
if c.clientCA != nil { |
| 129 |
currClientCABundle := c.clientCA.CurrentCABundleContent() |
| 130 |
// we allow removing all client ca bundles because the server is still secure when this happens. it just means |
| 131 |
// that there isn't a hint to clients about which client-cert to used. this happens when there is no client-ca |
| 132 |
// yet known for authentication, which can happen in aggregated apiservers and some kube-apiserver deployment modes. |
| 133 |
newContent.clientCA = caBundleContent{caBundle: currClientCABundle} |
| 134 |
} |
| 135 |
|
| 136 |
if c.servingCert != nil { |
| 137 |
currServingCert, currServingKey := c.servingCert.CurrentCertKeyContent() |
| 138 |
if len(currServingCert) == 0 || len(currServingKey) == 0 { |
| 139 |
return nil, fmt.Errorf("not loading an empty serving certificate from %q", c.servingCert.Name()) |
| 140 |
} |
| 141 |
|
| 142 |
newContent.servingCert = certKeyContent{cert: currServingCert, key: currServingKey} |
| 143 |
} |
| 144 |
|
| 145 |
for i, sniCert := range c.sniCerts { |
| 146 |
currCert, currKey := sniCert.CurrentCertKeyContent() |
| 147 |
if len(currCert) == 0 || len(currKey) == 0 { |
| 148 |
return nil, fmt.Errorf("not loading an empty SNI certificate from %d/%q", i, sniCert.Name()) |
| 149 |
} |
| 150 |
|
| 151 |
newContent.sniCerts = append(newContent.sniCerts, sniCertKeyContent{certKeyContent: certKeyContent{cert: currCert, key: currKey}, sniNames: sniCert.SNINames()}) |
| 152 |
} |
| 153 |
|
| 154 |
return newContent, nil |
| 155 |
} |
| 156 |
|
| 157 |
// syncCerts gets newTLSContent, if it has changed from the existing, the content is parsed and stored for usage in |
| 158 |
// GetConfigForClient. |
| 159 |
func (c *DynamicServingCertificateController) syncCerts() error { |
| 160 |
newContent, err := c.newTLSContent() |
| 161 |
if err != nil { |
| 162 |
return err |
| 163 |
} |
| 164 |
// if the content is the same as what we currently have, we can simply skip it. This works because we are single |
| 165 |
// threaded. If you ever make this multi-threaded, add a lock. |
| 166 |
if newContent.Equal(c.currentlyServedContent) { |
| 167 |
return nil |
| 168 |
} |
| 169 |
|
| 170 |
// make a shallow copy and override the dynamic pieces which have changed. |
| 171 |
newTLSConfigCopy := c.baseTLSConfig.Clone() |
| 172 |
|
| 173 |
// parse new content to add to TLSConfig |
| 174 |
if len(newContent.clientCA.caBundle) > 0 { |
| 175 |
newClientCAPool := x509.NewCertPool() |
| 176 |
newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle) |
| 177 |
if err != nil { |
| 178 |
return fmt.Errorf("unable to load client CA file %q: %v", string(newContent.clientCA.caBundle), err) |
| 179 |
} |
| 180 |
for i, cert := range newClientCAs { |
| 181 |
klog.V(2).InfoS("Loaded client CA", "index", i, "certName", c.clientCA.Name(), "certDetail", GetHumanCertDetail(cert)) |
| 182 |
if c.eventRecorder != nil { |
| 183 |
c.eventRecorder.Eventf(&corev1.ObjectReference{Name: c.clientCA.Name()}, nil, corev1.EventTypeWarning, "TLSConfigChanged", "CACertificateReload", "loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert)) |
| 184 |
} |
| 185 |
|
| 186 |
newClientCAPool.AddCert(cert) |
| 187 |
} |
| 188 |
|
| 189 |
newTLSConfigCopy.ClientCAs = newClientCAPool |
| 190 |
} |
| 191 |
|
| 192 |
if len(newContent.servingCert.cert) > 0 && len(newContent.servingCert.key) > 0 { |
| 193 |
cert, err := tls.X509KeyPair(newContent.servingCert.cert, newContent.servingCert.key) |
| 194 |
if err != nil { |
| 195 |
return fmt.Errorf("invalid serving cert keypair: %v", err) |
| 196 |
} |
| 197 |
|
| 198 |
x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) |
| 199 |
if err != nil { |
| 200 |
return fmt.Errorf("invalid serving cert: %v", err) |
| 201 |
} |
| 202 |
|
| 203 |
klog.V(2).InfoS("Loaded serving cert", "certName", c.servingCert.Name(), "certDetail", GetHumanCertDetail(x509Cert)) |
| 204 |
if c.eventRecorder != nil { |
| 205 |
c.eventRecorder.Eventf(&corev1.ObjectReference{Name: c.servingCert.Name()}, nil, corev1.EventTypeWarning, "TLSConfigChanged", "ServingCertificateReload", "loaded serving cert [%q]: %s", c.servingCert.Name(), GetHumanCertDetail(x509Cert)) |
| 206 |
} |
| 207 |
|
| 208 |
newTLSConfigCopy.Certificates = []tls.Certificate{cert} |
| 209 |
} |
| 210 |
|
| 211 |
if len(newContent.sniCerts) > 0 { |
| 212 |
newTLSConfigCopy.NameToCertificate, err = c.BuildNamedCertificates(newContent.sniCerts) |
| 213 |
if err != nil { |
| 214 |
return fmt.Errorf("unable to build named certificate map: %v", err) |
| 215 |
} |
| 216 |
|
| 217 |
// append all named certs. Otherwise, the go tls stack will think no SNI processing |
| 218 |
// is necessary because there is only one cert anyway. |
| 219 |
// Moreover, if servingCert is not set, the first SNI |
| 220 |
// cert will become the default cert. That's what we expect anyway. |
| 221 |
for _, sniCert := range newTLSConfigCopy.NameToCertificate { |
| 222 |
newTLSConfigCopy.Certificates = append(newTLSConfigCopy.Certificates, *sniCert) |
| 223 |
} |
| 224 |
} |
| 225 |
|
| 226 |
// store new values of content for serving. |
| 227 |
c.currentServingTLSConfig.Store(newTLSConfigCopy) |
| 228 |
c.currentlyServedContent = newContent // this is single threaded, so we have no locking issue |
| 229 |
|
| 230 |
return nil |
| 231 |
} |
| 232 |
|
| 233 |
// RunOnce runs a single sync step to ensure that we have a valid starting configuration. |
| 234 |
func (c *DynamicServingCertificateController) RunOnce() error { |
| 235 |
return c.syncCerts() |
| 236 |
} |
| 237 |
|
| 238 |
// Run starts the kube-apiserver and blocks until stopCh is closed. |
| 239 |
func (c *DynamicServingCertificateController) Run(workers int, stopCh <-chan struct{}) { |
| 240 |
defer utilruntime.HandleCrash() |
| 241 |
defer c.queue.ShutDown() |
| 242 |
|
| 243 |
klog.InfoS("Starting DynamicServingCertificateController") |
| 244 |
defer klog.InfoS("Shutting down DynamicServingCertificateController") |
| 245 |
|
| 246 |
// synchronously load once. We will trigger again, so ignoring any error is fine |
| 247 |
_ = c.RunOnce() |
| 248 |
|
| 249 |
// doesn't matter what workers say, only start one. |
| 250 |
go wait.Until(c.runWorker, time.Second, stopCh) |
| 251 |
|
| 252 |
// start timer that rechecks every minute, just in case. this also serves to prime the controller quickly. |
| 253 |
go wait.Until(func() { |
| 254 |
c.Enqueue() |
| 255 |
}, 1*time.Minute, stopCh) |
| 256 |
|
| 257 |
<-stopCh |
| 258 |
} |
| 259 |
|
| 260 |
func (c *DynamicServingCertificateController) runWorker() { |
| 261 |
for c.processNextWorkItem() { |
| 262 |
} |
| 263 |
} |
| 264 |
|
| 265 |
func (c *DynamicServingCertificateController) processNextWorkItem() bool { |
| 266 |
dsKey, quit := c.queue.Get() |
| 267 |
if quit { |
| 268 |
return false |
| 269 |
} |
| 270 |
defer c.queue.Done(dsKey) |
| 271 |
|
| 272 |
err := c.syncCerts() |
| 273 |
if err == nil { |
| 274 |
c.queue.Forget(dsKey) |
| 275 |
return true |
| 276 |
} |
| 277 |
|
| 278 |
utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err)) |
| 279 |
c.queue.AddRateLimited(dsKey) |
| 280 |
|
| 281 |
return true |
| 282 |
} |
| 283 |
|
| 284 |
// Enqueue a method to allow separate control loops to cause the certificate controller to trigger and read content. |
| 285 |
func (c *DynamicServingCertificateController) Enqueue() { |
| 286 |
c.queue.Add(workItemKey) |
| 287 |
} |