Debian Codegraph @

pkg/storage/storagebackend/factory/etcd3.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package factory
18
19 import (
20 "context"
21 "fmt"
22 "log"
23 "math/rand"
24 "net"
25 "net/url"
26 "os"
27 "path"
28 "strings"
29 "sync"
30 "time"
31
32 grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
33 "go.etcd.io/etcd/client/pkg/v3/logutil"
34 "go.etcd.io/etcd/client/pkg/v3/transport"
35 clientv3 "go.etcd.io/etcd/client/v3"
36 "go.etcd.io/etcd/client/v3/kubernetes"
37 "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
38 "go.uber.org/zap"
39 "go.uber.org/zap/zapcore"
40 "golang.org/x/time/rate"
41 "google.golang.org/grpc"
42 "k8s.io/klog/v2"
43
44 "k8s.io/apimachinery/pkg/runtime"
45 utilnet "k8s.io/apimachinery/pkg/util/net"
46 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
47 "k8s.io/apimachinery/pkg/util/wait"
48 genericfeatures "k8s.io/apiserver/pkg/features"
49 "k8s.io/apiserver/pkg/server/egressselector"
50 "k8s.io/apiserver/pkg/storage"
51 "k8s.io/apiserver/pkg/storage/etcd3"
52 "k8s.io/apiserver/pkg/storage/etcd3/metrics"
53 "k8s.io/apiserver/pkg/storage/storagebackend"
54 "k8s.io/apiserver/pkg/storage/value/encrypt/identity"
55 utilfeature "k8s.io/apiserver/pkg/util/feature"
56 "k8s.io/component-base/metrics/legacyregistry"
57 tracing "k8s.io/component-base/tracing"
58 )
59
60 const (
61 // The short keepalive timeout and interval have been chosen to aggressively
62 // detect a failed etcd server without introducing much overhead.
63 keepaliveTime = 30 * time.Second
64 keepaliveTimeout = 10 * time.Second
65
66 // dialTimeout is the timeout for failing to establish a connection.
67 // It is set to 20 seconds as times shorter than that will cause TLS connections to fail
68 // on heavily loaded arm64 CPUs (issue #64649)
69 dialTimeout = 20 * time.Second
70
71 dbMetricsMonitorJitter = 0.5
72 )
73
74 // TODO(negz): Stop using a package scoped logger. At the time of writing we're
75 // creating an etcd client for each CRD. We need to pass each etcd client a
76 // logger or each client will create its own, which comes with a significant
77 // memory cost (around 20% of the API server's memory when hundreds of CRDs are
78 // present). The correct fix here is to not create a client per CRD. See
79 // https://github.com/kubernetes/kubernetes/issues/111476 for more.
80 var etcd3ClientLogger *zap.Logger
81
82 func init() {
83 // grpcprom auto-registers (via an init function) their client metrics, since we are opting out of
84 // using the global prometheus registry and using our own wrapped global registry,
85 // we need to explicitly register these metrics to our global registry here.
86 // For reference: https://github.com/kubernetes/kubernetes/pull/81387
87 legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics)
88 dbMetricsMonitors = make(map[string]struct{})
89
90 l, err := logutil.CreateDefaultZapLogger(etcdClientDebugLevel())
91 if err != nil {
92 l = zap.NewNop()
93 }
94 etcd3ClientLogger = l.Named("etcd-client")
95 }
96
97 // etcdClientDebugLevel translates ETCD_CLIENT_DEBUG into zap log level.
98 // NOTE(negz): This is a copy of a private etcd client function:
99 // https://github.com/etcd-io/etcd/blob/v3.5.4/client/v3/logger.go#L47
100 func etcdClientDebugLevel() zapcore.Level {
101 envLevel := os.Getenv("ETCD_CLIENT_DEBUG")
102 if envLevel == "" || envLevel == "true" {
103 return zapcore.InfoLevel
104 }
105 var l zapcore.Level
106 if err := l.Set(envLevel); err == nil {
107 log.Printf("Deprecated env ETCD_CLIENT_DEBUG value. Using default level: 'info'")
108 return zapcore.InfoLevel
109 }
110 return l
111 }
112
113 func newETCD3HealthCheck(c storagebackend.Config, stopCh <-chan struct{}) (func() error, error) {
114 timeout := storagebackend.DefaultHealthcheckTimeout
115 if c.HealthcheckTimeout != time.Duration(0) {
116 timeout = c.HealthcheckTimeout
117 }
118 return newETCD3Check(c, timeout, stopCh)
119 }
120
121 func newETCD3ReadyCheck(c storagebackend.Config, stopCh <-chan struct{}) (func() error, error) {
122 timeout := storagebackend.DefaultReadinessTimeout
123 if c.ReadycheckTimeout != time.Duration(0) {
124 timeout = c.ReadycheckTimeout
125 }
126 return newETCD3Check(c, timeout, stopCh)
127 }
128
129 // atomic error acts as a cache for atomically store an error
130 // the error is only updated if the timestamp is more recent than
131 // current stored error.
132 type atomicLastError struct {
133 mu sync.RWMutex
134 err error
135 timestamp time.Time
136 }
137
138 func (a *atomicLastError) Store(err error, t time.Time) {
139 a.mu.Lock()
140 defer a.mu.Unlock()
141 if a.timestamp.IsZero() || a.timestamp.Before(t) {
142 a.err = err
143 a.timestamp = t
144 }
145 }
146
147 func (a *atomicLastError) Load() error {
148 a.mu.RLock()
149 defer a.mu.RUnlock()
150 return a.err
151 }
152
153 func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan struct{}) (func() error, error) {
154 // constructing the etcd v3 client blocks and times out if etcd is not available.
155 // retry in a loop in the background until we successfully create the client, storing the client or error encountered
156
157 lock := sync.RWMutex{}
158 var prober *etcd3ProberMonitor
159 clientErr := fmt.Errorf("etcd client connection not yet established")
160
161 go wait.PollImmediateUntil(time.Second, func() (bool, error) {
162 lock.Lock()
163 defer lock.Unlock()
164 newProber, err := newETCD3ProberMonitor(c)
165 // Ensure that server is already not shutting down.
166 select {
167 case <-stopCh:
168 if err == nil {
169 newProber.Close()
170 }
171 return true, nil
172 default:
173 }
174 if err != nil {
175 clientErr = err
176 return false, nil
177 }
178 prober = newProber
179 clientErr = nil
180 return true, nil
181 }, stopCh)
182
183 // Close the client on shutdown.
184 go func() {
185 defer utilruntime.HandleCrash()
186 <-stopCh
187
188 lock.Lock()
189 defer lock.Unlock()
190 if prober != nil {
191 prober.Close()
192 clientErr = fmt.Errorf("server is shutting down")
193 }
194 }()
195
196 // limit to a request every half of the configured timeout with a maximum burst of one
197 // rate limited requests will receive the last request sent error (note: not the last received response)
198 limiter := rate.NewLimiter(rate.Every(timeout/2), 1)
199 // initial state is the clientErr
200 lastError := &atomicLastError{err: fmt.Errorf("etcd client connection not yet established")}
201
202 return func() error {
203 // Given that client is closed on shutdown we hold the lock for
204 // the entire period of healthcheck call to ensure that client will
205 // not be closed during healthcheck.
206 // Given that healthchecks has a 2s timeout, worst case of blocking
207 // shutdown for additional 2s seems acceptable.
208 lock.RLock()
209 defer lock.RUnlock()
210
211 if clientErr != nil {
212 return clientErr
213 }
214 if limiter.Allow() == false {
215 return lastError.Load()
216 }
217 ctx, cancel := context.WithTimeout(context.Background(), timeout)
218 defer cancel()
219 now := time.Now()
220 err := prober.Probe(ctx)
221 lastError.Store(err, now)
222 return err
223 }, nil
224 }
225
226 func newETCD3ProberMonitor(c storagebackend.Config) (*etcd3ProberMonitor, error) {
227 client, err := newETCD3Client(c.Transport)
228 if err != nil {
229 return nil, err
230 }
231 return &etcd3ProberMonitor{
232 client: client.Client,
233 prefix: c.Prefix,
234 endpoints: c.Transport.ServerList,
235 }, nil
236 }
237
238 type etcd3ProberMonitor struct {
239 prefix string
240 endpoints []string
241
242 mux sync.RWMutex
243 client *clientv3.Client
244 closed bool
245 }
246
247 func (t *etcd3ProberMonitor) Close() error {
248 t.mux.Lock()
249 defer t.mux.Unlock()
250 if !t.closed {
251 t.closed = true
252 return t.client.Close()
253 }
254 return fmt.Errorf("closed")
255 }
256
257 func (t *etcd3ProberMonitor) Probe(ctx context.Context) error {
258 t.mux.RLock()
259 defer t.mux.RUnlock()
260 if t.closed {
261 return fmt.Errorf("closed")
262 }
263 // See https://github.com/etcd-io/etcd/blob/c57f8b3af865d1b531b979889c602ba14377420e/etcdctl/ctlv3/command/ep_command.go#L118
264 _, err := t.client.Get(ctx, path.Join("/", t.prefix, "health"))
265 if err != nil {
266 return fmt.Errorf("error getting data from etcd: %w", err)
267 }
268 return nil
269 }
270
271 func (t *etcd3ProberMonitor) Monitor(ctx context.Context) (metrics.StorageMetrics, error) {
272 t.mux.RLock()
273 defer t.mux.RUnlock()
274 if t.closed {
275 return metrics.StorageMetrics{}, fmt.Errorf("closed")
276 }
277 status, err := t.client.Status(ctx, t.endpoints[rand.Int()%len(t.endpoints)])
278 if err != nil {
279 return metrics.StorageMetrics{}, err
280 }
281 return metrics.StorageMetrics{
282 Size: status.DbSize,
283 }, nil
284 }
285
286 var newETCD3Client = func(c storagebackend.TransportConfig) (*kubernetes.Client, error) {
287 tlsInfo := transport.TLSInfo{
288 CertFile: c.CertFile,
289 KeyFile: c.KeyFile,
290 TrustedCAFile: c.TrustedCAFile,
291 }
292 tlsConfig, err := tlsInfo.ClientConfig()
293 if err != nil {
294 return nil, err
295 }
296 // NOTE: Client relies on nil tlsConfig
297 // for non-secure connections, update the implicit variable
298 if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.TrustedCAFile) == 0 {
299 tlsConfig = nil
300 }
301 networkContext := egressselector.Etcd.AsNetworkContext()
302 var egressDialer utilnet.DialFunc
303 if c.EgressLookup != nil {
304 egressDialer, err = c.EgressLookup(networkContext)
305 if err != nil {
306 return nil, err
307 }
308 }
309 dialOptions := []grpc.DialOption{
310 grpc.WithBlock(), // block until the underlying connection is up
311 // use chained interceptors so that the default (retry and backoff) interceptors are added.
312 // otherwise they will be overwritten by the metric interceptor.
313 //
314 // these optional interceptors will be placed after the default ones.
315 // which seems to be what we want as the metrics will be collected on each attempt (retry)
316 grpc.WithChainUnaryInterceptor(grpcprom.UnaryClientInterceptor),
317 grpc.WithChainStreamInterceptor(grpcprom.StreamClientInterceptor),
318 }
319 if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
320 tracingOpts := []otelgrpc.Option{
321 otelgrpc.WithMessageEvents(otelgrpc.ReceivedEvents, otelgrpc.SentEvents),
322 otelgrpc.WithPropagators(tracing.Propagators()),
323 otelgrpc.WithTracerProvider(c.TracerProvider),
324 }
325 // Even with Noop TracerProvider, the otelgrpc still handles context propagation.
326 // See https://github.com/open-telemetry/opentelemetry-go/tree/main/example/passthrough
327 dialOptions = append(dialOptions,
328 grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor(tracingOpts...)),
329 grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor(tracingOpts...)))
330 }
331 if egressDialer != nil {
332 dialer := func(ctx context.Context, addr string) (net.Conn, error) {
333 if strings.Contains(addr, "//") {
334 // etcd client prior to 3.5 passed URLs to dialer, normalize to address
335 u, err := url.Parse(addr)
336 if err != nil {
337 return nil, err
338 }
339 addr = u.Host
340 }
341 return egressDialer(ctx, "tcp", addr)
342 }
343 dialOptions = append(dialOptions, grpc.WithContextDialer(dialer))
344 }
345
346 cfg := clientv3.Config{
347 DialTimeout: dialTimeout,
348 DialKeepAliveTime: keepaliveTime,
349 DialKeepAliveTimeout: keepaliveTimeout,
350 DialOptions: dialOptions,
351 Endpoints: c.ServerList,
352 TLS: tlsConfig,
353 Logger: etcd3ClientLogger,
354 }
355
356 return kubernetes.New(cfg)
357 }
358
359 type runningCompactor struct {
360 interval time.Duration
361 cancel context.CancelFunc
362 client *clientv3.Client
363 refs int
364 }
365
366 var (
367 // compactorsMu guards access to compactors map
368 compactorsMu sync.Mutex
369 compactors = map[string]*runningCompactor{}
370 // dbMetricsMonitorsMu guards access to dbMetricsMonitors map
371 dbMetricsMonitorsMu sync.Mutex
372 dbMetricsMonitors map[string]struct{}
373 )
374
375 // startCompactorOnce start one compactor per transport. If the interval get smaller on repeated calls, the
376 // compactor is replaced. A destroy func is returned. If all destroy funcs with the same transport are called,
377 // the compactor is stopped.
378 func startCompactorOnce(c storagebackend.TransportConfig, interval time.Duration) (func(), error) {
379 compactorsMu.Lock()
380 defer compactorsMu.Unlock()
381
382 if interval == 0 {
383 // short circuit, if the compaction request from apiserver is disabled
384 return func() {}, nil
385 }
386 key := fmt.Sprintf("%v", c) // gives: {[server1 server2] keyFile certFile caFile}
387 if compactor, foundBefore := compactors[key]; !foundBefore || compactor.interval > interval {
388 client, err := newETCD3Client(c)
389 if err != nil {
390 return nil, err
391 }
392 compactorClient := client.Client
393
394 if foundBefore {
395 // replace compactor
396 compactor.cancel()
397 compactor.client.Close()
398 } else {
399 // start new compactor
400 compactor = &runningCompactor{}
401 compactors[key] = compactor
402 }
403
404 ctx, cancel := context.WithCancel(context.Background())
405
406 compactor.interval = interval
407 compactor.cancel = cancel
408 compactor.client = compactorClient
409
410 etcd3.StartCompactor(ctx, compactorClient, interval)
411 }
412
413 compactors[key].refs++
414
415 return func() {
416 compactorsMu.Lock()
417 defer compactorsMu.Unlock()
418
419 compactor := compactors[key]
420 compactor.refs--
421 if compactor.refs == 0 {
422 compactor.cancel()
423 compactor.client.Close()
424 delete(compactors, key)
425 }
426 }, nil
427 }
428
429 func newETCD3Storage(c storagebackend.ConfigForResource, newFunc, newListFunc func() runtime.Object, resourcePrefix string) (storage.Interface, DestroyFunc, error) {
430 stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
431 if err != nil {
432 return nil, nil, err
433 }
434
435 client, err := newETCD3Client(c.Transport)
436 if err != nil {
437 stopCompactor()
438 return nil, nil, err
439 }
440
441 // decorate the KV instance so we can track etcd latency per request.
442 client.KV = etcd3.NewETCDLatencyTracker(client.KV)
443
444 stopDBSizeMonitor, err := startDBSizeMonitorPerEndpoint(client.Client, c.DBMetricPollInterval)
445 if err != nil {
446 return nil, nil, err
447 }
448
449 var once sync.Once
450 destroyFunc := func() {
451 // we know that storage destroy funcs are called multiple times (due to reuse in subresources).
452 // Hence, we only destroy once.
453 // TODO: fix duplicated storage destroy calls higher level
454 once.Do(func() {
455 stopCompactor()
456 stopDBSizeMonitor()
457 client.Close()
458 })
459 }
460 transformer := c.Transformer
461 if transformer == nil {
462 transformer = identity.NewEncryptCheckTransformer()
463 }
464
465 versioner := storage.APIObjectVersioner{}
466 decoder := etcd3.NewDefaultDecoder(c.Codec, versioner)
467 store := etcd3.New(client, c.Codec, newFunc, newListFunc, c.Prefix, resourcePrefix, c.GroupResource, transformer, c.LeaseManagerConfig, decoder, versioner)
468 return store, destroyFunc, nil
469 }
470
471 // startDBSizeMonitorPerEndpoint starts a loop to monitor etcd database size and update the
472 // corresponding metric etcd_db_total_size_in_bytes for each etcd server endpoint.
473 // Deprecated: Will be replaced with newETCD3ProberMonitor
474 func startDBSizeMonitorPerEndpoint(client *clientv3.Client, interval time.Duration) (func(), error) {
475 if interval == 0 {
476 return func() {}, nil
477 }
478 dbMetricsMonitorsMu.Lock()
479 defer dbMetricsMonitorsMu.Unlock()
480
481 ctx, cancel := context.WithCancel(context.Background())
482 for _, ep := range client.Endpoints() {
483 if _, found := dbMetricsMonitors[ep]; found {
484 continue
485 }
486 dbMetricsMonitors[ep] = struct{}{}
487 endpoint := ep
488 klog.V(4).Infof("Start monitoring storage db size metric for endpoint %s with polling interval %v", endpoint, interval)
489 go wait.JitterUntilWithContext(ctx, func(context.Context) {
490 epStatus, err := client.Maintenance.Status(ctx, endpoint)
491 if err != nil {
492 klog.V(4).Infof("Failed to get storage db size for ep %s: %v", endpoint, err)
493 metrics.UpdateEtcdDbSize(endpoint, -1)
494 } else {
495 metrics.UpdateEtcdDbSize(endpoint, epStatus.DbSize)
496 }
497 }, interval, dbMetricsMonitorJitter, true)
498 }
499
500 return func() {
501 cancel()
502 }, nil
503 }