| 1 |
/* |
| 2 |
Copyright 2016 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package factory |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"log" |
| 23 |
"math/rand" |
| 24 |
"net" |
| 25 |
"net/url" |
| 26 |
"os" |
| 27 |
"path" |
| 28 |
"strings" |
| 29 |
"sync" |
| 30 |
"time" |
| 31 |
|
| 32 |
grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" |
| 33 |
"go.etcd.io/etcd/client/pkg/v3/logutil" |
| 34 |
"go.etcd.io/etcd/client/pkg/v3/transport" |
| 35 |
clientv3 "go.etcd.io/etcd/client/v3" |
| 36 |
"go.etcd.io/etcd/client/v3/kubernetes" |
| 37 |
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" |
| 38 |
"go.uber.org/zap" |
| 39 |
"go.uber.org/zap/zapcore" |
| 40 |
"golang.org/x/time/rate" |
| 41 |
"google.golang.org/grpc" |
| 42 |
"k8s.io/klog/v2" |
| 43 |
|
| 44 |
"k8s.io/apimachinery/pkg/runtime" |
| 45 |
utilnet "k8s.io/apimachinery/pkg/util/net" |
| 46 |
utilruntime "k8s.io/apimachinery/pkg/util/runtime" |
| 47 |
"k8s.io/apimachinery/pkg/util/wait" |
| 48 |
genericfeatures "k8s.io/apiserver/pkg/features" |
| 49 |
"k8s.io/apiserver/pkg/server/egressselector" |
| 50 |
"k8s.io/apiserver/pkg/storage" |
| 51 |
"k8s.io/apiserver/pkg/storage/etcd3" |
| 52 |
"k8s.io/apiserver/pkg/storage/etcd3/metrics" |
| 53 |
"k8s.io/apiserver/pkg/storage/storagebackend" |
| 54 |
"k8s.io/apiserver/pkg/storage/value/encrypt/identity" |
| 55 |
utilfeature "k8s.io/apiserver/pkg/util/feature" |
| 56 |
"k8s.io/component-base/metrics/legacyregistry" |
| 57 |
tracing "k8s.io/component-base/tracing" |
| 58 |
) |
| 59 |
|
| 60 |
const ( |
| 61 |
// The short keepalive timeout and interval have been chosen to aggressively |
| 62 |
// detect a failed etcd server without introducing much overhead. |
| 63 |
keepaliveTime = 30 * time.Second |
| 64 |
keepaliveTimeout = 10 * time.Second |
| 65 |
|
| 66 |
// dialTimeout is the timeout for failing to establish a connection. |
| 67 |
// It is set to 20 seconds as times shorter than that will cause TLS connections to fail |
| 68 |
// on heavily loaded arm64 CPUs (issue #64649) |
| 69 |
dialTimeout = 20 * time.Second |
| 70 |
|
| 71 |
dbMetricsMonitorJitter = 0.5 |
| 72 |
) |
| 73 |
|
| 74 |
// TODO(negz): Stop using a package scoped logger. At the time of writing we're |
| 75 |
// creating an etcd client for each CRD. We need to pass each etcd client a |
| 76 |
// logger or each client will create its own, which comes with a significant |
| 77 |
// memory cost (around 20% of the API server's memory when hundreds of CRDs are |
| 78 |
// present). The correct fix here is to not create a client per CRD. See |
| 79 |
// https://github.com/kubernetes/kubernetes/issues/111476 for more. |
| 80 |
var etcd3ClientLogger *zap.Logger |
| 81 |
|
| 82 |
func init() { |
| 83 |
// grpcprom auto-registers (via an init function) their client metrics, since we are opting out of |
| 84 |
// using the global prometheus registry and using our own wrapped global registry, |
| 85 |
// we need to explicitly register these metrics to our global registry here. |
| 86 |
// For reference: https://github.com/kubernetes/kubernetes/pull/81387 |
| 87 |
legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics) |
| 88 |
dbMetricsMonitors = make(map[string]struct{}) |
| 89 |
|
| 90 |
l, err := logutil.CreateDefaultZapLogger(etcdClientDebugLevel()) |
| 91 |
if err != nil { |
| 92 |
l = zap.NewNop() |
| 93 |
} |
| 94 |
etcd3ClientLogger = l.Named("etcd-client") |
| 95 |
} |
| 96 |
|
| 97 |
// etcdClientDebugLevel translates ETCD_CLIENT_DEBUG into zap log level. |
| 98 |
// NOTE(negz): This is a copy of a private etcd client function: |
| 99 |
// https://github.com/etcd-io/etcd/blob/v3.5.4/client/v3/logger.go#L47 |
| 100 |
func etcdClientDebugLevel() zapcore.Level { |
| 101 |
envLevel := os.Getenv("ETCD_CLIENT_DEBUG") |
| 102 |
if envLevel == "" || envLevel == "true" { |
| 103 |
return zapcore.InfoLevel |
| 104 |
} |
| 105 |
var l zapcore.Level |
| 106 |
if err := l.Set(envLevel); err == nil { |
| 107 |
log.Printf("Deprecated env ETCD_CLIENT_DEBUG value. Using default level: 'info'") |
| 108 |
return zapcore.InfoLevel |
| 109 |
} |
| 110 |
return l |
| 111 |
} |
| 112 |
|
| 113 |
func newETCD3HealthCheck(c storagebackend.Config, stopCh <-chan struct{}) (func() error, error) { |
| 114 |
timeout := storagebackend.DefaultHealthcheckTimeout |
| 115 |
if c.HealthcheckTimeout != time.Duration(0) { |
| 116 |
timeout = c.HealthcheckTimeout |
| 117 |
} |
| 118 |
return newETCD3Check(c, timeout, stopCh) |
| 119 |
} |
| 120 |
|
| 121 |
func newETCD3ReadyCheck(c storagebackend.Config, stopCh <-chan struct{}) (func() error, error) { |
| 122 |
timeout := storagebackend.DefaultReadinessTimeout |
| 123 |
if c.ReadycheckTimeout != time.Duration(0) { |
| 124 |
timeout = c.ReadycheckTimeout |
| 125 |
} |
| 126 |
return newETCD3Check(c, timeout, stopCh) |
| 127 |
} |
| 128 |
|
| 129 |
// atomic error acts as a cache for atomically store an error |
| 130 |
// the error is only updated if the timestamp is more recent than |
| 131 |
// current stored error. |
| 132 |
type atomicLastError struct { |
| 133 |
mu sync.RWMutex |
| 134 |
err error |
| 135 |
timestamp time.Time |
| 136 |
} |
| 137 |
|
| 138 |
func (a *atomicLastError) Store(err error, t time.Time) { |
| 139 |
a.mu.Lock() |
| 140 |
defer a.mu.Unlock() |
| 141 |
if a.timestamp.IsZero() || a.timestamp.Before(t) { |
| 142 |
a.err = err |
| 143 |
a.timestamp = t |
| 144 |
} |
| 145 |
} |
| 146 |
|
| 147 |
func (a *atomicLastError) Load() error { |
| 148 |
a.mu.RLock() |
| 149 |
defer a.mu.RUnlock() |
| 150 |
return a.err |
| 151 |
} |
| 152 |
|
| 153 |
func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan struct{}) (func() error, error) { |
| 154 |
// constructing the etcd v3 client blocks and times out if etcd is not available. |
| 155 |
// retry in a loop in the background until we successfully create the client, storing the client or error encountered |
| 156 |
|
| 157 |
lock := sync.RWMutex{} |
| 158 |
var prober *etcd3ProberMonitor |
| 159 |
clientErr := fmt.Errorf("etcd client connection not yet established") |
| 160 |
|
| 161 |
go wait.PollImmediateUntil(time.Second, func() (bool, error) { |
| 162 |
lock.Lock() |
| 163 |
defer lock.Unlock() |
| 164 |
newProber, err := newETCD3ProberMonitor(c) |
| 165 |
// Ensure that server is already not shutting down. |
| 166 |
select { |
| 167 |
case <-stopCh: |
| 168 |
if err == nil { |
| 169 |
newProber.Close() |
| 170 |
} |
| 171 |
return true, nil |
| 172 |
default: |
| 173 |
} |
| 174 |
if err != nil { |
| 175 |
clientErr = err |
| 176 |
return false, nil |
| 177 |
} |
| 178 |
prober = newProber |
| 179 |
clientErr = nil |
| 180 |
return true, nil |
| 181 |
}, stopCh) |
| 182 |
|
| 183 |
// Close the client on shutdown. |
| 184 |
go func() { |
| 185 |
defer utilruntime.HandleCrash() |
| 186 |
<-stopCh |
| 187 |
|
| 188 |
lock.Lock() |
| 189 |
defer lock.Unlock() |
| 190 |
if prober != nil { |
| 191 |
prober.Close() |
| 192 |
clientErr = fmt.Errorf("server is shutting down") |
| 193 |
} |
| 194 |
}() |
| 195 |
|
| 196 |
// limit to a request every half of the configured timeout with a maximum burst of one |
| 197 |
// rate limited requests will receive the last request sent error (note: not the last received response) |
| 198 |
limiter := rate.NewLimiter(rate.Every(timeout/2), 1) |
| 199 |
// initial state is the clientErr |
| 200 |
lastError := &atomicLastError{err: fmt.Errorf("etcd client connection not yet established")} |
| 201 |
|
| 202 |
return func() error { |
| 203 |
// Given that client is closed on shutdown we hold the lock for |
| 204 |
// the entire period of healthcheck call to ensure that client will |
| 205 |
// not be closed during healthcheck. |
| 206 |
// Given that healthchecks has a 2s timeout, worst case of blocking |
| 207 |
// shutdown for additional 2s seems acceptable. |
| 208 |
lock.RLock() |
| 209 |
defer lock.RUnlock() |
| 210 |
|
| 211 |
if clientErr != nil { |
| 212 |
return clientErr |
| 213 |
} |
| 214 |
if limiter.Allow() == false { |
| 215 |
return lastError.Load() |
| 216 |
} |
| 217 |
ctx, cancel := context.WithTimeout(context.Background(), timeout) |
| 218 |
defer cancel() |
| 219 |
now := time.Now() |
| 220 |
err := prober.Probe(ctx) |
| 221 |
lastError.Store(err, now) |
| 222 |
return err |
| 223 |
}, nil |
| 224 |
} |
| 225 |
|
| 226 |
func newETCD3ProberMonitor(c storagebackend.Config) (*etcd3ProberMonitor, error) { |
| 227 |
client, err := newETCD3Client(c.Transport) |
| 228 |
if err != nil { |
| 229 |
return nil, err |
| 230 |
} |
| 231 |
return &etcd3ProberMonitor{ |
| 232 |
client: client.Client, |
| 233 |
prefix: c.Prefix, |
| 234 |
endpoints: c.Transport.ServerList, |
| 235 |
}, nil |
| 236 |
} |
| 237 |
|
| 238 |
type etcd3ProberMonitor struct { |
| 239 |
prefix string |
| 240 |
endpoints []string |
| 241 |
|
| 242 |
mux sync.RWMutex |
| 243 |
client *clientv3.Client |
| 244 |
closed bool |
| 245 |
} |
| 246 |
|
| 247 |
func (t *etcd3ProberMonitor) Close() error { |
| 248 |
t.mux.Lock() |
| 249 |
defer t.mux.Unlock() |
| 250 |
if !t.closed { |
| 251 |
t.closed = true |
| 252 |
return t.client.Close() |
| 253 |
} |
| 254 |
return fmt.Errorf("closed") |
| 255 |
} |
| 256 |
|
| 257 |
func (t *etcd3ProberMonitor) Probe(ctx context.Context) error { |
| 258 |
t.mux.RLock() |
| 259 |
defer t.mux.RUnlock() |
| 260 |
if t.closed { |
| 261 |
return fmt.Errorf("closed") |
| 262 |
} |
| 263 |
// See https://github.com/etcd-io/etcd/blob/c57f8b3af865d1b531b979889c602ba14377420e/etcdctl/ctlv3/command/ep_command.go#L118 |
| 264 |
_, err := t.client.Get(ctx, path.Join("/", t.prefix, "health")) |
| 265 |
if err != nil { |
| 266 |
return fmt.Errorf("error getting data from etcd: %w", err) |
| 267 |
} |
| 268 |
return nil |
| 269 |
} |
| 270 |
|
| 271 |
func (t *etcd3ProberMonitor) Monitor(ctx context.Context) (metrics.StorageMetrics, error) { |
| 272 |
t.mux.RLock() |
| 273 |
defer t.mux.RUnlock() |
| 274 |
if t.closed { |
| 275 |
return metrics.StorageMetrics{}, fmt.Errorf("closed") |
| 276 |
} |
| 277 |
status, err := t.client.Status(ctx, t.endpoints[rand.Int()%len(t.endpoints)]) |
| 278 |
if err != nil { |
| 279 |
return metrics.StorageMetrics{}, err |
| 280 |
} |
| 281 |
return metrics.StorageMetrics{ |
| 282 |
Size: status.DbSize, |
| 283 |
}, nil |
| 284 |
} |
| 285 |
|
| 286 |
var newETCD3Client = func(c storagebackend.TransportConfig) (*kubernetes.Client, error) { |
| 287 |
tlsInfo := transport.TLSInfo{ |
| 288 |
CertFile: c.CertFile, |
| 289 |
KeyFile: c.KeyFile, |
| 290 |
TrustedCAFile: c.TrustedCAFile, |
| 291 |
} |
| 292 |
tlsConfig, err := tlsInfo.ClientConfig() |
| 293 |
if err != nil { |
| 294 |
return nil, err |
| 295 |
} |
| 296 |
// NOTE: Client relies on nil tlsConfig |
| 297 |
// for non-secure connections, update the implicit variable |
| 298 |
if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.TrustedCAFile) == 0 { |
| 299 |
tlsConfig = nil |
| 300 |
} |
| 301 |
networkContext := egressselector.Etcd.AsNetworkContext() |
| 302 |
var egressDialer utilnet.DialFunc |
| 303 |
if c.EgressLookup != nil { |
| 304 |
egressDialer, err = c.EgressLookup(networkContext) |
| 305 |
if err != nil { |
| 306 |
return nil, err |
| 307 |
} |
| 308 |
} |
| 309 |
dialOptions := []grpc.DialOption{ |
| 310 |
grpc.WithBlock(), // block until the underlying connection is up |
| 311 |
// use chained interceptors so that the default (retry and backoff) interceptors are added. |
| 312 |
// otherwise they will be overwritten by the metric interceptor. |
| 313 |
// |
| 314 |
// these optional interceptors will be placed after the default ones. |
| 315 |
// which seems to be what we want as the metrics will be collected on each attempt (retry) |
| 316 |
grpc.WithChainUnaryInterceptor(grpcprom.UnaryClientInterceptor), |
| 317 |
grpc.WithChainStreamInterceptor(grpcprom.StreamClientInterceptor), |
| 318 |
} |
| 319 |
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) { |
| 320 |
tracingOpts := []otelgrpc.Option{ |
| 321 |
otelgrpc.WithMessageEvents(otelgrpc.ReceivedEvents, otelgrpc.SentEvents), |
| 322 |
otelgrpc.WithPropagators(tracing.Propagators()), |
| 323 |
otelgrpc.WithTracerProvider(c.TracerProvider), |
| 324 |
} |
| 325 |
// Even with Noop TracerProvider, the otelgrpc still handles context propagation. |
| 326 |
// See https://github.com/open-telemetry/opentelemetry-go/tree/main/example/passthrough |
| 327 |
dialOptions = append(dialOptions, |
| 328 |
grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor(tracingOpts...)), |
| 329 |
grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor(tracingOpts...))) |
| 330 |
} |
| 331 |
if egressDialer != nil { |
| 332 |
dialer := func(ctx context.Context, addr string) (net.Conn, error) { |
| 333 |
if strings.Contains(addr, "//") { |
| 334 |
// etcd client prior to 3.5 passed URLs to dialer, normalize to address |
| 335 |
u, err := url.Parse(addr) |
| 336 |
if err != nil { |
| 337 |
return nil, err |
| 338 |
} |
| 339 |
addr = u.Host |
| 340 |
} |
| 341 |
return egressDialer(ctx, "tcp", addr) |
| 342 |
} |
| 343 |
dialOptions = append(dialOptions, grpc.WithContextDialer(dialer)) |
| 344 |
} |
| 345 |
|
| 346 |
cfg := clientv3.Config{ |
| 347 |
DialTimeout: dialTimeout, |
| 348 |
DialKeepAliveTime: keepaliveTime, |
| 349 |
DialKeepAliveTimeout: keepaliveTimeout, |
| 350 |
DialOptions: dialOptions, |
| 351 |
Endpoints: c.ServerList, |
| 352 |
TLS: tlsConfig, |
| 353 |
Logger: etcd3ClientLogger, |
| 354 |
} |
| 355 |
|
| 356 |
return kubernetes.New(cfg) |
| 357 |
} |
| 358 |
|
| 359 |
type runningCompactor struct { |
| 360 |
interval time.Duration |
| 361 |
cancel context.CancelFunc |
| 362 |
client *clientv3.Client |
| 363 |
refs int |
| 364 |
} |
| 365 |
|
| 366 |
var ( |
| 367 |
// compactorsMu guards access to compactors map |
| 368 |
compactorsMu sync.Mutex |
| 369 |
compactors = map[string]*runningCompactor{} |
| 370 |
// dbMetricsMonitorsMu guards access to dbMetricsMonitors map |
| 371 |
dbMetricsMonitorsMu sync.Mutex |
| 372 |
dbMetricsMonitors map[string]struct{} |
| 373 |
) |
| 374 |
|
| 375 |
// startCompactorOnce start one compactor per transport. If the interval get smaller on repeated calls, the |
| 376 |
// compactor is replaced. A destroy func is returned. If all destroy funcs with the same transport are called, |
| 377 |
// the compactor is stopped. |
| 378 |
func startCompactorOnce(c storagebackend.TransportConfig, interval time.Duration) (func(), error) { |
| 379 |
compactorsMu.Lock() |
| 380 |
defer compactorsMu.Unlock() |
| 381 |
|
| 382 |
if interval == 0 { |
| 383 |
// short circuit, if the compaction request from apiserver is disabled |
| 384 |
return func() {}, nil |
| 385 |
} |
| 386 |
key := fmt.Sprintf("%v", c) // gives: {[server1 server2] keyFile certFile caFile} |
| 387 |
if compactor, foundBefore := compactors[key]; !foundBefore || compactor.interval > interval { |
| 388 |
client, err := newETCD3Client(c) |
| 389 |
if err != nil { |
| 390 |
return nil, err |
| 391 |
} |
| 392 |
compactorClient := client.Client |
| 393 |
|
| 394 |
if foundBefore { |
| 395 |
// replace compactor |
| 396 |
compactor.cancel() |
| 397 |
compactor.client.Close() |
| 398 |
} else { |
| 399 |
// start new compactor |
| 400 |
compactor = &runningCompactor{} |
| 401 |
compactors[key] = compactor |
| 402 |
} |
| 403 |
|
| 404 |
ctx, cancel := context.WithCancel(context.Background()) |
| 405 |
|
| 406 |
compactor.interval = interval |
| 407 |
compactor.cancel = cancel |
| 408 |
compactor.client = compactorClient |
| 409 |
|
| 410 |
etcd3.StartCompactor(ctx, compactorClient, interval) |
| 411 |
} |
| 412 |
|
| 413 |
compactors[key].refs++ |
| 414 |
|
| 415 |
return func() { |
| 416 |
compactorsMu.Lock() |
| 417 |
defer compactorsMu.Unlock() |
| 418 |
|
| 419 |
compactor := compactors[key] |
| 420 |
compactor.refs-- |
| 421 |
if compactor.refs == 0 { |
| 422 |
compactor.cancel() |
| 423 |
compactor.client.Close() |
| 424 |
delete(compactors, key) |
| 425 |
} |
| 426 |
}, nil |
| 427 |
} |
| 428 |
|
| 429 |
func newETCD3Storage(c storagebackend.ConfigForResource, newFunc, newListFunc func() runtime.Object, resourcePrefix string) (storage.Interface, DestroyFunc, error) { |
| 430 |
stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval) |
| 431 |
if err != nil { |
| 432 |
return nil, nil, err |
| 433 |
} |
| 434 |
|
| 435 |
client, err := newETCD3Client(c.Transport) |
| 436 |
if err != nil { |
| 437 |
stopCompactor() |
| 438 |
return nil, nil, err |
| 439 |
} |
| 440 |
|
| 441 |
// decorate the KV instance so we can track etcd latency per request. |
| 442 |
client.KV = etcd3.NewETCDLatencyTracker(client.KV) |
| 443 |
|
| 444 |
stopDBSizeMonitor, err := startDBSizeMonitorPerEndpoint(client.Client, c.DBMetricPollInterval) |
| 445 |
if err != nil { |
| 446 |
return nil, nil, err |
| 447 |
} |
| 448 |
|
| 449 |
var once sync.Once |
| 450 |
destroyFunc := func() { |
| 451 |
// we know that storage destroy funcs are called multiple times (due to reuse in subresources). |
| 452 |
// Hence, we only destroy once. |
| 453 |
// TODO: fix duplicated storage destroy calls higher level |
| 454 |
once.Do(func() { |
| 455 |
stopCompactor() |
| 456 |
stopDBSizeMonitor() |
| 457 |
client.Close() |
| 458 |
}) |
| 459 |
} |
| 460 |
transformer := c.Transformer |
| 461 |
if transformer == nil { |
| 462 |
transformer = identity.NewEncryptCheckTransformer() |
| 463 |
} |
| 464 |
|
| 465 |
versioner := storage.APIObjectVersioner{} |
| 466 |
decoder := etcd3.NewDefaultDecoder(c.Codec, versioner) |
| 467 |
store := etcd3.New(client, c.Codec, newFunc, newListFunc, c.Prefix, resourcePrefix, c.GroupResource, transformer, c.LeaseManagerConfig, decoder, versioner) |
| 468 |
return store, destroyFunc, nil |
| 469 |
} |
| 470 |
|
| 471 |
// startDBSizeMonitorPerEndpoint starts a loop to monitor etcd database size and update the |
| 472 |
// corresponding metric etcd_db_total_size_in_bytes for each etcd server endpoint. |
| 473 |
// Deprecated: Will be replaced with newETCD3ProberMonitor |
| 474 |
func startDBSizeMonitorPerEndpoint(client *clientv3.Client, interval time.Duration) (func(), error) { |
| 475 |
if interval == 0 { |
| 476 |
return func() {}, nil |
| 477 |
} |
| 478 |
dbMetricsMonitorsMu.Lock() |
| 479 |
defer dbMetricsMonitorsMu.Unlock() |
| 480 |
|
| 481 |
ctx, cancel := context.WithCancel(context.Background()) |
| 482 |
for _, ep := range client.Endpoints() { |
| 483 |
if _, found := dbMetricsMonitors[ep]; found { |
| 484 |
continue |
| 485 |
} |
| 486 |
dbMetricsMonitors[ep] = struct{}{} |
| 487 |
endpoint := ep |
| 488 |
klog.V(4).Infof("Start monitoring storage db size metric for endpoint %s with polling interval %v", endpoint, interval) |
| 489 |
go wait.JitterUntilWithContext(ctx, func(context.Context) { |
| 490 |
epStatus, err := client.Maintenance.Status(ctx, endpoint) |
| 491 |
if err != nil { |
| 492 |
klog.V(4).Infof("Failed to get storage db size for ep %s: %v", endpoint, err) |
| 493 |
metrics.UpdateEtcdDbSize(endpoint, -1) |
| 494 |
} else { |
| 495 |
metrics.UpdateEtcdDbSize(endpoint, epStatus.DbSize) |
| 496 |
} |
| 497 |
}, interval, dbMetricsMonitorJitter, true) |
| 498 |
} |
| 499 |
|
| 500 |
return func() { |
| 501 |
cancel() |
| 502 |
}, nil |
| 503 |
} |