| 1 |
/* |
| 2 |
Copyright 2024 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package feature |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"sync" |
| 23 |
"time" |
| 24 |
|
| 25 |
clientv3 "go.etcd.io/etcd/client/v3" |
| 26 |
"k8s.io/apimachinery/pkg/util/runtime" |
| 27 |
"k8s.io/apimachinery/pkg/util/version" |
| 28 |
"k8s.io/apimachinery/pkg/util/wait" |
| 29 |
"k8s.io/apiserver/pkg/storage" |
| 30 |
"k8s.io/klog/v2" |
| 31 |
"k8s.io/utils/ptr" |
| 32 |
) |
| 33 |
|
| 34 |
var ( |
| 35 |
// Define these static versions to use for checking version of etcd, issue on kubernetes #123192 |
| 36 |
v3_4_31 = version.MustParseSemantic("3.4.31") |
| 37 |
v3_5_0 = version.MustParseSemantic("3.5.0") |
| 38 |
v3_5_13 = version.MustParseSemantic("3.5.13") |
| 39 |
|
| 40 |
// DefaultFeatureSupportChecker is a shared global etcd FeatureSupportChecker. |
| 41 |
DefaultFeatureSupportChecker FeatureSupportChecker = newDefaultFeatureSupportChecker() |
| 42 |
) |
| 43 |
|
| 44 |
// FeatureSupportChecker to define Supports functions. |
| 45 |
type FeatureSupportChecker interface { |
| 46 |
// Supports check if the feature is supported or not by checking internal cache. |
| 47 |
// By default all calls to this function before calling CheckClient returns false. |
| 48 |
// Returns true if all endpoints in etcd clients are supporting the feature. |
| 49 |
// If client A supports and client B doesn't support the feature, the `Supports` will |
| 50 |
// first return true at client A initializtion and then return false on client B |
| 51 |
// initialzation, it can flip the support at runtime. |
| 52 |
Supports(feature storage.Feature) bool |
| 53 |
// CheckClient works with etcd client to recalcualte feature support and cache it internally. |
| 54 |
// All etcd clients should support feature to cause `Supports` return true. |
| 55 |
// If client A supports and client B doesn't support the feature, the `Supports` will |
| 56 |
// first return true at client A initializtion and then return false on client B |
| 57 |
// initialzation, it can flip the support at runtime. |
| 58 |
CheckClient(ctx context.Context, c client, feature storage.Feature) |
| 59 |
} |
| 60 |
|
| 61 |
type defaultFeatureSupportChecker struct { |
| 62 |
lock sync.Mutex |
| 63 |
progressNotifySupported *bool |
| 64 |
checkingEndpoint map[string]struct{} |
| 65 |
} |
| 66 |
|
| 67 |
func newDefaultFeatureSupportChecker() *defaultFeatureSupportChecker { |
| 68 |
return &defaultFeatureSupportChecker{ |
| 69 |
checkingEndpoint: make(map[string]struct{}), |
| 70 |
} |
| 71 |
} |
| 72 |
|
| 73 |
// Supports can check the featue from anywhere without storage if it was cached before. |
| 74 |
func (f *defaultFeatureSupportChecker) Supports(feature storage.Feature) bool { |
| 75 |
switch feature { |
| 76 |
case storage.RequestWatchProgress: |
| 77 |
f.lock.Lock() |
| 78 |
defer f.lock.Unlock() |
| 79 |
|
| 80 |
return ptr.Deref(f.progressNotifySupported, false) |
| 81 |
default: |
| 82 |
runtime.HandleError(fmt.Errorf("feature %q is not implemented in DefaultFeatureSupportChecker", feature)) |
| 83 |
return false |
| 84 |
} |
| 85 |
} |
| 86 |
|
| 87 |
// CheckClient accepts client and calculate the support per endpoint and caches it. |
| 88 |
func (f *defaultFeatureSupportChecker) CheckClient(ctx context.Context, c client, feature storage.Feature) { |
| 89 |
switch feature { |
| 90 |
case storage.RequestWatchProgress: |
| 91 |
f.checkClient(ctx, c) |
| 92 |
default: |
| 93 |
runtime.HandleError(fmt.Errorf("feature %q is not implemented in DefaultFeatureSupportChecker", feature)) |
| 94 |
} |
| 95 |
} |
| 96 |
|
| 97 |
func (f *defaultFeatureSupportChecker) checkClient(ctx context.Context, c client) { |
| 98 |
// start with 10 ms, multiply by 2 each step, until 15 s and stays on 15 seconds. |
| 99 |
delayFunc := wait.Backoff{ |
| 100 |
Duration: 10 * time.Millisecond, |
| 101 |
Cap: 15 * time.Second, |
| 102 |
Factor: 2.0, |
| 103 |
Steps: 11}.DelayFunc() |
| 104 |
f.lock.Lock() |
| 105 |
defer f.lock.Unlock() |
| 106 |
for _, ep := range c.Endpoints() { |
| 107 |
if _, found := f.checkingEndpoint[ep]; found { |
| 108 |
continue |
| 109 |
} |
| 110 |
f.checkingEndpoint[ep] = struct{}{} |
| 111 |
go func(ep string) { |
| 112 |
defer runtime.HandleCrash() |
| 113 |
err := delayFunc.Until(ctx, true, true, func(ctx context.Context) (done bool, err error) { |
| 114 |
internalErr := f.clientSupportsRequestWatchProgress(ctx, c, ep) |
| 115 |
return internalErr == nil, nil |
| 116 |
}) |
| 117 |
if err != nil { |
| 118 |
klog.ErrorS(err, "Failed to check if RequestWatchProgress is supported by etcd after retrying") |
| 119 |
} |
| 120 |
}(ep) |
| 121 |
} |
| 122 |
} |
| 123 |
|
| 124 |
func (f *defaultFeatureSupportChecker) clientSupportsRequestWatchProgress(ctx context.Context, c client, ep string) error { |
| 125 |
supported, err := endpointSupportsRequestWatchProgress(ctx, c, ep) |
| 126 |
if err != nil { |
| 127 |
return err |
| 128 |
} |
| 129 |
f.lock.Lock() |
| 130 |
defer f.lock.Unlock() |
| 131 |
|
| 132 |
if !supported { |
| 133 |
klog.Infof("RequestWatchProgress feature is not supported by %q endpoint", ep) |
| 134 |
f.progressNotifySupported = ptr.To(false) |
| 135 |
return nil |
| 136 |
} |
| 137 |
if f.progressNotifySupported == nil { |
| 138 |
f.progressNotifySupported = ptr.To(true) |
| 139 |
} |
| 140 |
return nil |
| 141 |
} |
| 142 |
|
| 143 |
// Sub interface of etcd client. |
| 144 |
type client interface { |
| 145 |
// Endpoints returns list of endpoints in etcd client. |
| 146 |
Endpoints() []string |
| 147 |
// Status retrieves the status information from the etcd client connected to the specified endpoint. |
| 148 |
// It takes a context.Context parameter for cancellation or timeout control. |
| 149 |
// It returns a clientv3.StatusResponse containing the status information or an error if the operation fails. |
| 150 |
Status(ctx context.Context, endpoint string) (*clientv3.StatusResponse, error) |
| 151 |
} |
| 152 |
|
| 153 |
// endpointSupportsRequestWatchProgress evaluates whether RequestWatchProgress supported by current version of etcd endpoint. |
| 154 |
// Based on this issues: |
| 155 |
// - https://github.com/etcd-io/etcd/issues/15220 - Fixed in etcd v3.4.25+ and v3.5.8+ |
| 156 |
// - https://github.com/etcd-io/etcd/issues/17507 - Fixed in etcd v3.4.31+ and v3.5.13+ |
| 157 |
func endpointSupportsRequestWatchProgress(ctx context.Context, c client, endpoint string) (bool, error) { |
| 158 |
resp, err := c.Status(ctx, endpoint) |
| 159 |
if err != nil { |
| 160 |
return false, fmt.Errorf("failed checking etcd version, endpoint: %q: %w", endpoint, err) |
| 161 |
} |
| 162 |
ver, err := version.ParseSemantic(resp.Version) |
| 163 |
if err != nil { |
| 164 |
// Assume feature is not supported if etcd version cannot be parsed. |
| 165 |
klog.ErrorS(err, "Failed to parse etcd version", "version", resp.Version) |
| 166 |
return false, nil |
| 167 |
} |
| 168 |
if ver.LessThan(v3_4_31) || ver.AtLeast(v3_5_0) && ver.LessThan(v3_5_13) { |
| 169 |
return false, nil |
| 170 |
} |
| 171 |
return true, nil |
| 172 |
} |