| 1 |
/* |
| 2 |
Copyright 2024 The Kubernetes Authors. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
package testing |
| 18 |
|
| 19 |
import ( |
| 20 |
"context" |
| 21 |
"fmt" |
| 22 |
"sync" |
| 23 |
"sync/atomic" |
| 24 |
"testing" |
| 25 |
|
| 26 |
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 27 |
"k8s.io/apimachinery/pkg/fields" |
| 28 |
"k8s.io/apimachinery/pkg/labels" |
| 29 |
"k8s.io/apimachinery/pkg/runtime" |
| 30 |
"k8s.io/apimachinery/pkg/util/rand" |
| 31 |
"k8s.io/apiserver/pkg/apis/example" |
| 32 |
"k8s.io/apiserver/pkg/endpoints/request" |
| 33 |
"k8s.io/apiserver/pkg/storage" |
| 34 |
) |
| 35 |
|
| 36 |
type scope string |
| 37 |
|
| 38 |
var ( |
| 39 |
cluster scope = "Cluster" |
| 40 |
node scope = "Node" |
| 41 |
namespace scope = "Namespace" |
| 42 |
) |
| 43 |
|
| 44 |
func RunBenchmarkStoreListCreate(ctx context.Context, b *testing.B, store storage.Interface, match metav1.ResourceVersionMatch) { |
| 45 |
objectCount := atomic.Uint64{} |
| 46 |
pods := make([]*example.Pod, 0, b.N) |
| 47 |
for i := 0; i < b.N; i++ { |
| 48 |
name := rand.String(100) |
| 49 |
pods = append(pods, &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: name}}) |
| 50 |
} |
| 51 |
b.ResetTimer() |
| 52 |
for i := 0; i < b.N; i++ { |
| 53 |
pod := pods[i] |
| 54 |
podOut := &example.Pod{} |
| 55 |
err := store.Create(ctx, computePodKey(pod), pod, podOut, 0) |
| 56 |
if err != nil { |
| 57 |
panic(fmt.Sprintf("Unexpected error %s", err)) |
| 58 |
} |
| 59 |
listOut := &example.PodList{} |
| 60 |
err = store.GetList(ctx, "/pods", storage.ListOptions{ |
| 61 |
Recursive: true, |
| 62 |
ResourceVersion: podOut.ResourceVersion, |
| 63 |
ResourceVersionMatch: match, |
| 64 |
Predicate: storage.SelectionPredicate{ |
| 65 |
Label: labels.Everything(), |
| 66 |
Field: fields.Everything(), |
| 67 |
Limit: 1, |
| 68 |
}, |
| 69 |
}, listOut) |
| 70 |
if err != nil { |
| 71 |
panic(fmt.Sprintf("Unexpected error %s", err)) |
| 72 |
} |
| 73 |
if len(listOut.Items) != 1 { |
| 74 |
b.Errorf("Expected to get 1 element, got %d", len(listOut.Items)) |
| 75 |
} |
| 76 |
objectCount.Add(uint64(len(listOut.Items))) |
| 77 |
} |
| 78 |
b.ReportMetric(float64(objectCount.Load())/float64(b.N), "objects/op") |
| 79 |
} |
| 80 |
|
| 81 |
func RunBenchmarkStoreList(ctx context.Context, b *testing.B, store storage.Interface, data BenchmarkData, useIndex bool) { |
| 82 |
for _, rvm := range []metav1.ResourceVersionMatch{"", metav1.ResourceVersionMatchExact, metav1.ResourceVersionMatchNotOlderThan} { |
| 83 |
b.Run(fmt.Sprintf("RV=%s", rvm), func(b *testing.B) { |
| 84 |
for _, scope := range []scope{cluster, node, namespace} { |
| 85 |
b.Run(fmt.Sprintf("Scope=%s", scope), func(b *testing.B) { |
| 86 |
var expectedElements int |
| 87 |
switch scope { |
| 88 |
case namespace: |
| 89 |
expectedElements = len(data.Pods) / len(data.NamespaceNames) |
| 90 |
case node: |
| 91 |
expectedElements = len(data.Pods) / len(data.NodeNames) |
| 92 |
case cluster: |
| 93 |
expectedElements = len(data.Pods) |
| 94 |
} |
| 95 |
limitOptions := []int64{0} |
| 96 |
switch { |
| 97 |
case expectedElements > 1000: |
| 98 |
limitOptions = append(limitOptions, 1000) |
| 99 |
case expectedElements > 100: |
| 100 |
limitOptions = append(limitOptions, 100) |
| 101 |
} |
| 102 |
for _, limit := range limitOptions { |
| 103 |
b.Run(fmt.Sprintf("Paginate=%v", limit), func(b *testing.B) { |
| 104 |
runBenchmarkStoreList(ctx, b, store, limit, rvm, scope, data, useIndex) |
| 105 |
}) |
| 106 |
} |
| 107 |
}) |
| 108 |
} |
| 109 |
}) |
| 110 |
} |
| 111 |
} |
| 112 |
|
| 113 |
func runBenchmarkStoreList(ctx context.Context, b *testing.B, store storage.Interface, limit int64, match metav1.ResourceVersionMatch, scope scope, data BenchmarkData, useIndex bool) { |
| 114 |
wg := sync.WaitGroup{} |
| 115 |
objectCount := atomic.Uint64{} |
| 116 |
pageCount := atomic.Uint64{} |
| 117 |
for i := 0; i < b.N; i++ { |
| 118 |
wg.Add(1) |
| 119 |
resourceVersion := "" |
| 120 |
switch match { |
| 121 |
case metav1.ResourceVersionMatchExact, metav1.ResourceVersionMatchNotOlderThan: |
| 122 |
maxRevision := 1 + len(data.Pods) |
| 123 |
resourceVersion = fmt.Sprintf("%d", maxRevision-99+i%100) |
| 124 |
} |
| 125 |
go func(resourceVersion, nodeName, namespaceName string) { |
| 126 |
defer wg.Done() |
| 127 |
opts := storage.ListOptions{ |
| 128 |
Recursive: true, |
| 129 |
ResourceVersion: resourceVersion, |
| 130 |
ResourceVersionMatch: match, |
| 131 |
Predicate: storage.SelectionPredicate{ |
| 132 |
GetAttrs: podAttr, |
| 133 |
Label: labels.Everything(), |
| 134 |
Field: fields.Everything(), |
| 135 |
Limit: limit, |
| 136 |
}, |
| 137 |
} |
| 138 |
switch scope { |
| 139 |
case cluster: |
| 140 |
objects, pages := paginateList(ctx, store, "/pods/", opts) |
| 141 |
objectCount.Add(uint64(objects)) |
| 142 |
pageCount.Add(uint64(pages)) |
| 143 |
case node: |
| 144 |
if useIndex { |
| 145 |
opts.Predicate.GetAttrs = podAttr |
| 146 |
opts.Predicate.IndexFields = []string{"spec.nodeName"} |
| 147 |
opts.Predicate.Field = fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}) |
| 148 |
} |
| 149 |
objects, pages := paginateList(ctx, store, "/pods/", opts) |
| 150 |
objectCount.Add(uint64(objects)) |
| 151 |
pageCount.Add(uint64(pages)) |
| 152 |
case namespace: |
| 153 |
ctx := ctx |
| 154 |
if useIndex { |
| 155 |
opts.Predicate.IndexFields = []string{"metadata.namespace"} |
| 156 |
ctx = request.WithRequestInfo(ctx, &request.RequestInfo{Namespace: namespaceName}) |
| 157 |
} |
| 158 |
objects, pages := paginateList(ctx, store, "/pods/"+namespaceName, opts) |
| 159 |
objectCount.Add(uint64(objects)) |
| 160 |
pageCount.Add(uint64(pages)) |
| 161 |
} |
| 162 |
}(resourceVersion, data.NodeNames[i%len(data.NodeNames)], data.NamespaceNames[i%len(data.NamespaceNames)]) |
| 163 |
} |
| 164 |
wg.Wait() |
| 165 |
b.ReportMetric(float64(objectCount.Load())/float64(b.N), "objects/op") |
| 166 |
b.ReportMetric(float64(pageCount.Load())/float64(b.N), "pages/op") |
| 167 |
} |
| 168 |
|
| 169 |
func paginateList(ctx context.Context, store storage.Interface, key string, opts storage.ListOptions) (objectCount int, pageCount int) { |
| 170 |
listOut := &example.PodList{} |
| 171 |
err := store.GetList(ctx, key, opts, listOut) |
| 172 |
if err != nil { |
| 173 |
panic(fmt.Sprintf("Unexpected error %s", err)) |
| 174 |
} |
| 175 |
opts.Predicate.Continue = listOut.Continue |
| 176 |
opts.ResourceVersion = "" |
| 177 |
opts.ResourceVersionMatch = "" |
| 178 |
pageCount += 1 |
| 179 |
objectCount += len(listOut.Items) |
| 180 |
for opts.Predicate.Continue != "" { |
| 181 |
listOut := &example.PodList{} |
| 182 |
err := store.GetList(ctx, key, opts, listOut) |
| 183 |
if err != nil { |
| 184 |
panic(fmt.Sprintf("Unexpected error %s", err)) |
| 185 |
} |
| 186 |
opts.Predicate.Continue = listOut.Continue |
| 187 |
pageCount += 1 |
| 188 |
objectCount += len(listOut.Items) |
| 189 |
} |
| 190 |
return objectCount, pageCount |
| 191 |
} |
| 192 |
|
| 193 |
func podAttr(obj runtime.Object) (labels.Set, fields.Set, error) { |
| 194 |
pod := obj.(*example.Pod) |
| 195 |
return nil, fields.Set{ |
| 196 |
"spec.nodeName": pod.Spec.NodeName, |
| 197 |
"metadata.namespace": pod.Namespace, |
| 198 |
}, nil |
| 199 |
} |
| 200 |
|
| 201 |
func PrepareBenchchmarkData(namespaceCount, podPerNamespaceCount, nodeCount int) (data BenchmarkData) { |
| 202 |
data.NodeNames = make([]string, nodeCount) |
| 203 |
for i := 0; i < nodeCount; i++ { |
| 204 |
data.NodeNames[i] = rand.String(10) |
| 205 |
} |
| 206 |
data.NamespaceNames = make([]string, namespaceCount) |
| 207 |
for i := 0; i < namespaceCount; i++ { |
| 208 |
namespace := rand.String(10) |
| 209 |
data.NamespaceNames[i] = namespace |
| 210 |
for j := 0; j < podPerNamespaceCount; j++ { |
| 211 |
name := rand.String(10) |
| 212 |
data.Pods = append(data.Pods, &example.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, Spec: example.PodSpec{NodeName: data.NodeNames[rand.Intn(nodeCount)]}}) |
| 213 |
} |
| 214 |
} |
| 215 |
return data |
| 216 |
} |
| 217 |
|
| 218 |
type BenchmarkData struct { |
| 219 |
Pods []*example.Pod |
| 220 |
NamespaceNames []string |
| 221 |
NodeNames []string |
| 222 |
} |