Debian Codegraph @

pkg/util/flowcontrol/borrowing_test.go

raw
1 /*
2 Copyright 2022 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package flowcontrol
18
19 import (
20 "context"
21 "fmt"
22 "sync"
23 "testing"
24 "time"
25
26 flowcontrol "k8s.io/api/flowcontrol/v1"
27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28 "k8s.io/apimachinery/pkg/runtime"
29 "k8s.io/apimachinery/pkg/util/wait"
30 "k8s.io/apiserver/pkg/authentication/user"
31 "k8s.io/apiserver/pkg/endpoints/request"
32 fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
33 "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/eventclock"
34 fqs "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset"
35 "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
36 fcrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
37 "k8s.io/client-go/informers"
38 clientsetfake "k8s.io/client-go/kubernetes/fake"
39 "k8s.io/utils/ptr"
40 )
41
42 type borrowingTestConstraints struct {
43 lendable, borrowing int32
44 }
45
46 // TestBorrowing tests borrowing of concurrency between priority levels.
47 // It runs two scenarios, one where the borrowing hits the limit on
48 // lendable concurrency and one where the borrowing hits the limit on
49 // borrowing of concurrency.
50 // Both scenarios are the same except for the limits.
51 // The test defines two priority levels, identified as "flows" 0 and 1.
52 // Both priority levels have a nominal concurrency limit of 12.
53 // The test maintains 24 concurrent clients for priority level 0
54 // and 6 for level 1,
55 // using an exec func that simply sleeps for 250 ms, for
56 // 25 seconds. The first 10 seconds of behavior are ignored, allowing
57 // the borrowing to start at any point during that time. The test
58 // continues for another 15 seconds, and checks that the delivered
59 // concurrency is about 16 for flow 0 and 6 for flow 1.
60 func TestBorrowing(t *testing.T) {
61 clientsPerFlow := [2]int{24, 6}
62 metrics.Register()
63 for _, testCase := range []struct {
64 name string
65 constraints []borrowingTestConstraints
66 }{
67 {
68 name: "lendable-limited",
69 constraints: []borrowingTestConstraints{
70 {lendable: 50, borrowing: 67},
71 {lendable: 33, borrowing: 50},
72 }},
73 {
74 name: "borrowing-limited",
75 constraints: []borrowingTestConstraints{
76 {lendable: 50, borrowing: 33},
77 {lendable: 67, borrowing: 50},
78 }},
79 } {
80 t.Run(testCase.name, func(t *testing.T) {
81 fsObjs := make([]*flowcontrol.FlowSchema, 2)
82 plcObjs := make([]*flowcontrol.PriorityLevelConfiguration, 2)
83 usernames := make([]string, 2)
84 cfgObjs := []runtime.Object{}
85 for flow := 0; flow < 2; flow++ {
86 usernames[flow] = fmt.Sprintf("test-user%d", flow)
87 plName := fmt.Sprintf("test-pl%d", flow)
88 fsObjs[flow] = &flowcontrol.FlowSchema{
89 ObjectMeta: metav1.ObjectMeta{
90 Name: fmt.Sprintf("test-fs%d", flow),
91 },
92 Spec: flowcontrol.FlowSchemaSpec{
93 MatchingPrecedence: 100,
94 PriorityLevelConfiguration: flowcontrol.PriorityLevelConfigurationReference{
95 Name: plName,
96 },
97 DistinguisherMethod: &flowcontrol.FlowDistinguisherMethod{
98 Type: flowcontrol.FlowDistinguisherMethodByUserType,
99 },
100 Rules: []flowcontrol.PolicyRulesWithSubjects{{
101 Subjects: []flowcontrol.Subject{{
102 Kind: flowcontrol.SubjectKindUser,
103 User: &flowcontrol.UserSubject{Name: usernames[flow]},
104 }},
105 NonResourceRules: []flowcontrol.NonResourcePolicyRule{{
106 Verbs: []string{"*"},
107 NonResourceURLs: []string{"*"},
108 }},
109 }},
110 },
111 }
112 plcObjs[flow] = &flowcontrol.PriorityLevelConfiguration{
113 ObjectMeta: metav1.ObjectMeta{
114 Name: plName,
115 },
116 Spec: flowcontrol.PriorityLevelConfigurationSpec{
117 Type: flowcontrol.PriorityLevelEnablementLimited,
118 Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
119 NominalConcurrencyShares: ptr.To(int32(100)),
120 LendablePercent: &testCase.constraints[flow].lendable,
121 BorrowingLimitPercent: &testCase.constraints[flow].borrowing,
122 LimitResponse: flowcontrol.LimitResponse{
123 Type: flowcontrol.LimitResponseTypeQueue,
124 Queuing: &flowcontrol.QueuingConfiguration{
125 Queues: 10,
126 HandSize: 2,
127 QueueLengthLimit: 10,
128 },
129 },
130 },
131 },
132 }
133 cfgObjs = append(cfgObjs, fsObjs[flow], plcObjs[flow])
134 }
135 clientset := clientsetfake.NewSimpleClientset(cfgObjs...)
136 informerFactory := informers.NewSharedInformerFactory(clientset, time.Second)
137 flowcontrolClient := clientset.FlowcontrolV1()
138 clk := eventclock.Real{}
139 controller := newTestableController(TestableConfig{
140 Name: "Controller",
141 Clock: clk,
142 AsFieldManager: ConfigConsumerAsFieldManager,
143 FoundToDangling: func(found bool) bool { return !found },
144 InformerFactory: informerFactory,
145 FlowcontrolClient: flowcontrolClient,
146 ServerConcurrencyLimit: 24,
147 ReqsGaugeVec: metrics.PriorityLevelConcurrencyGaugeVec,
148 ExecSeatsGaugeVec: metrics.PriorityLevelExecutionSeatsGaugeVec,
149 QueueSetFactory: fqs.NewQueueSetFactory(clk),
150 })
151
152 ctx, cancel := context.WithTimeout(context.Background(), 50*time.Second)
153 stopCh := ctx.Done()
154 controllerCompletionCh := make(chan error)
155
156 informerFactory.Start(stopCh)
157
158 status := informerFactory.WaitForCacheSync(ctx.Done())
159 if names := unsynced(status); len(names) > 0 {
160 t.Fatalf("WaitForCacheSync did not successfully complete, resources=%#v", names)
161 }
162
163 go func() {
164 controllerCompletionCh <- controller.Run(stopCh)
165 }()
166
167 // ensure that the controller has run its first loop.
168 err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {
169 return controller.hasPriorityLevelState(plcObjs[0].Name), nil
170 })
171 if err != nil {
172 t.Errorf("expected the controller to reconcile the priority level configuration object: %s, error: %s", plcObjs[0].Name, err)
173 }
174
175 concIntegrators := make([]fq.Integrator, 2)
176 reqInfo := &request.RequestInfo{
177 IsResourceRequest: false,
178 Path: "/foobar",
179 Verb: "GET",
180 }
181 noteFn := func(fs *flowcontrol.FlowSchema, plc *flowcontrol.PriorityLevelConfiguration, fd string) {}
182 workEstr := func() fcrequest.WorkEstimate { return fcrequest.WorkEstimate{InitialSeats: 1} }
183 qnf := fq.QueueNoteFn(func(bool) {})
184 var startWG sync.WaitGroup
185 startWG.Add(clientsPerFlow[0] + clientsPerFlow[1])
186 // Launch 20 client threads for each flow
187 for flow := 0; flow < 2; flow++ {
188 username := usernames[flow]
189 flowUser := testUser{name: username}
190 rd := RequestDigest{
191 RequestInfo: reqInfo,
192 User: flowUser,
193 }
194 concIntegrator := fq.NewNamedIntegrator(clk, username)
195 concIntegrators[flow] = concIntegrator
196 exec := func() {
197 concIntegrator.Inc()
198 clk.Sleep(250 * time.Millisecond)
199 concIntegrator.Dec()
200 }
201 nThreads := clientsPerFlow[flow]
202 for thread := 0; thread < nThreads; thread++ {
203 go func() {
204 startWG.Done()
205 wait.Until(func() { controller.Handle(ctx, rd, noteFn, workEstr, qnf, exec) }, 0, ctx.Done())
206 }()
207 }
208 }
209 startWG.Wait()
210 // Make sure the controller has had time to sense the load and adjust
211 clk.Sleep(10 * time.Second)
212 // Start the stats that matter from now
213 for _, ci := range concIntegrators {
214 ci.Reset()
215 }
216 // Run for 15 seconds
217 clk.Sleep(15 * time.Second)
218 // Collect the delivered concurrency stats
219 results0 := concIntegrators[0].Reset()
220 results1 := concIntegrators[1].Reset()
221 // shut down all the async stuff
222 cancel()
223
224 // Do the checking
225
226 t.Log("waiting for the controller Run function to shutdown gracefully")
227 controllerErr := <-controllerCompletionCh
228 close(controllerCompletionCh)
229 if controllerErr != nil {
230 t.Errorf("expected nil error from controller Run function, but got: %#v", controllerErr)
231 }
232 if results0.Average < 15.5 || results0.Average > 16.1 {
233 t.Errorf("Flow 0 got average concurrency of %v but expected about 16", results0.Average)
234 } else {
235 t.Logf("Flow 0 got average concurrency of %v and expected about 16", results0.Average)
236 }
237 if results1.Average < 5.5 || results1.Average > 6.1 {
238 t.Errorf("Flow 1 got average concurrency of %v but expected about 6", results1.Average)
239 } else {
240 t.Logf("Flow 1 got average concurrency of %v and expected about 6", results1.Average)
241 }
242 })
243 }
244 }
245
246 type testUser struct{ name string }
247
248 func (tu testUser) GetName() string { return tu.name }
249 func (tu testUser) GetUID() string { return tu.name }
250 func (tu testUser) GetGroups() []string { return []string{user.AllAuthenticated} }
251 func (tu testUser) GetExtra() map[string][]string { return map[string][]string{} }