Debian Codegraph @

pkg/server/filters/maxinflight_test.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package filters
18
19 import (
20 "context"
21 "fmt"
22 "net/http"
23 "net/http/httptest"
24 "strings"
25 "sync"
26 "testing"
27
28 "k8s.io/apimachinery/pkg/util/sets"
29 "k8s.io/apiserver/pkg/authentication/user"
30 apifilters "k8s.io/apiserver/pkg/endpoints/filters"
31 apirequest "k8s.io/apiserver/pkg/endpoints/request"
32 fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
33 )
34
35 func createMaxInflightServer(t *testing.T, callsWg, blockWg *sync.WaitGroup, disableCallsWg *bool, disableCallsWgMutex *sync.Mutex, nonMutating, mutating int) *httptest.Server {
36 fcmetrics.Register()
37 longRunningRequestCheck := BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString("proxy"))
38
39 requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")}
40 handler := WithMaxInFlightLimit(
41 http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
42 // A short, accounted request that does not wait for block WaitGroup.
43 if strings.Contains(r.URL.Path, "dontwait") {
44 return
45 }
46 disableCallsWgMutex.Lock()
47 waitForCalls := *disableCallsWg
48 disableCallsWgMutex.Unlock()
49 if waitForCalls {
50 callsWg.Done()
51 }
52 t.Logf("About to blockWg.Wait(), requestURI=%v, remoteAddr=%v", r.RequestURI, r.RemoteAddr)
53 blockWg.Wait()
54 t.Logf("Returned from blockWg.Wait(), requestURI=%v, remoteAddr=%v", r.RequestURI, r.RemoteAddr)
55 }),
56 nonMutating,
57 mutating,
58 longRunningRequestCheck,
59 )
60 handler = withFakeUser(handler)
61 handler = apifilters.WithRequestInfo(handler, requestInfoFactory)
62
63 return httptest.NewServer(handler)
64 }
65
66 func withFakeUser(handler http.Handler) http.Handler {
67 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
68 if len(r.Header["Groups"]) > 0 {
69 r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{
70 Groups: r.Header["Groups"],
71 }))
72 }
73 handler.ServeHTTP(w, r)
74 })
75 }
76
77 // Tests that MaxInFlightLimit works, i.e.
78 // - "long" requests such as proxy or watch, identified by regexp are not accounted despite
79 // hanging for the long time,
80 // - "short" requests are correctly accounted, i.e. there can be only size of channel passed to the
81 // constructor in flight at any given moment,
82 // - subsequent "short" requests are rejected instantly with appropriate error,
83 // - subsequent "long" requests are handled normally,
84 // - we correctly recover after some "short" requests finish, i.e. we can process new ones.
85 func TestMaxInFlightNonMutating(t *testing.T) {
86 const AllowedNonMutatingInflightRequestsNo = 3
87
88 // Calls is used to wait until all server calls are received. We are sending
89 // AllowedNonMutatingInflightRequestsNo of 'long' not-accounted requests and the same number of
90 // 'short' accounted ones.
91 calls := &sync.WaitGroup{}
92 calls.Add(AllowedNonMutatingInflightRequestsNo * 2)
93
94 // Responses is used to wait until all responses are
95 // received. This prevents some async requests getting EOF
96 // errors from prematurely closing the server
97 responses := &sync.WaitGroup{}
98 responses.Add(AllowedNonMutatingInflightRequestsNo * 2)
99
100 // Block is used to keep requests in flight for as long as we need to. All requests will
101 // be unblocked at the same time.
102 block := &sync.WaitGroup{}
103 block.Add(1)
104
105 waitForCalls := true
106 waitForCallsMutex := sync.Mutex{}
107
108 server := createMaxInflightServer(t, calls, block, &waitForCalls, &waitForCallsMutex, AllowedNonMutatingInflightRequestsNo, 1)
109 defer server.Close()
110
111 ctx, cancel := context.WithCancel(context.Background())
112 defer cancel()
113 StartMaxInFlightWatermarkMaintenance(ctx.Done())
114
115 // These should hang, but not affect accounting. use a query param match
116 for i := 0; i < AllowedNonMutatingInflightRequestsNo; i++ {
117 // These should hang waiting on block...
118 go func() {
119 if err := expectHTTPGet(server.URL+"/api/v1/namespaces/default/wait?watch=true", http.StatusOK); err != nil {
120 t.Error(err)
121 }
122 responses.Done()
123 }()
124 }
125
126 // Check that sever is not saturated by not-accounted calls
127 if err := expectHTTPGet(server.URL+"/dontwait", http.StatusOK); err != nil {
128 t.Error(err)
129 }
130
131 // These should hang and be accounted, i.e. saturate the server
132 for i := 0; i < AllowedNonMutatingInflightRequestsNo; i++ {
133 // These should hang waiting on block...
134 go func() {
135 if err := expectHTTPGet(server.URL, http.StatusOK); err != nil {
136 t.Error(err)
137 }
138 responses.Done()
139 }()
140 }
141 // We wait for all calls to be received by the server
142 calls.Wait()
143 // Disable calls notifications in the server
144 waitForCallsMutex.Lock()
145 waitForCalls = false
146 waitForCallsMutex.Unlock()
147
148 // Do this multiple times to show that rate limit rejected requests don't block.
149 for i := 0; i < 2; i++ {
150 if err := expectHTTPGet(server.URL, http.StatusTooManyRequests); err != nil {
151 t.Error(err)
152 }
153 }
154 // Validate that non-accounted URLs still work. use a path regex match
155 if err := expectHTTPGet(server.URL+"/api/v1/watch/namespaces/default/dontwait", http.StatusOK); err != nil {
156 t.Error(err)
157 }
158
159 // We should allow a single mutating request.
160 if err := expectHTTPPost(server.URL+"/dontwait", http.StatusOK); err != nil {
161 t.Error(err)
162 }
163
164 // Let all hanging requests finish
165 block.Done()
166
167 // Show that we recover from being blocked up.
168 // Too avoid flakyness we need to wait until at least one of the requests really finishes.
169 responses.Wait()
170 if err := expectHTTPGet(server.URL, http.StatusOK); err != nil {
171 t.Error(err)
172 }
173 }
174
175 func TestMaxInFlightMutating(t *testing.T) {
176 const AllowedMutatingInflightRequestsNo = 3
177
178 calls := &sync.WaitGroup{}
179 calls.Add(AllowedMutatingInflightRequestsNo)
180
181 responses := &sync.WaitGroup{}
182 responses.Add(AllowedMutatingInflightRequestsNo)
183
184 // Block is used to keep requests in flight for as long as we need to. All requests will
185 // be unblocked at the same time.
186 block := &sync.WaitGroup{}
187 block.Add(1)
188
189 waitForCalls := true
190 waitForCallsMutex := sync.Mutex{}
191
192 server := createMaxInflightServer(t, calls, block, &waitForCalls, &waitForCallsMutex, 1, AllowedMutatingInflightRequestsNo)
193 defer server.Close()
194
195 ctx, cancel := context.WithCancel(context.Background())
196 defer cancel()
197 StartMaxInFlightWatermarkMaintenance(ctx.Done())
198
199 // These should hang and be accounted, i.e. saturate the server
200 for i := 0; i < AllowedMutatingInflightRequestsNo; i++ {
201 // These should hang waiting on block...
202 go func() {
203 if err := expectHTTPPost(server.URL+"/foo/bar", http.StatusOK); err != nil {
204 t.Error(err)
205 }
206 responses.Done()
207 }()
208 }
209 // We wait for all calls to be received by the server
210 calls.Wait()
211 // Disable calls notifications in the server
212 // Disable calls notifications in the server
213 waitForCallsMutex.Lock()
214 waitForCalls = false
215 waitForCallsMutex.Unlock()
216
217 // Do this multiple times to show that rate limit rejected requests don't block.
218 for i := 0; i < 2; i++ {
219 if err := expectHTTPPost(server.URL+"/foo/bar/", http.StatusTooManyRequests); err != nil {
220 t.Error(err)
221 }
222 }
223 // Validate that non-mutating URLs still work. use a path regex match
224 if err := expectHTTPGet(server.URL+"/dontwait", http.StatusOK); err != nil {
225 t.Error(err)
226 }
227
228 // Let all hanging requests finish
229 block.Done()
230
231 // Show that we recover from being blocked up.
232 // Too avoid flakyness we need to wait until at least one of the requests really finishes.
233 responses.Wait()
234 if err := expectHTTPPost(server.URL+"/foo/bar", http.StatusOK); err != nil {
235 t.Error(err)
236 }
237 }
238
239 // We use GET as a sample non-mutating request.
240 func expectHTTPGet(url string, code int) error {
241 r, err := http.Get(url)
242 if err != nil {
243 return fmt.Errorf("unexpected error: %v", err)
244 }
245 if r.StatusCode != code {
246 return fmt.Errorf("unexpected response: %v", r.StatusCode)
247 }
248 return nil
249 }
250
251 // We use POST as a sample mutating request.
252 func expectHTTPPost(url string, code int, groups ...string) error {
253 req, err := http.NewRequest(http.MethodPost, url, strings.NewReader("foo bar"))
254 if err != nil {
255 return err
256 }
257 for _, group := range groups {
258 req.Header.Add("Groups", group)
259 }
260
261 r, err := http.DefaultClient.Do(req)
262 if err != nil {
263 return fmt.Errorf("unexpected error: %v", err)
264 }
265 if r.StatusCode != code {
266 return fmt.Errorf("unexpected response: %v", r.StatusCode)
267 }
268 return nil
269 }
270
271 func TestMaxInFlightSkipsMasters(t *testing.T) {
272 const AllowedMutatingInflightRequestsNo = 3
273
274 calls := &sync.WaitGroup{}
275 calls.Add(AllowedMutatingInflightRequestsNo)
276
277 responses := &sync.WaitGroup{}
278 responses.Add(AllowedMutatingInflightRequestsNo)
279
280 // Block is used to keep requests in flight for as long as we need to. All requests will
281 // be unblocked at the same time.
282 block := &sync.WaitGroup{}
283 block.Add(1)
284
285 waitForCalls := true
286 waitForCallsMutex := sync.Mutex{}
287
288 server := createMaxInflightServer(t, calls, block, &waitForCalls, &waitForCallsMutex, 1, AllowedMutatingInflightRequestsNo)
289 defer server.Close()
290
291 ctx, cancel := context.WithCancel(context.Background())
292 defer cancel()
293 StartMaxInFlightWatermarkMaintenance(ctx.Done())
294
295 // These should hang and be accounted, i.e. saturate the server
296 for i := 0; i < AllowedMutatingInflightRequestsNo; i++ {
297 // These should hang waiting on block...
298 go func() {
299 if err := expectHTTPPost(server.URL+"/foo/bar", http.StatusOK); err != nil {
300 t.Error(err)
301 }
302 responses.Done()
303 }()
304 }
305 // We wait for all calls to be received by the server
306 calls.Wait()
307 // Disable calls notifications in the server
308 // Disable calls notifications in the server
309 waitForCallsMutex.Lock()
310 waitForCalls = false
311 waitForCallsMutex.Unlock()
312
313 // Do this multiple times to show that rate limit rejected requests don't block.
314 for i := 0; i < 2; i++ {
315 if err := expectHTTPPost(server.URL+"/dontwait", http.StatusOK, user.SystemPrivilegedGroup); err != nil {
316 t.Error(err)
317 }
318 }
319
320 // Let all hanging requests finish
321 block.Done()
322
323 responses.Wait()
324 }