Debian Codegraph @

pkg/util/proxy/streamtranslator_test.go

raw
1 /*
2 Copyright 2023 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package proxy
18
19 import (
20 "bytes"
21 "context"
22 "crypto/rand"
23 "encoding/json"
24 "errors"
25 "fmt"
26 "io"
27 "math"
28 mrand "math/rand"
29 "net/http"
30 "net/http/httptest"
31 "net/url"
32 "reflect"
33 "strings"
34 "testing"
35 "time"
36
37 v1 "k8s.io/api/core/v1"
38 apierrors "k8s.io/apimachinery/pkg/api/errors"
39 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
40 "k8s.io/apimachinery/pkg/util/httpstream"
41 "k8s.io/apimachinery/pkg/util/httpstream/spdy"
42 rcconstants "k8s.io/apimachinery/pkg/util/remotecommand"
43 "k8s.io/apimachinery/pkg/util/wait"
44 "k8s.io/apiserver/pkg/util/proxy/metrics"
45 "k8s.io/client-go/rest"
46 "k8s.io/client-go/tools/remotecommand"
47 "k8s.io/client-go/transport"
48 "k8s.io/component-base/metrics/legacyregistry"
49 "k8s.io/component-base/metrics/testutil"
50 )
51
52 // TestStreamTranslator_LoopbackStdinToStdout returns random data sent on the client's
53 // STDIN channel back onto the client's STDOUT channel. There are two servers in this test: the
54 // upstream fake SPDY server, and the StreamTranslator server. The StreamTranslator proxys the
55 // data received from the websocket client upstream to the SPDY server (by translating the
56 // websocket data into spdy). The returned data read on the websocket client STDOUT is then
57 // compared the random data sent on STDIN to ensure they are the same.
58 func TestStreamTranslator_LoopbackStdinToStdout(t *testing.T) {
59 metrics.Register()
60 metrics.ResetForTest()
61 t.Cleanup(metrics.ResetForTest)
62 // Create upstream fake SPDY server which copies STDIN back onto STDOUT stream.
63 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
64 ctx, err := createSPDYServerStreams(w, req, Options{
65 Stdin: true,
66 Stdout: true,
67 })
68 if err != nil {
69 t.Errorf("error on createHTTPStreams: %v", err)
70 return
71 }
72 defer ctx.conn.Close()
73 // Loopback STDIN data onto STDOUT stream.
74 _, err = io.Copy(ctx.stdoutStream, ctx.stdinStream)
75 if err != nil {
76 t.Fatalf("error copying STDIN to STDOUT: %v", err)
77 }
78
79 }))
80 defer spdyServer.Close()
81 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with
82 // streams STDIN and STDOUT. Create test server from StreamTranslatorHandler.
83 spdyLocation, err := url.Parse(spdyServer.URL)
84 if err != nil {
85 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
86 }
87 spdyTransport, err := fakeTransport()
88 if err != nil {
89 t.Fatalf("Unexpected error creating transport: %v", err)
90 }
91 streams := Options{Stdin: true, Stdout: true}
92 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
93 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
94 streamTranslator.ServeHTTP(w, req)
95 }))
96 defer streamTranslatorServer.Close()
97 // Now create the websocket client (executor), and point it to the "streamTranslatorServer".
98 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
99 if err != nil {
100 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
101 }
102 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
103 if err != nil {
104 t.Errorf("unexpected error creating websocket executor: %v", err)
105 }
106 // Generate random data, and set it up to stream on STDIN. The data will be
107 // returned on the STDOUT buffer.
108 randomSize := 1024 * 1024
109 randomData := make([]byte, randomSize)
110 if _, err := rand.Read(randomData); err != nil {
111 t.Errorf("unexpected error reading random data: %v", err)
112 }
113 var stdout bytes.Buffer
114 options := &remotecommand.StreamOptions{
115 Stdin: bytes.NewReader(randomData),
116 Stdout: &stdout,
117 }
118 errorChan := make(chan error)
119 go func() {
120 // Start the streaming on the WebSocket "exec" client.
121 errorChan <- exec.StreamWithContext(context.Background(), *options)
122 }()
123
124 select {
125 case <-time.After(wait.ForeverTestTimeout):
126 t.Fatalf("expect stream to be closed after connection is closed.")
127 case err := <-errorChan:
128 if err != nil {
129 t.Errorf("unexpected error: %v", err)
130 }
131 }
132 data, err := io.ReadAll(bytes.NewReader(stdout.Bytes()))
133 if err != nil {
134 t.Errorf("error reading the stream: %v", err)
135 return
136 }
137 // Check the random data sent on STDIN was the same returned on STDOUT.
138 if !bytes.Equal(randomData, data) {
139 t.Errorf("unexpected data received: %d sent: %d", len(data), len(randomData))
140 }
141 // Validate the streamtranslator metrics; should be one 200 success.
142 metricNames := []string{"apiserver_stream_translator_requests_total"}
143 expected := `
144 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
145 # TYPE apiserver_stream_translator_requests_total counter
146 apiserver_stream_translator_requests_total{code="200"} 1
147 `
148 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
149 t.Fatal(err)
150 }
151 }
152
153 // TestStreamTranslator_LoopbackStdinToStderr returns random data sent on the client's
154 // STDIN channel back onto the client's STDERR channel. There are two servers in this test: the
155 // upstream fake SPDY server, and the StreamTranslator server. The StreamTranslator proxys the
156 // data received from the websocket client upstream to the SPDY server (by translating the
157 // websocket data into spdy). The returned data read on the websocket client STDERR is then
158 // compared the random data sent on STDIN to ensure they are the same.
159 func TestStreamTranslator_LoopbackStdinToStderr(t *testing.T) {
160 metrics.Register()
161 metrics.ResetForTest()
162 t.Cleanup(metrics.ResetForTest)
163 // Create upstream fake SPDY server which copies STDIN back onto STDERR stream.
164 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
165 ctx, err := createSPDYServerStreams(w, req, Options{
166 Stdin: true,
167 Stderr: true,
168 })
169 if err != nil {
170 t.Errorf("error on createHTTPStreams: %v", err)
171 return
172 }
173 defer ctx.conn.Close()
174 // Loopback STDIN data onto STDERR stream.
175 _, err = io.Copy(ctx.stderrStream, ctx.stdinStream)
176 if err != nil {
177 t.Fatalf("error copying STDIN to STDERR: %v", err)
178 }
179 }))
180 defer spdyServer.Close()
181 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with
182 // streams STDIN and STDERR. Create test server from StreamTranslatorHandler.
183 spdyLocation, err := url.Parse(spdyServer.URL)
184 if err != nil {
185 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
186 }
187 spdyTransport, err := fakeTransport()
188 if err != nil {
189 t.Fatalf("Unexpected error creating transport: %v", err)
190 }
191 streams := Options{Stdin: true, Stderr: true}
192 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
193 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
194 streamTranslator.ServeHTTP(w, req)
195 }))
196 defer streamTranslatorServer.Close()
197 // Now create the websocket client (executor), and point it to the "streamTranslatorServer".
198 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
199 if err != nil {
200 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
201 }
202 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
203 if err != nil {
204 t.Errorf("unexpected error creating websocket executor: %v", err)
205 }
206 // Generate random data, and set it up to stream on STDIN. The data will be
207 // returned on the STDERR buffer.
208 randomSize := 1024 * 1024
209 randomData := make([]byte, randomSize)
210 if _, err := rand.Read(randomData); err != nil {
211 t.Errorf("unexpected error reading random data: %v", err)
212 }
213 var stderr bytes.Buffer
214 options := &remotecommand.StreamOptions{
215 Stdin: bytes.NewReader(randomData),
216 Stderr: &stderr,
217 }
218 errorChan := make(chan error)
219 go func() {
220 // Start the streaming on the WebSocket "exec" client.
221 errorChan <- exec.StreamWithContext(context.Background(), *options)
222 }()
223
224 select {
225 case <-time.After(wait.ForeverTestTimeout):
226 t.Fatalf("expect stream to be closed after connection is closed.")
227 case err := <-errorChan:
228 if err != nil {
229 t.Errorf("unexpected error: %v", err)
230 }
231 }
232 data, err := io.ReadAll(bytes.NewReader(stderr.Bytes()))
233 if err != nil {
234 t.Errorf("error reading the stream: %v", err)
235 return
236 }
237 // Check the random data sent on STDIN was the same returned on STDERR.
238 if !bytes.Equal(randomData, data) {
239 t.Errorf("unexpected data received: %d sent: %d", len(data), len(randomData))
240 }
241 // Validate the streamtranslator metrics; should be one 200 success.
242 metricNames := []string{"apiserver_stream_translator_requests_total"}
243 expected := `
244 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
245 # TYPE apiserver_stream_translator_requests_total counter
246 apiserver_stream_translator_requests_total{code="200"} 1
247 `
248 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
249 t.Fatal(err)
250 }
251 }
252
253 // Returns a random exit code in the range(1-127).
254 func randomExitCode() int {
255 errorCode := mrand.Intn(127) // Range: (0 - 126)
256 errorCode += 1 // Range: (1 - 127)
257 return errorCode
258 }
259
260 // TestStreamTranslator_ErrorStream tests the error stream by sending an error with a random
261 // exit code, then validating the error arrives on the error stream.
262 func TestStreamTranslator_ErrorStream(t *testing.T) {
263 metrics.Register()
264 metrics.ResetForTest()
265 t.Cleanup(metrics.ResetForTest)
266 expectedExitCode := randomExitCode()
267 // Create upstream fake SPDY server, returning a non-zero exit code
268 // on error stream within the structured error.
269 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
270 ctx, err := createSPDYServerStreams(w, req, Options{
271 Stdout: true,
272 })
273 if err != nil {
274 t.Errorf("error on createHTTPStreams: %v", err)
275 return
276 }
277 defer ctx.conn.Close()
278 // Read/discard STDIN data before returning error on error stream.
279 _, err = io.Copy(io.Discard, ctx.stdinStream)
280 if err != nil {
281 t.Fatalf("error copying STDIN to DISCARD: %v", err)
282 }
283 // Force an non-zero exit code error returned on the error stream.
284 err = ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
285 Status: metav1.StatusFailure,
286 Reason: rcconstants.NonZeroExitCodeReason,
287 Details: &metav1.StatusDetails{
288 Causes: []metav1.StatusCause{
289 {
290 Type: rcconstants.ExitCodeCauseType,
291 Message: fmt.Sprintf("%d", expectedExitCode),
292 },
293 },
294 },
295 }})
296 if err != nil {
297 t.Fatalf("error writing status: %v", err)
298 }
299 }))
300 defer spdyServer.Close()
301 // Create StreamTranslatorHandler, which points upstream to fake SPDY server, and
302 // create a test server using the StreamTranslatorHandler.
303 spdyLocation, err := url.Parse(spdyServer.URL)
304 if err != nil {
305 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
306 }
307 spdyTransport, err := fakeTransport()
308 if err != nil {
309 t.Fatalf("Unexpected error creating transport: %v", err)
310 }
311 streams := Options{Stdin: true}
312 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
313 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
314 streamTranslator.ServeHTTP(w, req)
315 }))
316 defer streamTranslatorServer.Close()
317 // Now create the websocket client (executor), and point it to the "streamTranslatorServer".
318 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
319 if err != nil {
320 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
321 }
322 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
323 if err != nil {
324 t.Errorf("unexpected error creating websocket executor: %v", err)
325 }
326 // Generate random data, and set it up to stream on STDIN. The data will be discarded at
327 // upstream SDPY server.
328 randomSize := 1024 * 1024
329 randomData := make([]byte, randomSize)
330 if _, err := rand.Read(randomData); err != nil {
331 t.Errorf("unexpected error reading random data: %v", err)
332 }
333 options := &remotecommand.StreamOptions{
334 Stdin: bytes.NewReader(randomData),
335 }
336 errorChan := make(chan error)
337 go func() {
338 // Start the streaming on the WebSocket "exec" client.
339 errorChan <- exec.StreamWithContext(context.Background(), *options)
340 }()
341
342 select {
343 case <-time.After(wait.ForeverTestTimeout):
344 t.Fatalf("expect stream to be closed after connection is closed.")
345 case err := <-errorChan:
346 // Expect exit code error on error stream.
347 if err == nil {
348 t.Errorf("expected error, but received none")
349 }
350 expectedError := fmt.Sprintf("command terminated with exit code %d", expectedExitCode)
351 // Compare expected error with exit code to actual error.
352 if expectedError != err.Error() {
353 t.Errorf("expected error (%s), got (%s)", expectedError, err)
354 }
355 }
356 // Validate the streamtranslator metrics; an exit code error is considered 200 success.
357 metricNames := []string{"apiserver_stream_translator_requests_total"}
358 expected := `
359 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
360 # TYPE apiserver_stream_translator_requests_total counter
361 apiserver_stream_translator_requests_total{code="200"} 1
362 `
363 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
364 t.Fatal(err)
365 }
366 }
367
368 // TestStreamTranslator_MultipleReadChannels tests two streams (STDOUT, STDERR) reading from
369 // the connections at the same time.
370 func TestStreamTranslator_MultipleReadChannels(t *testing.T) {
371 metrics.Register()
372 metrics.ResetForTest()
373 t.Cleanup(metrics.ResetForTest)
374 // Create upstream fake SPDY server which copies STDIN back onto STDOUT and STDERR stream.
375 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
376 ctx, err := createSPDYServerStreams(w, req, Options{
377 Stdin: true,
378 Stdout: true,
379 Stderr: true,
380 })
381 if err != nil {
382 t.Errorf("error on createHTTPStreams: %v", err)
383 return
384 }
385 defer ctx.conn.Close()
386 // TeeReader copies data read on STDIN onto STDERR.
387 stdinReader := io.TeeReader(ctx.stdinStream, ctx.stderrStream)
388 // Also copy STDIN to STDOUT.
389 _, err = io.Copy(ctx.stdoutStream, stdinReader)
390 if err != nil {
391 t.Errorf("error copying STDIN to STDOUT: %v", err)
392 }
393 }))
394 defer spdyServer.Close()
395 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with
396 // streams STDIN, STDOUT, and STDERR. Create test server from StreamTranslatorHandler.
397 spdyLocation, err := url.Parse(spdyServer.URL)
398 if err != nil {
399 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
400 }
401 spdyTransport, err := fakeTransport()
402 if err != nil {
403 t.Fatalf("Unexpected error creating transport: %v", err)
404 }
405 streams := Options{Stdin: true, Stdout: true, Stderr: true}
406 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
407 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
408 streamTranslator.ServeHTTP(w, req)
409 }))
410 defer streamTranslatorServer.Close()
411 // Now create the websocket client (executor), and point it to the "streamTranslatorServer".
412 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
413 if err != nil {
414 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
415 }
416 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
417 if err != nil {
418 t.Errorf("unexpected error creating websocket executor: %v", err)
419 }
420 // Generate random data, and set it up to stream on STDIN. The data will be
421 // returned on the STDOUT and STDERR buffer.
422 randomSize := 1024 * 1024
423 randomData := make([]byte, randomSize)
424 if _, err := rand.Read(randomData); err != nil {
425 t.Errorf("unexpected error reading random data: %v", err)
426 }
427 var stdout, stderr bytes.Buffer
428 options := &remotecommand.StreamOptions{
429 Stdin: bytes.NewReader(randomData),
430 Stdout: &stdout,
431 Stderr: &stderr,
432 }
433 errorChan := make(chan error)
434 go func() {
435 // Start the streaming on the WebSocket "exec" client.
436 errorChan <- exec.StreamWithContext(context.Background(), *options)
437 }()
438
439 select {
440 case <-time.After(wait.ForeverTestTimeout):
441 t.Fatalf("expect stream to be closed after connection is closed.")
442 case err := <-errorChan:
443 if err != nil {
444 t.Errorf("unexpected error: %v", err)
445 }
446 }
447 stdoutBytes, err := io.ReadAll(bytes.NewReader(stdout.Bytes()))
448 if err != nil {
449 t.Errorf("error reading the stream: %v", err)
450 return
451 }
452 // Check the random data sent on STDIN was the same returned on STDOUT.
453 if !bytes.Equal(stdoutBytes, randomData) {
454 t.Errorf("unexpected data received: %d sent: %d", len(stdoutBytes), len(randomData))
455 }
456 stderrBytes, err := io.ReadAll(bytes.NewReader(stderr.Bytes()))
457 if err != nil {
458 t.Errorf("error reading the stream: %v", err)
459 return
460 }
461 // Check the random data sent on STDIN was the same returned on STDERR.
462 if !bytes.Equal(stderrBytes, randomData) {
463 t.Errorf("unexpected data received: %d sent: %d", len(stderrBytes), len(randomData))
464 }
465 // Validate the streamtranslator metrics; should have one 200 success.
466 metricNames := []string{"apiserver_stream_translator_requests_total"}
467 expected := `
468 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
469 # TYPE apiserver_stream_translator_requests_total counter
470 apiserver_stream_translator_requests_total{code="200"} 1
471 `
472 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
473 t.Fatal(err)
474 }
475 }
476
477 // TestStreamTranslator_ThrottleReadChannels tests two streams (STDOUT, STDERR) using rate limited streams.
478 func TestStreamTranslator_ThrottleReadChannels(t *testing.T) {
479 // Create upstream fake SPDY server which copies STDIN back onto STDOUT and STDERR stream.
480 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
481 ctx, err := createSPDYServerStreams(w, req, Options{
482 Stdin: true,
483 Stdout: true,
484 Stderr: true,
485 })
486 if err != nil {
487 t.Errorf("error on createHTTPStreams: %v", err)
488 return
489 }
490 defer ctx.conn.Close()
491 // TeeReader copies data read on STDIN onto STDERR.
492 stdinReader := io.TeeReader(ctx.stdinStream, ctx.stderrStream)
493 // Also copy STDIN to STDOUT.
494 _, err = io.Copy(ctx.stdoutStream, stdinReader)
495 if err != nil {
496 t.Errorf("error copying STDIN to STDOUT: %v", err)
497 }
498 }))
499 defer spdyServer.Close()
500 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with
501 // streams STDIN, STDOUT, and STDERR. Create test server from StreamTranslatorHandler.
502 spdyLocation, err := url.Parse(spdyServer.URL)
503 if err != nil {
504 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
505 }
506 spdyTransport, err := fakeTransport()
507 if err != nil {
508 t.Fatalf("Unexpected error creating transport: %v", err)
509 }
510 streams := Options{Stdin: true, Stdout: true, Stderr: true}
511 maxBytesPerSec := 900 * 1024 // slightly less than the 1MB that is being transferred to exercise throttling.
512 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, int64(maxBytesPerSec), streams)
513 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
514 streamTranslator.ServeHTTP(w, req)
515 }))
516 defer streamTranslatorServer.Close()
517 // Now create the websocket client (executor), and point it to the "streamTranslatorServer".
518 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
519 if err != nil {
520 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
521 }
522 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
523 if err != nil {
524 t.Errorf("unexpected error creating websocket executor: %v", err)
525 }
526 // Generate random data, and set it up to stream on STDIN. The data will be
527 // returned on the STDOUT and STDERR buffer.
528 randomSize := 1024 * 1024
529 randomData := make([]byte, randomSize)
530 if _, err := rand.Read(randomData); err != nil {
531 t.Errorf("unexpected error reading random data: %v", err)
532 }
533 var stdout, stderr bytes.Buffer
534 options := &remotecommand.StreamOptions{
535 Stdin: bytes.NewReader(randomData),
536 Stdout: &stdout,
537 Stderr: &stderr,
538 }
539 errorChan := make(chan error)
540 go func() {
541 // Start the streaming on the WebSocket "exec" client.
542 errorChan <- exec.StreamWithContext(context.Background(), *options)
543 }()
544
545 select {
546 case <-time.After(wait.ForeverTestTimeout):
547 t.Fatalf("expect stream to be closed after connection is closed.")
548 case err := <-errorChan:
549 if err != nil {
550 t.Errorf("unexpected error: %v", err)
551 }
552 }
553 stdoutBytes, err := io.ReadAll(bytes.NewReader(stdout.Bytes()))
554 if err != nil {
555 t.Errorf("error reading the stream: %v", err)
556 return
557 }
558 // Check the random data sent on STDIN was the same returned on STDOUT.
559 if !bytes.Equal(stdoutBytes, randomData) {
560 t.Errorf("unexpected data received: %d sent: %d", len(stdoutBytes), len(randomData))
561 }
562 stderrBytes, err := io.ReadAll(bytes.NewReader(stderr.Bytes()))
563 if err != nil {
564 t.Errorf("error reading the stream: %v", err)
565 return
566 }
567 // Check the random data sent on STDIN was the same returned on STDERR.
568 if !bytes.Equal(stderrBytes, randomData) {
569 t.Errorf("unexpected data received: %d sent: %d", len(stderrBytes), len(randomData))
570 }
571 }
572
573 // fakeTerminalSizeQueue implements TerminalSizeQueue, returning a random set of
574 // "maxSizes" number of TerminalSizes, storing the TerminalSizes in "sizes" slice.
575 type fakeTerminalSizeQueue struct {
576 maxSizes int
577 terminalSizes []remotecommand.TerminalSize
578 }
579
580 // newTerminalSizeQueue returns a pointer to a fakeTerminalSizeQueue passing
581 // "max" number of random TerminalSizes created.
582 func newTerminalSizeQueue(max int) *fakeTerminalSizeQueue {
583 return &fakeTerminalSizeQueue{
584 maxSizes: max,
585 terminalSizes: make([]remotecommand.TerminalSize, 0, max),
586 }
587 }
588
589 // Next returns a pointer to the next random TerminalSize, or nil if we have
590 // already returned "maxSizes" TerminalSizes already. Stores the randomly
591 // created TerminalSize in "terminalSizes" field for later validation.
592 func (f *fakeTerminalSizeQueue) Next() *remotecommand.TerminalSize {
593 if len(f.terminalSizes) >= f.maxSizes {
594 return nil
595 }
596 size := randomTerminalSize()
597 f.terminalSizes = append(f.terminalSizes, size)
598 return &size
599 }
600
601 // randomTerminalSize returns a TerminalSize with random values in the
602 // range (0-65535) for the fields Width and Height.
603 func randomTerminalSize() remotecommand.TerminalSize {
604 randWidth := uint16(mrand.Intn(int(math.Pow(2, 16))))
605 randHeight := uint16(mrand.Intn(int(math.Pow(2, 16))))
606 return remotecommand.TerminalSize{
607 Width: randWidth,
608 Height: randHeight,
609 }
610 }
611
612 // TestStreamTranslator_MultipleWriteChannels
613 func TestStreamTranslator_TTYResizeChannel(t *testing.T) {
614 metrics.Register()
615 metrics.ResetForTest()
616 t.Cleanup(metrics.ResetForTest)
617 // Create the fake terminal size queue and the actualTerminalSizes which
618 // will be received at the opposite websocket endpoint.
619 numSizeQueue := 10000
620 sizeQueue := newTerminalSizeQueue(numSizeQueue)
621 actualTerminalSizes := make([]remotecommand.TerminalSize, 0, numSizeQueue)
622 // Create upstream fake SPDY server which copies STDIN back onto STDERR stream.
623 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
624 ctx, err := createSPDYServerStreams(w, req, Options{
625 Tty: true,
626 })
627 if err != nil {
628 t.Errorf("error on createHTTPStreams: %v", err)
629 return
630 }
631 defer ctx.conn.Close()
632 // Read the terminal resize requests, storing them in actualTerminalSizes
633 for i := 0; i < numSizeQueue; i++ {
634 actualTerminalSize := <-ctx.resizeChan
635 actualTerminalSizes = append(actualTerminalSizes, actualTerminalSize)
636 }
637 }))
638 defer spdyServer.Close()
639 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with
640 // resize (TTY resize) stream. Create test server from StreamTranslatorHandler.
641 spdyLocation, err := url.Parse(spdyServer.URL)
642 if err != nil {
643 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
644 }
645 spdyTransport, err := fakeTransport()
646 if err != nil {
647 t.Fatalf("Unexpected error creating transport: %v", err)
648 }
649 streams := Options{Tty: true}
650 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
651 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
652 streamTranslator.ServeHTTP(w, req)
653 }))
654 defer streamTranslatorServer.Close()
655 // Now create the websocket client (executor), and point it to the "streamTranslatorServer".
656 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
657 if err != nil {
658 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
659 }
660 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
661 if err != nil {
662 t.Errorf("unexpected error creating websocket executor: %v", err)
663 }
664 options := &remotecommand.StreamOptions{
665 Tty: true,
666 TerminalSizeQueue: sizeQueue,
667 }
668 errorChan := make(chan error)
669 go func() {
670 // Start the streaming on the WebSocket "exec" client.
671 errorChan <- exec.StreamWithContext(context.Background(), *options)
672 }()
673
674 select {
675 case <-time.After(wait.ForeverTestTimeout):
676 t.Fatalf("expect stream to be closed after connection is closed.")
677 case err := <-errorChan:
678 if err != nil {
679 t.Errorf("unexpected error: %v", err)
680 }
681 }
682 // Validate the random TerminalSizes sent on the resize stream are the same
683 // as the actual TerminalSizes received at the websocket server.
684 if len(actualTerminalSizes) != numSizeQueue {
685 t.Fatalf("expected to receive num terminal resizes (%d), got (%d)",
686 numSizeQueue, len(actualTerminalSizes))
687 }
688 for i, actual := range actualTerminalSizes {
689 expected := sizeQueue.terminalSizes[i]
690 if !reflect.DeepEqual(expected, actual) {
691 t.Errorf("expected terminal resize window %v, got %v", expected, actual)
692 }
693 }
694 // Validate the streamtranslator metrics; should have one 200 success.
695 metricNames := []string{"apiserver_stream_translator_requests_total"}
696 expected := `
697 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
698 # TYPE apiserver_stream_translator_requests_total counter
699 apiserver_stream_translator_requests_total{code="200"} 1
700 `
701 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
702 t.Fatal(err)
703 }
704 }
705
706 // TestStreamTranslator_WebSocketServerErrors validates that when there is a problem creating
707 // the websocket server as the first step of the StreamTranslator an error is properly returned.
708 func TestStreamTranslator_WebSocketServerErrors(t *testing.T) {
709 metrics.Register()
710 metrics.ResetForTest()
711 t.Cleanup(metrics.ResetForTest)
712 spdyLocation, err := url.Parse("http://127.0.0.1")
713 if err != nil {
714 t.Fatalf("Unable to parse spdy server URL")
715 }
716 spdyTransport, err := fakeTransport()
717 if err != nil {
718 t.Fatalf("Unexpected error creating transport: %v", err)
719 }
720 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, Options{})
721 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
722 streamTranslator.ServeHTTP(w, req)
723 }))
724 defer streamTranslatorServer.Close()
725 // Now create the websocket client (executor), and point it to the "streamTranslatorServer".
726 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
727 if err != nil {
728 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
729 }
730 exec, err := remotecommand.NewWebSocketExecutorForProtocols(
731 &rest.Config{Host: streamTranslatorLocation.Host},
732 "GET",
733 streamTranslatorServer.URL,
734 rcconstants.StreamProtocolV4Name, // RemoteCommand V4 protocol is unsupported
735 )
736 if err != nil {
737 t.Errorf("unexpected error creating websocket executor: %v", err)
738 }
739 errorChan := make(chan error)
740 go func() {
741 // Start the streaming on the WebSocket "exec" client. The WebSocket server within the
742 // StreamTranslator propagates an error here because the V4 protocol is not supported.
743 errorChan <- exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{})
744 }()
745
746 select {
747 case <-time.After(wait.ForeverTestTimeout):
748 t.Fatalf("expect stream to be closed after connection is closed.")
749 case err := <-errorChan:
750 // Must return "websocket unable to upgrade" (bad handshake) error.
751 if err == nil {
752 t.Fatalf("expected error, but received none")
753 }
754 if !strings.Contains(err.Error(), "unable to upgrade streaming request") {
755 t.Errorf("expected websocket bad handshake error, got (%s)", err)
756 }
757 }
758 // Validate the streamtranslator metrics; should have one 500 failure.
759 metricNames := []string{"apiserver_stream_translator_requests_total"}
760 expected := `
761 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
762 # TYPE apiserver_stream_translator_requests_total counter
763 apiserver_stream_translator_requests_total{code="400"} 1
764 `
765 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
766 t.Fatal(err)
767 }
768 }
769
770 // TestStreamTranslator_BlockRedirects verifies that the StreamTranslator will *not* follow
771 // redirects; it will thrown an error instead.
772 func TestStreamTranslator_BlockRedirects(t *testing.T) {
773 metrics.Register()
774 metrics.ResetForTest()
775 t.Cleanup(metrics.ResetForTest)
776 for _, statusCode := range []int{
777 http.StatusMovedPermanently, // 301
778 http.StatusFound, // 302
779 http.StatusSeeOther, // 303
780 http.StatusTemporaryRedirect, // 307
781 http.StatusPermanentRedirect, // 308
782 } {
783 // Create upstream fake SPDY server which returns a redirect.
784 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
785 w.Header().Set("Location", "/")
786 w.WriteHeader(statusCode)
787 }))
788 defer spdyServer.Close()
789 spdyLocation, err := url.Parse(spdyServer.URL)
790 if err != nil {
791 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
792 }
793 spdyTransport, err := fakeTransport()
794 if err != nil {
795 t.Fatalf("Unexpected error creating transport: %v", err)
796 }
797 streams := Options{Stdout: true}
798 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
799 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
800 streamTranslator.ServeHTTP(w, req)
801 }))
802 defer streamTranslatorServer.Close()
803 // Now create the websocket client (executor), and point it to the "streamTranslatorServer".
804 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
805 if err != nil {
806 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
807 }
808 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
809 if err != nil {
810 t.Errorf("unexpected error creating websocket executor: %v", err)
811 }
812 errorChan := make(chan error)
813 go func() {
814 // Start the streaming on the WebSocket "exec" client.
815 // Should return "redirect not allowed" error.
816 errorChan <- exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{})
817 }()
818
819 select {
820 case <-time.After(wait.ForeverTestTimeout):
821 t.Fatalf("expect stream to be closed after connection is closed.")
822 case err := <-errorChan:
823 // Must return "redirect now allowed" error.
824 if err == nil {
825 t.Fatalf("expected error, but received none")
826 }
827 if !strings.Contains(err.Error(), "redirect not allowed") {
828 t.Errorf("expected redirect not allowed error, got (%s)", err)
829 }
830 }
831 // Validate the streamtranslator metrics; should have one 500 failure each loop.
832 metricNames := []string{"apiserver_stream_translator_requests_total"}
833 expected := `
834 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
835 # TYPE apiserver_stream_translator_requests_total counter
836 apiserver_stream_translator_requests_total{code="500"} 1
837 `
838 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
839 t.Fatal(err)
840 }
841 metrics.ResetForTest() // Clear metrics each loop
842 }
843 }
844
845 // streamContext encapsulates the structures necessary to communicate through
846 // a SPDY connection, including the Reader/Writer streams.
847 type streamContext struct {
848 conn io.Closer
849 stdinStream io.ReadCloser
850 stdoutStream io.WriteCloser
851 stderrStream io.WriteCloser
852 resizeStream io.ReadCloser
853 resizeChan chan remotecommand.TerminalSize
854 writeStatus func(status *apierrors.StatusError) error
855 }
856
857 type streamAndReply struct {
858 httpstream.Stream
859 replySent <-chan struct{}
860 }
861
862 // CreateSPDYServerStreams upgrades the passed HTTP request to a SPDY bi-directional streaming
863 // connection with remote command streams defined in passed options. Returns a streamContext
864 // structure containing the Reader/Writer streams to communicate through the SDPY connection.
865 // Returns an error if unable to upgrade the HTTP connection to a SPDY connection.
866 func createSPDYServerStreams(w http.ResponseWriter, req *http.Request, opts Options) (*streamContext, error) {
867 _, err := httpstream.Handshake(req, w, []string{rcconstants.StreamProtocolV4Name})
868 if err != nil {
869 return nil, err
870 }
871
872 upgrader := spdy.NewResponseUpgrader()
873 streamCh := make(chan streamAndReply)
874 conn := upgrader.UpgradeResponse(w, req, func(stream httpstream.Stream, replySent <-chan struct{}) error {
875 streamCh <- streamAndReply{Stream: stream, replySent: replySent}
876 return nil
877 })
878 ctx := &streamContext{
879 conn: conn,
880 }
881
882 // wait for stream
883 replyChan := make(chan struct{}, 5)
884 defer close(replyChan)
885 receivedStreams := 0
886 expectedStreams := 1 // expect at least the error stream
887 if opts.Stdout {
888 expectedStreams++
889 }
890 if opts.Stdin {
891 expectedStreams++
892 }
893 if opts.Stderr {
894 expectedStreams++
895 }
896 if opts.Tty {
897 expectedStreams++
898 }
899 WaitForStreams:
900 for {
901 select {
902 case stream := <-streamCh:
903 streamType := stream.Headers().Get(v1.StreamType)
904 switch streamType {
905 case v1.StreamTypeError:
906 replyChan <- struct{}{}
907 ctx.writeStatus = v4WriteStatusFunc(stream)
908 case v1.StreamTypeStdout:
909 replyChan <- struct{}{}
910 ctx.stdoutStream = stream
911 case v1.StreamTypeStdin:
912 replyChan <- struct{}{}
913 ctx.stdinStream = stream
914 case v1.StreamTypeStderr:
915 replyChan <- struct{}{}
916 ctx.stderrStream = stream
917 case v1.StreamTypeResize:
918 replyChan <- struct{}{}
919 ctx.resizeStream = stream
920 default:
921 // add other stream ...
922 return nil, errors.New("unimplemented stream type")
923 }
924 case <-replyChan:
925 receivedStreams++
926 if receivedStreams == expectedStreams {
927 break WaitForStreams
928 }
929 }
930 }
931
932 if ctx.resizeStream != nil {
933 ctx.resizeChan = make(chan remotecommand.TerminalSize)
934 go handleResizeEvents(req.Context(), ctx.resizeStream, ctx.resizeChan)
935 }
936
937 return ctx, nil
938 }
939
940 func v4WriteStatusFunc(stream io.Writer) func(status *apierrors.StatusError) error {
941 return func(status *apierrors.StatusError) error {
942 bs, err := json.Marshal(status.Status())
943 if err != nil {
944 return err
945 }
946 _, err = stream.Write(bs)
947 return err
948 }
949 }
950
951 func fakeTransport() (*http.Transport, error) {
952 cfg := &transport.Config{
953 TLS: transport.TLSConfig{
954 Insecure: true,
955 CAFile: "",
956 },
957 }
958 rt, err := transport.New(cfg)
959 if err != nil {
960 return nil, err
961 }
962 t, ok := rt.(*http.Transport)
963 if !ok {
964 return nil, fmt.Errorf("unknown transport type: %T", rt)
965 }
966 return t, nil
967 }