Debian Codegraph @

pkg/util/proxy/streamtunnel.go

raw
1 /*
2 Copyright 2024 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package proxy
18
19 import (
20 "bufio"
21 "bytes"
22 "context"
23 "errors"
24 "fmt"
25 "net"
26 "net/http"
27 "strconv"
28 "strings"
29 "sync"
30 "time"
31
32 gwebsocket "github.com/gorilla/websocket"
33
34 "k8s.io/apimachinery/pkg/util/httpstream"
35 "k8s.io/apimachinery/pkg/util/httpstream/spdy"
36 "k8s.io/apimachinery/pkg/util/httpstream/wsstream"
37 utilnet "k8s.io/apimachinery/pkg/util/net"
38 constants "k8s.io/apimachinery/pkg/util/portforward"
39 "k8s.io/apiserver/pkg/util/proxy/metrics"
40 "k8s.io/client-go/tools/portforward"
41 "k8s.io/klog/v2"
42 )
43
44 // TunnelingHandler is a handler which tunnels SPDY through WebSockets.
45 type TunnelingHandler struct {
46 // Used to communicate between upstream SPDY and downstream tunnel.
47 upgradeHandler http.Handler
48 }
49
50 // NewTunnelingHandler is used to create the tunnel between an upstream
51 // SPDY connection and a downstream tunneling connection through the stored
52 // UpgradeAwareProxy.
53 func NewTunnelingHandler(upgradeHandler http.Handler) *TunnelingHandler {
54 return &TunnelingHandler{upgradeHandler: upgradeHandler}
55 }
56
57 // ServeHTTP uses the upgradeHandler to tunnel between a downstream tunneling
58 // connection and an upstream SPDY connection. The tunneling connection is
59 // a wrapped WebSockets connection which communicates SPDY framed data. In the
60 // case the upstream upgrade fails, we delegate communication to the passed
61 // in "w" ResponseWriter.
62 func (h *TunnelingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
63 klog.V(4).Infoln("TunnelingHandler ServeHTTP")
64
65 spdyProtocols := spdyProtocolsFromWebsocketProtocols(req)
66 if len(spdyProtocols) == 0 {
67 metrics.IncStreamTunnelRequest(req.Context(), strconv.Itoa(http.StatusBadRequest))
68 http.Error(w, "unable to upgrade: no tunneling spdy protocols provided", http.StatusBadRequest)
69 return
70 }
71
72 spdyRequest := createSPDYRequest(req, spdyProtocols...)
73
74 // The fields "w" and "conn" are mutually exclusive. Either a successful upgrade occurs
75 // and the "conn" is hijacked and used in the subsequent upgradeHandler, or
76 // the upgrade failed, and "w" is the delegate used for the non-upgrade response.
77 writer := &tunnelingResponseWriter{
78 // "w" is used in the non-upgrade error cases called in the upgradeHandler.
79 w: w,
80 // "conn" is returned in the successful upgrade case when hijacked in the upgradeHandler.
81 conn: &headerInterceptingConn{
82 initializableConn: &tunnelingWebsocketUpgraderConn{
83 w: w,
84 req: req,
85 },
86 },
87 }
88
89 klog.V(4).Infoln("Tunnel spdy through websockets using the UpgradeAwareProxy")
90 h.upgradeHandler.ServeHTTP(writer, spdyRequest)
91 }
92
93 // createSPDYRequest modifies the passed request to remove
94 // WebSockets headers and add SPDY upgrade information, including
95 // spdy protocols acceptable to the client.
96 func createSPDYRequest(req *http.Request, spdyProtocols ...string) *http.Request {
97 clone := utilnet.CloneRequest(req)
98 // Clean up the websocket headers from the http request.
99 clone.Header.Del(wsstream.WebSocketProtocolHeader)
100 clone.Header.Del("Sec-Websocket-Key")
101 clone.Header.Del("Sec-Websocket-Version")
102 clone.Header.Del(httpstream.HeaderUpgrade)
103 // Update the http request for an upstream SPDY upgrade.
104 clone.Method = "POST"
105 clone.Body = nil // Remove the request body which is unused.
106 clone.Header.Set(httpstream.HeaderUpgrade, spdy.HeaderSpdy31)
107 clone.Header.Del(httpstream.HeaderProtocolVersion)
108 for i := range spdyProtocols {
109 clone.Header.Add(httpstream.HeaderProtocolVersion, spdyProtocols[i])
110 }
111 return clone
112 }
113
114 // spdyProtocolsFromWebsocketProtocols returns a list of spdy protocols by filtering
115 // to Kubernetes websocket subprotocols prefixed with "SPDY/3.1+", then removing the prefix
116 func spdyProtocolsFromWebsocketProtocols(req *http.Request) []string {
117 var spdyProtocols []string
118 for _, protocol := range gwebsocket.Subprotocols(req) {
119 if strings.HasPrefix(protocol, constants.WebsocketsSPDYTunnelingPrefix) && strings.HasSuffix(protocol, constants.KubernetesSuffix) {
120 spdyProtocols = append(spdyProtocols, strings.TrimPrefix(protocol, constants.WebsocketsSPDYTunnelingPrefix))
121 }
122 }
123 return spdyProtocols
124 }
125
126 var _ http.ResponseWriter = &tunnelingResponseWriter{}
127 var _ http.Hijacker = &tunnelingResponseWriter{}
128
129 // tunnelingResponseWriter implements the http.ResponseWriter and http.Hijacker interfaces.
130 // Only non-upgrade responses can be written using WriteHeader() and Write().
131 // Once Write or WriteHeader is called, Hijack returns an error.
132 // Once Hijack is called, Write, WriteHeader, and Hijack return errors.
133 type tunnelingResponseWriter struct {
134 // w is used to delegate Header(), WriteHeader(), and Write() calls
135 w http.ResponseWriter
136 // conn is returned from Hijack()
137 conn net.Conn
138 // mu guards writes
139 mu sync.Mutex
140 // wrote tracks whether WriteHeader or Write has been called
141 written bool
142 // hijacked tracks whether Hijack has been called
143 hijacked bool
144 }
145
146 // Hijack returns a delegate "net.Conn".
147 // An error is returned if Write(), WriteHeader(), or Hijack() was previously called.
148 // The returned bufio.ReadWriter is always nil.
149 func (w *tunnelingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
150 w.mu.Lock()
151 defer w.mu.Unlock()
152 if w.written {
153 klog.Errorf("Hijack called after write")
154 return nil, nil, errors.New("connection has already been written to")
155 }
156 if w.hijacked {
157 klog.Errorf("Hijack called after hijack")
158 return nil, nil, errors.New("connection has already been hijacked")
159 }
160 w.hijacked = true
161 klog.V(6).Infof("Hijack returning websocket tunneling net.Conn")
162 return w.conn, nil, nil
163 }
164
165 // Header is delegated to the stored "http.ResponseWriter".
166 func (w *tunnelingResponseWriter) Header() http.Header {
167 return w.w.Header()
168 }
169
170 // Write is delegated to the stored "http.ResponseWriter".
171 func (w *tunnelingResponseWriter) Write(p []byte) (int, error) {
172 w.mu.Lock()
173 defer w.mu.Unlock()
174 if w.hijacked {
175 klog.Errorf("Write called after hijack")
176 return 0, http.ErrHijacked
177 }
178 w.written = true
179 return w.w.Write(p)
180 }
181
182 // WriteHeader is delegated to the stored "http.ResponseWriter".
183 func (w *tunnelingResponseWriter) WriteHeader(statusCode int) {
184 w.mu.Lock()
185 defer w.mu.Unlock()
186 if w.written {
187 klog.Errorf("WriteHeader called after write")
188 return
189 }
190 if w.hijacked {
191 klog.Errorf("WriteHeader called after hijack")
192 return
193 }
194 w.written = true
195
196 if statusCode == http.StatusSwitchingProtocols {
197 // 101 upgrade responses must come via the hijacked connection, not WriteHeader
198 klog.Errorf("WriteHeader called with 101 upgrade")
199 http.Error(w.w, "unexpected upgrade", http.StatusInternalServerError)
200 return
201 }
202
203 // pass through non-upgrade responses we don't need to translate
204 w.w.WriteHeader(statusCode)
205 }
206
207 // headerInterceptingConn wraps the tunneling "net.Conn" to drain the
208 // HTTP response status/headers from the upstream SPDY connection, then use
209 // that to decide how to initialize the delegate connection for writes.
210 type headerInterceptingConn struct {
211 // initializableConn is delegated to for all net.Conn methods.
212 // initializableConn.Write() is not called until response headers have been read
213 // and initializableConn#InitializeWrite() has been called with the result.
214 initializableConn
215
216 lock sync.Mutex
217 headerBuffer []byte
218 initialized bool
219 initializeErr error
220 }
221
222 // initializableConn is a connection that will be initialized before any calls to Write are made
223 type initializableConn interface {
224 net.Conn
225 // InitializeWrite is called when the backend response headers have been read.
226 // backendResponse contains the parsed headers.
227 // backendResponseBytes are the raw bytes the headers were parsed from.
228 InitializeWrite(backendResponse *http.Response, backendResponseBytes []byte) error
229 }
230
231 const maxHeaderBytes = 1 << 20
232
233 // token for normal header / body separation (\r\n\r\n, but go tolerates the leading \r being absent)
234 var lfCRLF = []byte("\n\r\n")
235
236 // token for header / body separation without \r (which go tolerates)
237 var lfLF = []byte("\n\n")
238
239 // Write intercepts to initially swallow the HTTP response, then
240 // delegate to the tunneling "net.Conn" once the response has been
241 // seen and processed.
242 func (h *headerInterceptingConn) Write(b []byte) (int, error) {
243 h.lock.Lock()
244 defer h.lock.Unlock()
245
246 if h.initializeErr != nil {
247 return 0, h.initializeErr
248 }
249 if h.initialized {
250 return h.initializableConn.Write(b)
251 }
252
253 // Guard against excessive buffering
254 if len(h.headerBuffer)+len(b) > maxHeaderBytes {
255 return 0, fmt.Errorf("header size limit exceeded")
256 }
257
258 // Accumulate into headerBuffer
259 h.headerBuffer = append(h.headerBuffer, b...)
260
261 // Attempt to parse http response headers
262 var headerBytes, bodyBytes []byte
263 if i := bytes.Index(h.headerBuffer, lfCRLF); i != -1 {
264 // headers terminated with \n\r\n
265 headerBytes = h.headerBuffer[0 : i+len(lfCRLF)]
266 bodyBytes = h.headerBuffer[i+len(lfCRLF):]
267 } else if i := bytes.Index(h.headerBuffer, lfLF); i != -1 {
268 // headers terminated with \n\n (which go tolerates)
269 headerBytes = h.headerBuffer[0 : i+len(lfLF)]
270 bodyBytes = h.headerBuffer[i+len(lfLF):]
271 } else {
272 // don't yet have a complete set of headers yet
273 return len(b), nil
274 }
275 resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(headerBytes)), nil)
276 if err != nil {
277 klog.Errorf("invalid headers: %v", err)
278 h.initializeErr = err
279 return len(b), err
280 }
281 resp.Body.Close() //nolint:errcheck
282
283 h.headerBuffer = nil
284 h.initialized = true
285 h.initializeErr = h.initializableConn.InitializeWrite(resp, headerBytes)
286 if h.initializeErr != nil {
287 return len(b), h.initializeErr
288 }
289 if len(bodyBytes) > 0 {
290 _, err = h.initializableConn.Write(bodyBytes)
291 }
292 return len(b), err
293 }
294
295 type tunnelingWebsocketUpgraderConn struct {
296 // req is the websocket request, used for upgrading
297 req *http.Request
298 // w is the websocket writer, used for upgrading and writing error responses
299 w http.ResponseWriter
300
301 // lock guards conn and err
302 lock sync.RWMutex
303 // if conn is non-nil, InitializeWrite succeeded
304 conn net.Conn
305 // if err is non-nil, InitializeWrite failed or Close was called before InitializeWrite
306 err error
307 }
308
309 func (u *tunnelingWebsocketUpgraderConn) InitializeWrite(backendResponse *http.Response, backendResponseBytes []byte) (err error) {
310 // make sure we close a connection we open in error cases
311 var conn net.Conn
312 defer func() {
313 if err != nil && conn != nil {
314 conn.Close() //nolint:errcheck
315 }
316 }()
317
318 u.lock.Lock()
319 defer u.lock.Unlock()
320 if u.conn != nil {
321 return fmt.Errorf("InitializeWrite already called")
322 }
323 if u.err != nil {
324 return u.err
325 }
326
327 if backendResponse.StatusCode == http.StatusSwitchingProtocols {
328 connectionHeader := strings.ToLower(backendResponse.Header.Get(httpstream.HeaderConnection))
329 upgradeHeader := strings.ToLower(backendResponse.Header.Get(httpstream.HeaderUpgrade))
330 if !strings.Contains(connectionHeader, strings.ToLower(httpstream.HeaderUpgrade)) || !strings.Contains(upgradeHeader, strings.ToLower(spdy.HeaderSpdy31)) {
331 klog.Errorf("unable to upgrade: missing upgrade headers in response: %#v", backendResponse.Header)
332 u.err = fmt.Errorf("unable to upgrade: missing upgrade headers in response")
333 metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(http.StatusInternalServerError))
334 http.Error(u.w, u.err.Error(), http.StatusInternalServerError)
335 return u.err
336 }
337
338 // Translate the server's chosen SPDY protocol into the tunneled websocket protocol for the handshake
339 var serverWebsocketProtocols []string
340 if backendSPDYProtocol := strings.TrimSpace(backendResponse.Header.Get(httpstream.HeaderProtocolVersion)); backendSPDYProtocol != "" {
341 serverWebsocketProtocols = []string{constants.WebsocketsSPDYTunnelingPrefix + backendSPDYProtocol}
342 } else {
343 serverWebsocketProtocols = []string{}
344 }
345
346 // Try to upgrade the websocket connection.
347 // Beyond this point, we don't need to write errors to the response.
348 var upgrader = gwebsocket.Upgrader{
349 CheckOrigin: func(r *http.Request) bool { return true },
350 Subprotocols: serverWebsocketProtocols,
351 }
352 conn, err := upgrader.Upgrade(u.w, u.req, nil)
353 if err != nil {
354 klog.Errorf("error upgrading websocket connection: %v", err)
355 metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(http.StatusInternalServerError))
356 u.err = err
357 return u.err
358 }
359
360 klog.V(4).Infof("websocket connection created: %s", conn.Subprotocol())
361 metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(http.StatusSwitchingProtocols))
362 u.conn = portforward.NewTunnelingConnection("server", conn)
363 return nil
364 }
365
366 // anything other than an upgrade should pass through the backend response
367 klog.Errorf("SPDY upgrade failed: %s", backendResponse.Status)
368 metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(backendResponse.StatusCode))
369
370 // try to hijack
371 conn, _, err = u.w.(http.Hijacker).Hijack()
372 if err != nil {
373 klog.Errorf("Unable to hijack response: %v", err)
374 u.err = err
375 return u.err
376 }
377 // replay the backend response bytes to the hijacked conn
378 conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) //nolint:errcheck
379 _, err = conn.Write(backendResponseBytes)
380 if err != nil {
381 u.err = err
382 return u.err
383 }
384 u.conn = conn
385 return nil
386 }
387
388 func (u *tunnelingWebsocketUpgraderConn) Read(b []byte) (n int, err error) {
389 u.lock.RLock()
390 defer u.lock.RUnlock()
391 if u.conn != nil {
392 return u.conn.Read(b)
393 }
394 if u.err != nil {
395 return 0, u.err
396 }
397 // return empty read without blocking until we are initialized
398 return 0, nil
399 }
400 func (u *tunnelingWebsocketUpgraderConn) Write(b []byte) (n int, err error) {
401 u.lock.RLock()
402 defer u.lock.RUnlock()
403 if u.conn != nil {
404 return u.conn.Write(b)
405 }
406 if u.err != nil {
407 return 0, u.err
408 }
409 return 0, fmt.Errorf("Write called before Initialize")
410 }
411 func (u *tunnelingWebsocketUpgraderConn) Close() error {
412 u.lock.Lock()
413 defer u.lock.Unlock()
414 if u.conn != nil {
415 return u.conn.Close()
416 }
417 if u.err != nil {
418 return u.err
419 }
420 // record that we closed so we don't write again or try to initialize
421 u.err = fmt.Errorf("connection closed")
422 // write a response
423 http.Error(u.w, u.err.Error(), http.StatusInternalServerError)
424 return nil
425 }
426 func (u *tunnelingWebsocketUpgraderConn) LocalAddr() net.Addr {
427 u.lock.RLock()
428 defer u.lock.RUnlock()
429 if u.conn != nil {
430 return u.conn.LocalAddr()
431 }
432 return noopAddr{}
433 }
434 func (u *tunnelingWebsocketUpgraderConn) RemoteAddr() net.Addr {
435 u.lock.RLock()
436 defer u.lock.RUnlock()
437 if u.conn != nil {
438 return u.conn.RemoteAddr()
439 }
440 return noopAddr{}
441 }
442 func (u *tunnelingWebsocketUpgraderConn) SetDeadline(t time.Time) error {
443 u.lock.RLock()
444 defer u.lock.RUnlock()
445 if u.conn != nil {
446 return u.conn.SetDeadline(t)
447 }
448 return nil
449 }
450 func (u *tunnelingWebsocketUpgraderConn) SetReadDeadline(t time.Time) error {
451 u.lock.RLock()
452 defer u.lock.RUnlock()
453 if u.conn != nil {
454 return u.conn.SetReadDeadline(t)
455 }
456 return nil
457 }
458 func (u *tunnelingWebsocketUpgraderConn) SetWriteDeadline(t time.Time) error {
459 u.lock.RLock()
460 defer u.lock.RUnlock()
461 if u.conn != nil {
462 return u.conn.SetWriteDeadline(t)
463 }
464 return nil
465 }
466
467 type noopAddr struct{}
468
469 func (n noopAddr) Network() string { return "" }
470 func (n noopAddr) String() string { return "" }