Debian Codegraph @

konnectivity-client/pkg/client/client.go

raw
1 /*
2 Copyright 2019 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package client
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "io"
24 "math/rand"
25 "net"
26 "sync"
27 "sync/atomic"
28 "time"
29
30 "google.golang.org/grpc"
31 "k8s.io/klog/v2"
32
33 "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics"
34 commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics"
35 "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client"
36 )
37
38 // Tunnel provides ability to dial a connection through a tunnel.
39 type Tunnel interface {
40 // Dial connects to the address on the named network, similar to
41 // what net.Dial does. The only supported protocol is tcp.
42 DialContext(requestCtx context.Context, protocol, address string) (net.Conn, error)
43 // Done returns a channel that is closed when the tunnel is no longer serving any connections,
44 // and can no longer be used.
45 Done() <-chan struct{}
46 }
47
48 type dialResult struct {
49 err *dialFailure
50 connid int64
51 }
52
53 type pendingDial struct {
54 // resultCh is the channel to send the dial result to
55 resultCh chan<- dialResult
56 // cancelCh is the channel closed when resultCh no longer has a receiver
57 cancelCh <-chan struct{}
58 }
59
60 // TODO: Replace with a generic implementation once it is safe to assume the client is built with go1.18+
61 type pendingDialManager struct {
62 pendingDials map[int64]pendingDial
63 mutex sync.RWMutex
64 }
65
66 func (p *pendingDialManager) add(dialID int64, pd pendingDial) {
67 p.mutex.Lock()
68 defer p.mutex.Unlock()
69 p.pendingDials[dialID] = pd
70 }
71
72 func (p *pendingDialManager) remove(dialID int64) {
73 p.mutex.Lock()
74 defer p.mutex.Unlock()
75 delete(p.pendingDials, dialID)
76 }
77
78 func (p *pendingDialManager) get(dialID int64) (pendingDial, bool) {
79 p.mutex.RLock()
80 defer p.mutex.RUnlock()
81 pd, ok := p.pendingDials[dialID]
82 return pd, ok
83 }
84
85 // TODO: Replace with a generic implementation once it is safe to assume the client is built with go1.18+
86 type connectionManager struct {
87 conns map[int64]*conn
88 mutex sync.RWMutex
89 }
90
91 func (cm *connectionManager) add(connID int64, c *conn) {
92 cm.mutex.Lock()
93 defer cm.mutex.Unlock()
94 cm.conns[connID] = c
95 }
96
97 func (cm *connectionManager) remove(connID int64) {
98 cm.mutex.Lock()
99 defer cm.mutex.Unlock()
100 delete(cm.conns, connID)
101 }
102
103 func (cm *connectionManager) get(connID int64) (*conn, bool) {
104 cm.mutex.RLock()
105 defer cm.mutex.RUnlock()
106 c, ok := cm.conns[connID]
107 return c, ok
108 }
109
110 func (cm *connectionManager) closeAll() {
111 cm.mutex.Lock()
112 defer cm.mutex.Unlock()
113 for _, conn := range cm.conns {
114 close(conn.readCh)
115 }
116 }
117
118 // grpcTunnel implements Tunnel
119 type grpcTunnel struct {
120 stream client.ProxyService_ProxyClient
121 sendLock sync.Mutex
122 recvLock sync.Mutex
123 grpcConn clientConn
124 pendingDial pendingDialManager
125 conns connectionManager
126
127 // The tunnel will be closed if the caller fails to read via conn.Read()
128 // more than readTimeoutSeconds after a packet has been received.
129 readTimeoutSeconds int
130
131 // The done channel is closed after the tunnel has cleaned up all connections and is no longer
132 // serving.
133 done chan struct{}
134
135 // started is an atomic bool represented as a 0 or 1, and set to true when a single-use tunnel has been started (dialed).
136 // started should only be accessed through atomic methods.
137 // TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+
138 started uint32
139
140 // closing is an atomic bool represented as a 0 or 1, and set to true when the tunnel is being closed.
141 // closing should only be accessed through atomic methods.
142 // TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+
143 closing uint32
144
145 // Stores the current metrics.ClientConnectionStatus
146 prevStatus atomic.Value
147 }
148
149 type clientConn interface {
150 Close() error
151 }
152
153 var _ clientConn = &grpc.ClientConn{}
154
155 var (
156 // Expose metrics for client to register.
157 Metrics = metrics.Metrics
158 )
159
160 // CreateSingleUseGrpcTunnel creates a Tunnel to dial to a remote server through a
161 // gRPC based proxy service.
162 // Currently, a single tunnel supports a single connection, and the tunnel is closed when the connection is terminated
163 // The Dial() method of the returned tunnel should only be called once
164 // Deprecated 2022-06-07: use CreateSingleUseGrpcTunnelWithContext
165 func CreateSingleUseGrpcTunnel(tunnelCtx context.Context, address string, opts ...grpc.DialOption) (Tunnel, error) {
166 return CreateSingleUseGrpcTunnelWithContext(context.TODO(), tunnelCtx, address, opts...)
167 }
168
169 // CreateSingleUseGrpcTunnelWithContext creates a Tunnel to dial to a remote server through a
170 // gRPC based proxy service.
171 // Currently, a single tunnel supports a single connection.
172 // The tunnel is normally closed when the connection is terminated.
173 // If createCtx is cancelled before tunnel creation, an error will be returned.
174 // If tunnelCtx is cancelled while the tunnel is still in use, the tunnel (and any in flight connections) will be closed.
175 // The Dial() method of the returned tunnel should only be called once
176 func CreateSingleUseGrpcTunnelWithContext(createCtx, tunnelCtx context.Context, address string, opts ...grpc.DialOption) (Tunnel, error) {
177 c, err := grpc.DialContext(createCtx, address, opts...)
178 if err != nil {
179 return nil, err
180 }
181
182 grpcClient := client.NewProxyServiceClient(c)
183
184 stream, err := grpcClient.Proxy(tunnelCtx)
185 if err != nil {
186 c.Close()
187 return nil, err
188 }
189
190 tunnel := newUnstartedTunnel(stream, c)
191
192 go tunnel.serve(tunnelCtx)
193
194 return tunnel, nil
195 }
196
197 func newUnstartedTunnel(stream client.ProxyService_ProxyClient, c clientConn) *grpcTunnel {
198 t := grpcTunnel{
199 stream: stream,
200 grpcConn: c,
201 pendingDial: pendingDialManager{pendingDials: make(map[int64]pendingDial)},
202 conns: connectionManager{conns: make(map[int64]*conn)},
203 readTimeoutSeconds: 10,
204 done: make(chan struct{}),
205 started: 0,
206 }
207 s := metrics.ClientConnectionStatusCreated
208 t.prevStatus.Store(s)
209 metrics.Metrics.GetClientConnectionsMetric().WithLabelValues(string(s)).Inc()
210 return &t
211 }
212
213 func (t *grpcTunnel) updateMetric(status metrics.ClientConnectionStatus) {
214 select {
215 case <-t.Done():
216 return
217 default:
218 }
219
220 prevStatus := t.prevStatus.Swap(status).(metrics.ClientConnectionStatus)
221
222 m := metrics.Metrics.GetClientConnectionsMetric()
223 m.WithLabelValues(string(prevStatus)).Dec()
224 m.WithLabelValues(string(status)).Inc()
225 }
226
227 // closeMetric should be called exactly once to finalize client_connections metric.
228 func (t *grpcTunnel) closeMetric() {
229 select {
230 case <-t.Done():
231 return
232 default:
233 }
234 prevStatus := t.prevStatus.Load().(metrics.ClientConnectionStatus)
235
236 metrics.Metrics.GetClientConnectionsMetric().WithLabelValues(string(prevStatus)).Dec()
237 }
238
239 func (t *grpcTunnel) serve(tunnelCtx context.Context) {
240 defer func() {
241 t.grpcConn.Close()
242
243 // A connection in t.conns after serve() returns means
244 // we never received a CLOSE_RSP for it, so we need to
245 // close any channels remaining for these connections.
246 t.conns.closeAll()
247
248 t.closeMetric()
249
250 close(t.done)
251 }()
252
253 for {
254 pkt, err := t.Recv()
255 if err == io.EOF {
256 return
257 }
258 isClosing := t.isClosing()
259 if err != nil || pkt == nil {
260 if !isClosing {
261 klog.ErrorS(err, "stream read failure")
262 }
263 return
264 }
265 if isClosing {
266 return
267 }
268 klog.V(5).InfoS("[tracing] recv packet", "type", pkt.Type)
269
270 switch pkt.Type {
271 case client.PacketType_DIAL_RSP:
272 resp := pkt.GetDialResponse()
273 pendingDial, ok := t.pendingDial.get(resp.Random)
274
275 if !ok {
276 // If the DIAL_RSP does not match a pending dial, it means one of two things:
277 // 1. There was a second DIAL_RSP for the connection request (this is very unlikely but possible)
278 // 2. grpcTunnel.DialContext() returned early due to a dial timeout or the client canceling the context
279 //
280 // In either scenario, we should return here and close the tunnel as it is no longer needed.
281 kvs := []interface{}{"dialID", resp.Random, "connectionID", resp.ConnectID}
282 if resp.Error != "" {
283 kvs = append(kvs, "error", resp.Error)
284 }
285 klog.V(1).InfoS("DialResp not recognized; dropped", kvs...)
286 return
287 }
288
289 result := dialResult{connid: resp.ConnectID}
290 if resp.Error != "" {
291 result.err = &dialFailure{resp.Error, metrics.DialFailureEndpoint}
292 } else {
293 t.updateMetric(metrics.ClientConnectionStatusOk)
294 }
295 select {
296 // try to send to the result channel
297 case pendingDial.resultCh <- result:
298 // unblock if the cancel channel is closed
299 case <-pendingDial.cancelCh:
300 // Note: this condition can only be hit by a race condition where the
301 // DialContext() returns early (timeout) after the pendingDial is already
302 // fetched here, but before the result is sent.
303 klog.V(1).InfoS("Pending dial has been cancelled; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random)
304 return
305 case <-tunnelCtx.Done():
306 klog.V(1).InfoS("Tunnel has been closed; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random)
307 return
308 }
309
310 if resp.Error != "" {
311 // On dial error, avoid leaking serve goroutine.
312 return
313 }
314
315 case client.PacketType_DIAL_CLS:
316 resp := pkt.GetCloseDial()
317 pendingDial, ok := t.pendingDial.get(resp.Random)
318
319 if !ok {
320 // If the DIAL_CLS does not match a pending dial, it means one of two things:
321 // 1. There was a DIAL_CLS receieved after a DIAL_RSP (unlikely but possible)
322 // 2. grpcTunnel.DialContext() returned early due to a dial timeout or the client canceling the context
323 //
324 // In either scenario, we should return here and close the tunnel as it is no longer needed.
325 klog.V(1).InfoS("DIAL_CLS after dial finished", "dialID", resp.Random)
326 } else {
327 result := dialResult{
328 err: &dialFailure{"dial closed", metrics.DialFailureDialClosed},
329 }
330 select {
331 case pendingDial.resultCh <- result:
332 case <-pendingDial.cancelCh:
333 // Note: this condition can only be hit by a race condition where the
334 // DialContext() returns early (timeout) after the pendingDial is already
335 // fetched here, but before the result is sent.
336 case <-tunnelCtx.Done():
337 }
338 }
339 return // Stop serving & close the tunnel.
340
341 case client.PacketType_DATA:
342 resp := pkt.GetData()
343 if resp.ConnectID == 0 {
344 klog.ErrorS(nil, "Received packet missing ConnectID", "packetType", "DATA")
345 continue
346 }
347 // TODO: flow control
348 conn, ok := t.conns.get(resp.ConnectID)
349
350 if !ok {
351 klog.ErrorS(nil, "Connection not recognized", "connectionID", resp.ConnectID, "packetType", "DATA")
352 t.sendCloseRequest(resp.ConnectID)
353 continue
354 }
355 timer := time.NewTimer((time.Duration)(t.readTimeoutSeconds) * time.Second)
356 select {
357 case conn.readCh <- resp.Data:
358 timer.Stop()
359 case <-timer.C:
360 klog.ErrorS(fmt.Errorf("timeout"), "readTimeout has been reached, the grpc connection to the proxy server will be closed", "connectionID", conn.connID, "readTimeoutSeconds", t.readTimeoutSeconds)
361 return
362 case <-tunnelCtx.Done():
363 klog.V(1).InfoS("Tunnel has been closed, the grpc connection to the proxy server will be closed", "connectionID", conn.connID)
364 }
365
366 case client.PacketType_CLOSE_RSP:
367 resp := pkt.GetCloseResponse()
368 conn, ok := t.conns.get(resp.ConnectID)
369
370 if !ok {
371 klog.V(1).InfoS("Connection not recognized", "connectionID", resp.ConnectID, "packetType", "CLOSE_RSP")
372 continue
373 }
374 close(conn.readCh)
375 conn.closeCh <- resp.Error
376 close(conn.closeCh)
377 t.conns.remove(resp.ConnectID)
378 return
379 }
380 }
381 }
382
383 // Dial connects to the address on the named network, similar to
384 // what net.Dial does. The only supported protocol is tcp.
385 func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) {
386 conn, err := t.dialContext(requestCtx, protocol, address)
387 if err != nil {
388 _, reason := GetDialFailureReason(err)
389 metrics.Metrics.ObserveDialFailure(reason)
390 }
391 return conn, err
392 }
393
394 func (t *grpcTunnel) dialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) {
395 prevStarted := atomic.SwapUint32(&t.started, 1)
396 if prevStarted != 0 {
397 return nil, &dialFailure{"single-use dialer already dialed", metrics.DialFailureAlreadyStarted}
398 }
399
400 select {
401 case <-t.done:
402 return nil, errors.New("tunnel is closed")
403 default: // Tunnel is open, carry on.
404 }
405
406 if protocol != "tcp" {
407 return nil, errors.New("protocol not supported")
408 }
409
410 t.updateMetric(metrics.ClientConnectionStatusDialing)
411
412 random := rand.Int63() /* #nosec G404 */
413
414 // This channel is closed once we're returning and no longer waiting on resultCh
415 cancelCh := make(chan struct{})
416 defer close(cancelCh)
417
418 // This channel MUST NOT be buffered. The sender needs to know when we are not receiving things, so they can abort.
419 resCh := make(chan dialResult)
420
421 t.pendingDial.add(random, pendingDial{resultCh: resCh, cancelCh: cancelCh})
422 defer t.pendingDial.remove(random)
423
424 req := &client.Packet{
425 Type: client.PacketType_DIAL_REQ,
426 Payload: &client.Packet_DialRequest{
427 DialRequest: &client.DialRequest{
428 Protocol: protocol,
429 Address: address,
430 Random: random,
431 },
432 },
433 }
434 klog.V(5).InfoS("[tracing] send packet", "type", req.Type)
435
436 err := t.Send(req)
437 if err != nil {
438 return nil, err
439 }
440
441 klog.V(5).Infoln("DIAL_REQ sent to proxy server")
442
443 c := &conn{
444 tunnel: t,
445 random: random,
446 }
447
448 select {
449 case res := <-resCh:
450 if res.err != nil {
451 return nil, res.err
452 }
453 c.connID = res.connid
454 c.readCh = make(chan []byte, 10)
455 c.closeCh = make(chan string, 1)
456 t.conns.add(res.connid, c)
457 case <-time.After(30 * time.Second):
458 klog.V(5).InfoS("Timed out waiting for DialResp", "dialID", random)
459 go func() {
460 defer t.closeTunnel()
461 t.sendDialClose(random)
462 }()
463 return nil, &dialFailure{"dial timeout, backstop", metrics.DialFailureTimeout}
464 case <-requestCtx.Done():
465 klog.V(5).InfoS("Context canceled waiting for DialResp", "ctxErr", requestCtx.Err(), "dialID", random)
466 go func() {
467 defer t.closeTunnel()
468 t.sendDialClose(random)
469 }()
470 return nil, &dialFailure{"dial timeout, context", metrics.DialFailureContext}
471 case <-t.done:
472 klog.V(5).InfoS("Tunnel closed while waiting for DialResp", "dialID", random)
473 return nil, &dialFailure{"tunnel closed", metrics.DialFailureTunnelClosed}
474 }
475
476 return c, nil
477 }
478
479 func (t *grpcTunnel) Done() <-chan struct{} {
480 return t.done
481 }
482
483 // Send a best-effort DIAL_CLS request for the given dial ID.
484
485 func (t *grpcTunnel) sendCloseRequest(connID int64) error {
486 req := &client.Packet{
487 Type: client.PacketType_CLOSE_REQ,
488 Payload: &client.Packet_CloseRequest{
489 CloseRequest: &client.CloseRequest{
490 ConnectID: connID,
491 },
492 },
493 }
494 klog.V(5).InfoS("[tracing] send req", "type", req.Type)
495 return t.Send(req)
496 }
497
498 func (t *grpcTunnel) sendDialClose(dialID int64) error {
499 req := &client.Packet{
500 Type: client.PacketType_DIAL_CLS,
501 Payload: &client.Packet_CloseDial{
502 CloseDial: &client.CloseDial{
503 Random: dialID,
504 },
505 },
506 }
507 klog.V(5).InfoS("[tracing] send req", "type", req.Type)
508 return t.Send(req)
509 }
510
511 func (t *grpcTunnel) closeTunnel() {
512 atomic.StoreUint32(&t.closing, 1)
513 t.grpcConn.Close()
514 }
515
516 func (t *grpcTunnel) isClosing() bool {
517 return atomic.LoadUint32(&t.closing) != 0
518 }
519
520 func (t *grpcTunnel) Send(pkt *client.Packet) error {
521 t.sendLock.Lock()
522 defer t.sendLock.Unlock()
523
524 const segment = commonmetrics.SegmentFromClient
525 metrics.Metrics.ObservePacket(segment, pkt.Type)
526 err := t.stream.Send(pkt)
527 if err != nil && err != io.EOF {
528 metrics.Metrics.ObserveStreamError(segment, err, pkt.Type)
529 }
530 return err
531 }
532
533 func (t *grpcTunnel) Recv() (*client.Packet, error) {
534 t.recvLock.Lock()
535 defer t.recvLock.Unlock()
536
537 const segment = commonmetrics.SegmentToClient
538 pkt, err := t.stream.Recv()
539 if err != nil {
540 if err != io.EOF {
541 metrics.Metrics.ObserveStreamErrorNoPacket(segment, err)
542 }
543 return nil, err
544 }
545 metrics.Metrics.ObservePacket(segment, pkt.Type)
546 return pkt, nil
547 }
548
549 func GetDialFailureReason(err error) (isDialFailure bool, reason metrics.DialFailureReason) {
550 var df *dialFailure
551 if errors.As(err, &df) {
552 return true, df.reason
553 }
554 return false, metrics.DialFailureUnknown
555 }
556
557 type dialFailure struct {
558 msg string
559 reason metrics.DialFailureReason
560 }
561
562 func (df *dialFailure) Error() string {
563 return df.msg
564 }