Debian Codegraph @

pkg/server/egressselector/egress_selector.go

raw
1 /*
2 Copyright 2019 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package egressselector
18
19 import (
20 "bufio"
21 "context"
22 "crypto/tls"
23 "crypto/x509"
24 "fmt"
25 "net"
26 "net/http"
27 "net/url"
28 "os"
29 "strings"
30 "time"
31
32 "go.opentelemetry.io/otel/attribute"
33 "google.golang.org/grpc"
34 "google.golang.org/grpc/credentials/insecure"
35
36 utilnet "k8s.io/apimachinery/pkg/util/net"
37 "k8s.io/apiserver/pkg/apis/apiserver"
38 egressmetrics "k8s.io/apiserver/pkg/server/egressselector/metrics"
39 "k8s.io/component-base/metrics/legacyregistry"
40 "k8s.io/component-base/tracing"
41 "k8s.io/klog/v2"
42 client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client"
43 )
44
45 var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext
46
47 func init() {
48 client.Metrics.RegisterMetrics(legacyregistry.Registerer())
49 }
50
51 // EgressSelector is the map of network context type to context dialer, for network egress.
52 type EgressSelector struct {
53 egressToDialer map[EgressType]utilnet.DialFunc
54 }
55
56 // EgressType is an indicator of which egress selection should be used for sending traffic.
57 // See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/1281-network-proxy/README.md#network-context
58 type EgressType int
59
60 const (
61 // ControlPlane is the EgressType for traffic intended to go to the control plane.
62 ControlPlane EgressType = iota
63 // Etcd is the EgressType for traffic intended to go to Kubernetes persistence store.
64 Etcd
65 // Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes.
66 Cluster
67 )
68
69 // NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent.
70 type NetworkContext struct {
71 // EgressSelectionName is the unique name of the
72 // EgressSelectorConfiguration which determines
73 // the network we route the traffic to.
74 EgressSelectionName EgressType
75 }
76
77 // Lookup is the interface to get the dialer function for the network context.
78 type Lookup func(networkContext NetworkContext) (utilnet.DialFunc, error)
79
80 // String returns the canonical string representation of the egress type
81 func (s EgressType) String() string {
82 switch s {
83 case ControlPlane:
84 return "controlplane"
85 case Etcd:
86 return "etcd"
87 case Cluster:
88 return "cluster"
89 default:
90 return "invalid"
91 }
92 }
93
94 // AsNetworkContext is a helper function to make it easy to get the basic NetworkContext objects.
95 func (s EgressType) AsNetworkContext() NetworkContext {
96 return NetworkContext{EgressSelectionName: s}
97 }
98
99 func lookupServiceName(name string) (EgressType, error) {
100 switch strings.ToLower(name) {
101 case "controlplane":
102 return ControlPlane, nil
103 case "etcd":
104 return Etcd, nil
105 case "cluster":
106 return Cluster, nil
107 }
108 return -1, fmt.Errorf("unrecognized service name %s", name)
109 }
110
111 func tunnelHTTPConnect(proxyConn net.Conn, proxyAddress, addr string) (net.Conn, error) {
112 fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1")
113 br := bufio.NewReader(proxyConn)
114 res, err := http.ReadResponse(br, nil)
115 if err != nil {
116 proxyConn.Close()
117 return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v",
118 addr, proxyAddress, err)
119 }
120 if res.StatusCode != 200 {
121 proxyConn.Close()
122 return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v",
123 proxyAddress, addr, res.StatusCode, res.Status)
124 }
125
126 // It's safe to discard the bufio.Reader here and return the
127 // original TCP conn directly because we only use this for
128 // TLS, and in TLS the client speaks first, so we know there's
129 // no unbuffered data. But we can double-check.
130 if br.Buffered() > 0 {
131 proxyConn.Close()
132 return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q",
133 br.Buffered(), proxyAddress)
134 }
135 return proxyConn, nil
136 }
137
138 type proxier interface {
139 // proxy returns a connection to addr.
140 proxy(ctx context.Context, addr string) (net.Conn, error)
141 }
142
143 var _ proxier = &httpConnectProxier{}
144
145 type httpConnectProxier struct {
146 conn net.Conn
147 proxyAddress string
148 }
149
150 func (t *httpConnectProxier) proxy(ctx context.Context, addr string) (net.Conn, error) {
151 return tunnelHTTPConnect(t.conn, t.proxyAddress, addr)
152 }
153
154 var _ proxier = &grpcProxier{}
155
156 type grpcProxier struct {
157 tunnel client.Tunnel
158 }
159
160 func (g *grpcProxier) proxy(ctx context.Context, addr string) (net.Conn, error) {
161 return g.tunnel.DialContext(ctx, "tcp", addr)
162 }
163
164 type proxyServerConnector interface {
165 // connect establishes connection to the proxy server, and returns a
166 // proxier based on the connection.
167 //
168 // The provided Context must be non-nil. The context is used for connecting to the proxy only.
169 // If the context expires before the connection is complete, an error is returned.
170 // Once successfully connected to the proxy, any expiration of the context will not affect the connection.
171 connect(context.Context) (proxier, error)
172 }
173
174 type tcpHTTPConnectConnector struct {
175 proxyAddress string
176 tlsConfig *tls.Config
177 }
178
179 func (t *tcpHTTPConnectConnector) connect(ctx context.Context) (proxier, error) {
180 d := tls.Dialer{
181 Config: t.tlsConfig,
182 }
183 conn, err := d.DialContext(ctx, "tcp", t.proxyAddress)
184 if err != nil {
185 return nil, err
186 }
187 return &httpConnectProxier{conn: conn, proxyAddress: t.proxyAddress}, nil
188 }
189
190 type udsHTTPConnectConnector struct {
191 udsName string
192 }
193
194 func (u *udsHTTPConnectConnector) connect(ctx context.Context) (proxier, error) {
195 var d net.Dialer
196 conn, err := d.DialContext(ctx, "unix", u.udsName)
197 if err != nil {
198 return nil, err
199 }
200 return &httpConnectProxier{conn: conn, proxyAddress: u.udsName}, nil
201 }
202
203 type udsGRPCConnector struct {
204 udsName string
205 }
206
207 // connect establishes a connection to a proxy over gRPC.
208 // TODO At the moment, it does not use the provided context.
209 func (u *udsGRPCConnector) connect(_ context.Context) (proxier, error) {
210 udsName := u.udsName
211 dialOption := grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
212 var d net.Dialer
213 c, err := d.DialContext(ctx, "unix", udsName)
214 if err != nil {
215 klog.Errorf("failed to create connection to uds name %s, error: %v", udsName, err)
216 }
217 return c, err
218 })
219
220 // CreateSingleUseGrpcTunnel() unfortunately couples dial and connection contexts. Because of that,
221 // we cannot use ctx just for dialing and control the connection lifetime separately.
222 // See https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/357.
223 tunnelCtx := context.TODO()
224 tunnel, err := client.CreateSingleUseGrpcTunnel(tunnelCtx, udsName, dialOption,
225 grpc.WithBlock(),
226 grpc.WithReturnConnectionError(),
227 grpc.WithTimeout(30*time.Second), // matches http.DefaultTransport dial timeout
228 grpc.WithTransportCredentials(insecure.NewCredentials()))
229 if err != nil {
230 return nil, err
231 }
232 return &grpcProxier{tunnel: tunnel}, nil
233 }
234
235 type dialerCreator struct {
236 connector proxyServerConnector
237 direct bool
238 options metricsOptions
239 }
240
241 type metricsOptions struct {
242 transport string
243 protocol string
244 }
245
246 func (d *dialerCreator) createDialer() utilnet.DialFunc {
247 if d.direct {
248 return directDialer
249 }
250 return func(ctx context.Context, network, addr string) (net.Conn, error) {
251 ctx, span := tracing.Start(ctx, fmt.Sprintf("Proxy via %s protocol over %s", d.options.protocol, d.options.transport), attribute.String("address", addr))
252 defer span.End(500 * time.Millisecond)
253 start := egressmetrics.Metrics.Clock().Now()
254 egressmetrics.Metrics.ObserveDialStart(d.options.protocol, d.options.transport)
255 proxier, err := d.connector.connect(ctx)
256 if err != nil {
257 egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageConnect)
258 return nil, err
259 }
260 conn, err := proxier.proxy(ctx, addr)
261 if err != nil {
262 egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageProxy)
263 return nil, err
264 }
265 egressmetrics.Metrics.ObserveDialLatency(egressmetrics.Metrics.Clock().Now().Sub(start), d.options.protocol, d.options.transport)
266 return conn, nil
267 }
268 }
269
270 func getTLSConfig(t *apiserver.TLSConfig) (*tls.Config, error) {
271 clientCert := t.ClientCert
272 clientKey := t.ClientKey
273 caCert := t.CABundle
274 clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey)
275 if err != nil {
276 return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err)
277 }
278 certPool := x509.NewCertPool()
279 if caCert != "" {
280 certBytes, err := os.ReadFile(caCert)
281 if err != nil {
282 return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err)
283 }
284 ok := certPool.AppendCertsFromPEM(certBytes)
285 if !ok {
286 return nil, fmt.Errorf("failed to append CA cert to the cert pool")
287 }
288 } else {
289 // Use host's root CA set instead of providing our own
290 certPool = nil
291 }
292 return &tls.Config{
293 Certificates: []tls.Certificate{clientCerts},
294 RootCAs: certPool,
295 }, nil
296 }
297
298 func getProxyAddress(urlString string) (string, error) {
299 proxyURL, err := url.Parse(urlString)
300 if err != nil {
301 return "", fmt.Errorf("invalid proxy server url %q: %v", urlString, err)
302 }
303 return proxyURL.Host, nil
304 }
305
306 func connectionToDialerCreator(c apiserver.Connection) (*dialerCreator, error) {
307 switch c.ProxyProtocol {
308
309 case apiserver.ProtocolHTTPConnect:
310 if c.Transport.UDS != nil {
311 return &dialerCreator{
312 connector: &udsHTTPConnectConnector{
313 udsName: c.Transport.UDS.UDSName,
314 },
315 options: metricsOptions{
316 transport: egressmetrics.TransportUDS,
317 protocol: egressmetrics.ProtocolHTTPConnect,
318 },
319 }, nil
320 } else if c.Transport.TCP != nil {
321 tlsConfig, err := getTLSConfig(c.Transport.TCP.TLSConfig)
322 if err != nil {
323 return nil, err
324 }
325 proxyAddress, err := getProxyAddress(c.Transport.TCP.URL)
326 if err != nil {
327 return nil, err
328 }
329 return &dialerCreator{
330 connector: &tcpHTTPConnectConnector{
331 tlsConfig: tlsConfig,
332 proxyAddress: proxyAddress,
333 },
334 options: metricsOptions{
335 transport: egressmetrics.TransportTCP,
336 protocol: egressmetrics.ProtocolHTTPConnect,
337 },
338 }, nil
339 } else {
340 return nil, fmt.Errorf("Either a TCP or UDS transport must be specified")
341 }
342 case apiserver.ProtocolGRPC:
343 if c.Transport.UDS != nil {
344 return &dialerCreator{
345 connector: &udsGRPCConnector{
346 udsName: c.Transport.UDS.UDSName,
347 },
348 options: metricsOptions{
349 transport: egressmetrics.TransportUDS,
350 protocol: egressmetrics.ProtocolGRPC,
351 },
352 }, nil
353 }
354 return nil, fmt.Errorf("UDS transport must be specified for GRPC")
355 case apiserver.ProtocolDirect:
356 return &dialerCreator{direct: true}, nil
357 default:
358 return nil, fmt.Errorf("unrecognized service connection protocol %q", c.ProxyProtocol)
359 }
360
361 }
362
363 // NewEgressSelector configures lookup mechanism for Lookup.
364 // It does so based on a EgressSelectorConfiguration which was read at startup.
365 func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) {
366 if config == nil || config.EgressSelections == nil {
367 // No Connection Services configured, leaving the serviceMap empty, will return default dialer.
368 return nil, nil
369 }
370 cs := &EgressSelector{
371 egressToDialer: make(map[EgressType]utilnet.DialFunc),
372 }
373 for _, service := range config.EgressSelections {
374 name, err := lookupServiceName(service.Name)
375 if err != nil {
376 return nil, err
377 }
378 dialerCreator, err := connectionToDialerCreator(service.Connection)
379 if err != nil {
380 return nil, fmt.Errorf("failed to create dialer for egressSelection %q: %v", name, err)
381 }
382 cs.egressToDialer[name] = dialerCreator.createDialer()
383 }
384 return cs, nil
385 }
386
387 // NewEgressSelectorWithMap returns a EgressSelector with the supplied EgressType to DialFunc map.
388 func NewEgressSelectorWithMap(m map[EgressType]utilnet.DialFunc) *EgressSelector {
389 if m == nil {
390 m = make(map[EgressType]utilnet.DialFunc)
391 }
392 return &EgressSelector{
393 egressToDialer: m,
394 }
395 }
396
397 // Lookup gets the dialer function for the network context.
398 // This is configured for the Kubernetes API Server at startup.
399 func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) {
400 if cs.egressToDialer == nil {
401 // The round trip wrapper will over-ride the dialContext method appropriately
402 return nil, nil
403 }
404
405 return cs.egressToDialer[networkContext.EgressSelectionName], nil
406 }