Debian Codegraph @

internal/rpc/namenode.go

raw
1 package rpc
2
3 import (
4 "context"
5 "errors"
6 "fmt"
7 "net"
8 "sync"
9 "time"
10
11 hadoop "github.com/colinmarc/hdfs/v2/internal/protocol/hadoop_common"
12 hdfs "github.com/colinmarc/hdfs/v2/internal/protocol/hadoop_hdfs"
13 krb "github.com/jcmturner/gokrb5/v8/client"
14 "google.golang.org/protobuf/proto"
15 )
16
17 const (
18 rpcVersion byte = 0x09
19 serviceClass byte = 0x0
20 noneAuthProtocol byte = 0x0
21 saslAuthProtocol byte = 0xdf
22 protocolClass = "org.apache.hadoop.hdfs.protocol.ClientProtocol"
23 protocolClassVersion = 1
24 handshakeCallID = -3
25 standbyExceptionClass = "org.apache.hadoop.ipc.StandbyException"
26 )
27
28 const (
29 backoffDuration = 5 * time.Second
30 leaseRenewInterval = 1 * time.Second
31 )
32
33 // NamenodeConnection represents an open connection to a namenode.
34 type NamenodeConnection struct {
35 ClientID []byte
36 ClientName string
37 User string
38
39 currentRequestID int32
40
41 kerberosClient *krb.Client
42 kerberosServicePrincipleName string
43 kerberosRealm string
44
45 dialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
46 conn net.Conn
47 host *namenodeHost
48 hostList []*namenodeHost
49 transport transport
50
51 reqLock sync.Mutex
52 done chan struct{}
53 }
54
55 // NamenodeConnectionOptions represents the configurable options available
56 // for a NamenodeConnection.
57 type NamenodeConnectionOptions struct {
58 // Addresses specifies the namenode(s) to connect to.
59 Addresses []string
60 // User specifies which HDFS user the client will act as. It is required
61 // unless kerberos authentication is enabled, in which case it is overridden
62 // by the username set in KerberosClient.
63 User string
64 // DialFunc is used to connect to the namenodes. If nil, then
65 // (&net.Dialer{}).DialContext is used.
66 DialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
67 // KerberosClient is used to connect to kerberized HDFS clusters. If provided,
68 // the NamenodeConnection will always mutually athenticate when connecting
69 // to the namenode(s).
70 KerberosClient *krb.Client
71 // KerberosServicePrincipleName specifiesthe Service Principle Name
72 // (<SERVICE>/<FQDN>) for the namenode(s). Like in the
73 // dfs.namenode.kerberos.principal property of core-site.xml, the special
74 // string '_HOST' can be substituted for the hostname in a multi-namenode
75 // setup (for example: 'nn/[email protected]'). It is required if
76 // KerberosClient is provided.
77 KerberosServicePrincipleName string
78 }
79
80 type namenodeHost struct {
81 address string
82 lastError error
83 lastErrorAt time.Time
84 }
85
86 // NewNamenodeConnectionWithOptions creates a new connection to a namenode with
87 // the given options and performs an initial handshake.
88 func NewNamenodeConnection(options NamenodeConnectionOptions) (*NamenodeConnection, error) {
89 // Build the list of hosts to be used for failover.
90 hostList := make([]*namenodeHost, len(options.Addresses))
91 for i, addr := range options.Addresses {
92 hostList[i] = &namenodeHost{address: addr}
93 }
94
95 var user, realm string
96 user = options.User
97 if options.KerberosClient != nil {
98 creds := options.KerberosClient.Credentials
99 user = creds.UserName()
100 realm = creds.Realm()
101 } else if user == "" {
102 return nil, errors.New("user not specified")
103 }
104
105 // The ClientID is reused here both in the RPC headers (which requires a
106 // "globally unique" ID) and as the "client name" in various requests.
107 clientId := newClientID()
108 c := &NamenodeConnection{
109 ClientID: clientId,
110 ClientName: "go-hdfs-" + string(clientId),
111 User: user,
112
113 kerberosClient: options.KerberosClient,
114 kerberosServicePrincipleName: options.KerberosServicePrincipleName,
115 kerberosRealm: realm,
116
117 dialFunc: options.DialFunc,
118 hostList: hostList,
119 transport: &basicTransport{clientID: clientId},
120
121 done: make(chan struct{}),
122 }
123
124 err := c.resolveConnection()
125 if err != nil {
126 return nil, err
127 }
128
129 // Periodically renew any file leases.
130 go c.renewLeases()
131
132 return c, nil
133 }
134
135 func (c *NamenodeConnection) resolveConnection() error {
136 if c.conn != nil {
137 return nil
138 }
139
140 var err error
141 if c.host != nil {
142 err = c.host.lastError
143 }
144
145 for _, host := range c.hostList {
146 if time.Since(host.lastErrorAt) < backoffDuration {
147 continue
148 }
149
150 if c.dialFunc == nil {
151 c.dialFunc = (&net.Dialer{}).DialContext
152 }
153
154 c.host = host
155 c.conn, err = c.dialFunc(context.Background(), "tcp", host.address)
156 if err != nil {
157 c.markFailure(err)
158 continue
159 }
160
161 err = c.doNamenodeHandshake()
162 if err != nil {
163 c.markFailure(err)
164 continue
165 }
166
167 break
168 }
169
170 if c.conn == nil {
171 return fmt.Errorf("no available namenodes: %s", err)
172 }
173
174 return nil
175 }
176
177 func (c *NamenodeConnection) markFailure(err error) {
178 if c.conn != nil {
179 c.conn.Close()
180 c.conn = nil
181 }
182 c.host.lastError = err
183 c.host.lastErrorAt = time.Now()
184 }
185
186 // Execute performs an rpc call. It does this by sending req over the wire and
187 // unmarshaling the result into resp.
188 func (c *NamenodeConnection) Execute(method string, req proto.Message, resp proto.Message) error {
189 c.reqLock.Lock()
190 defer c.reqLock.Unlock()
191
192 c.currentRequestID++
193 requestID := c.currentRequestID
194
195 for {
196 err := c.resolveConnection()
197 if err != nil {
198 return err
199 }
200
201 err = c.transport.writeRequest(c.conn, method, requestID, req)
202 if err != nil {
203 c.markFailure(err)
204 continue
205 }
206
207 err = c.transport.readResponse(c.conn, method, requestID, resp)
208 if err != nil {
209 // Only retry on a standby exception.
210 if nerr, ok := err.(*NamenodeError); ok && nerr.exception == standbyExceptionClass {
211 c.markFailure(err)
212 continue
213 }
214
215 return err
216 }
217
218 break
219 }
220
221 return nil
222 }
223
224 // A handshake packet:
225 // +-----------------------------------------------------------+
226 // | Header, 4 bytes ("hrpc") |
227 // +-----------------------------------------------------------+
228 // | Version, 1 byte (default verion 0x09) |
229 // +-----------------------------------------------------------+
230 // | RPC service class, 1 byte (0x00) |
231 // +-----------------------------------------------------------+
232 // | Auth protocol, 1 byte (Auth method None = 0x00) |
233 // +-----------------------------------------------------------+
234 //
235 // If the auth protocol is something other than 'none', the authentication
236 // handshake happens here. Otherwise, everything can be sent as one packet.
237 //
238 // +-----------------------------------------------------------+
239 // | uint32 length of the next two parts |
240 // +-----------------------------------------------------------+
241 // | varint length + RpcRequestHeaderProto |
242 // +-----------------------------------------------------------+
243 // | varint length + IpcConnectionContextProto |
244 // +-----------------------------------------------------------+
245 func (c *NamenodeConnection) doNamenodeHandshake() error {
246 authProtocol := noneAuthProtocol
247 kerberos := false
248 if c.kerberosClient != nil {
249 authProtocol = saslAuthProtocol
250 kerberos = true
251 }
252
253 rpcHeader := []byte{
254 0x68, 0x72, 0x70, 0x63, // "hrpc"
255 rpcVersion, serviceClass, authProtocol,
256 }
257
258 _, err := c.conn.Write(rpcHeader)
259 if err != nil {
260 return err
261 }
262
263 if kerberos {
264 err = c.doKerberosHandshake()
265 if err != nil {
266 return fmt.Errorf("SASL handshake: %s", err)
267 }
268 }
269
270 rrh := newRPCRequestHeader(handshakeCallID, c.ClientID)
271 cc := newConnectionContext(c.User, c.kerberosRealm)
272 packet, err := makeRPCPacket(rrh, cc)
273 if err != nil {
274 return err
275 }
276
277 _, err = c.conn.Write(packet)
278 return err
279 }
280
281 // renewLeases periodically renews all leases for the connection.
282 func (c *NamenodeConnection) renewLeases() {
283 ticker := time.NewTicker(leaseRenewInterval)
284 defer ticker.Stop()
285
286 for {
287 select {
288 case <-ticker.C:
289 req := &hdfs.RenewLeaseRequestProto{ClientName: proto.String(c.ClientName)}
290 resp := &hdfs.RenewLeaseResponseProto{}
291
292 // Ignore any errors.
293 c.Execute("renewLease", req, resp)
294 case <-c.done:
295 return
296 }
297 }
298 }
299
300 // Close terminates all underlying socket connections to remote server.
301 func (c *NamenodeConnection) Close() error {
302 close(c.done)
303
304 // Ensure that we're not concurrently renewing leases.
305 c.reqLock.Lock()
306 defer c.reqLock.Unlock()
307
308 if c.conn != nil {
309 return c.conn.Close()
310 }
311
312 return nil
313 }
314
315 func newRPCRequestHeader(id int32, clientID []byte) *hadoop.RpcRequestHeaderProto {
316 return &hadoop.RpcRequestHeaderProto{
317 RpcKind: hadoop.RpcKindProto_RPC_PROTOCOL_BUFFER.Enum(),
318 RpcOp: hadoop.RpcRequestHeaderProto_RPC_FINAL_PACKET.Enum(),
319 CallId: proto.Int32(id),
320 ClientId: clientID,
321 }
322 }
323
324 func newRequestHeader(methodName string) *hadoop.RequestHeaderProto {
325 return &hadoop.RequestHeaderProto{
326 MethodName: proto.String(methodName),
327 DeclaringClassProtocolName: proto.String(protocolClass),
328 ClientProtocolVersion: proto.Uint64(uint64(protocolClassVersion)),
329 }
330 }
331
332 func newConnectionContext(user, kerberosRealm string) *hadoop.IpcConnectionContextProto {
333 if kerberosRealm != "" {
334 user = user + "@" + kerberosRealm
335 }
336
337 return &hadoop.IpcConnectionContextProto{
338 UserInfo: &hadoop.UserInformationProto{
339 EffectiveUser: proto.String(user),
340 },
341 Protocol: proto.String(protocolClass),
342 }
343 }