Debian Codegraph @

ibacm/prov/acmp/src/acmp.c

raw
1 /*
2 * Copyright (c) 2009-2014 Intel Corporation. All rights reserved.
3 * Copyright (c) 2013 Mellanox Technologies LTD. All rights reserved.
4 *
5 * This software is available to you under the OpenIB.org BSD license
6 * below:
7 *
8 * Redistribution and use in source and binary forms, with or
9 * without modification, are permitted provided that the following
10 * conditions are met:
11 *
12 * - Redistributions of source code must retain the above
13 * copyright notice, this list of conditions and the following
14 * disclaimer.
15 *
16 * - Redistributions in binary form must reproduce the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer in the documentation and/or other materials
19 * provided with the distribution.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
23 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
24 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
25 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
26 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
27 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
28 * SOFTWARE.
29 */
30
31 #include <config.h>
32
33 #include <endian.h>
34 #include <stdio.h>
35 #include <stdarg.h>
36 #include <string.h>
37 #include <osd.h>
38 #include <arpa/inet.h>
39 #include <sys/types.h>
40 #include <sys/time.h>
41 #include <fcntl.h>
42 #include <dirent.h>
43 #include <infiniband/acm.h>
44 #include <infiniband/acm_prov.h>
45 #include <infiniband/umad.h>
46 #include <infiniband/verbs.h>
47 #include <infiniband/umad_sa.h>
48 #include <infiniband/umad_sa_mcm.h>
49 #include <ifaddrs.h>
50 #include <dlfcn.h>
51 #include <search.h>
52 #include <netdb.h>
53 #include <net/if.h>
54 #include <sys/ioctl.h>
55 #include <net/if_arp.h>
56 #include <netinet/in.h>
57 #include <linux/netlink.h>
58 #include <linux/rtnetlink.h>
59 #include <inttypes.h>
60 #include <ccan/list.h>
61 #include "acm_util.h"
62 #include "acm_mad.h"
63
64 #define IB_LID_MCAST_START 0xc000
65
66 #define MAX_EP_ADDR 4
67 #define MAX_EP_MC 2
68
69 enum acmp_state {
70 ACMP_INIT,
71 ACMP_QUERY_ADDR,
72 ACMP_ADDR_RESOLVED,
73 ACMP_QUERY_ROUTE,
74 ACMP_READY
75 };
76
77 enum acmp_addr_prot {
78 ACMP_ADDR_PROT_ACM
79 };
80
81 enum acmp_route_prot {
82 ACMP_ROUTE_PROT_ACM,
83 ACMP_ROUTE_PROT_SA
84 };
85
86 enum acmp_loopback_prot {
87 ACMP_LOOPBACK_PROT_NONE,
88 ACMP_LOOPBACK_PROT_LOCAL
89 };
90
91 enum acmp_route_preload {
92 ACMP_ROUTE_PRELOAD_NONE,
93 ACMP_ROUTE_PRELOAD_OSM_FULL_V1
94 };
95
96 enum acmp_addr_preload {
97 ACMP_ADDR_PRELOAD_NONE,
98 ACMP_ADDR_PRELOAD_HOSTS
99 };
100
101 /*
102 * Nested locking order: dest -> ep, dest -> port
103 */
104 struct acmp_ep;
105
106 struct acmp_dest {
107 uint8_t address[ACM_MAX_ADDRESS]; /* keep first */
108 char name[ACM_MAX_ADDRESS];
109 struct ibv_ah *ah;
110 struct ibv_ah_attr av;
111 struct ibv_path_record path;
112 union ibv_gid mgid;
113 __be64 req_id;
114 struct list_head req_queue;
115 uint32_t remote_qpn;
116 pthread_mutex_t lock;
117 enum acmp_state state;
118 atomic_t refcnt;
119 uint64_t addr_timeout;
120 uint64_t route_timeout;
121 uint8_t addr_type;
122 struct acmp_ep *ep;
123 };
124
125 struct acmp_device;
126
127 struct acmp_port {
128 struct acmp_device *dev;
129 const struct acm_port *port;
130 struct list_head ep_list;
131 pthread_mutex_t lock;
132 struct acmp_dest sa_dest;
133 enum ibv_port_state state;
134 enum ibv_mtu mtu;
135 enum ibv_rate rate;
136 int subnet_timeout;
137 uint16_t default_pkey_ix;
138 uint16_t lid;
139 uint16_t lid_mask;
140 uint8_t port_num;
141 };
142
143 struct acmp_device {
144 struct ibv_context *verbs;
145 const struct acm_device *device;
146 struct ibv_comp_channel *channel;
147 struct ibv_pd *pd;
148 __be64 guid;
149 struct list_node entry;
150 pthread_t comp_thread_id;
151 int port_cnt;
152 struct acmp_port port[0];
153 };
154
155 /* Maintain separate virtual send queues to avoid deadlock */
156 struct acmp_send_queue {
157 int credits;
158 struct list_head pending;
159 };
160
161 struct acmp_addr {
162 uint16_t type;
163 union acm_ep_info info;
164 struct acm_address addr;
165 struct acmp_ep *ep;
166 };
167
168 struct acmp_addr_ctx {
169 struct acmp_ep *ep;
170 int addr_inx;
171 };
172
173 struct acmp_ep {
174 struct acmp_port *port;
175 struct ibv_cq *cq;
176 struct ibv_qp *qp;
177 struct ibv_mr *mr;
178 uint8_t *recv_bufs;
179 struct list_node entry;
180 char id_string[IBV_SYSFS_NAME_MAX + 11];
181 void *dest_map[ACM_ADDRESS_RESERVED - 1];
182 struct acmp_dest mc_dest[MAX_EP_MC];
183 int mc_cnt;
184 uint16_t pkey_index;
185 uint16_t pkey;
186 const struct acm_endpoint *endpoint;
187 pthread_mutex_t lock;
188 struct acmp_send_queue resolve_queue;
189 struct acmp_send_queue resp_queue;
190 struct list_head active_queue;
191 struct list_head wait_queue;
192 enum acmp_state state;
193 /* This lock protects nmbr_ep_addrs and addr_info */
194 pthread_rwlock_t rwlock;
195 int nmbr_ep_addrs;
196 struct acmp_addr *addr_info;
197 atomic_t counters[ACM_MAX_COUNTER];
198 };
199
200 struct acmp_send_msg {
201 struct list_node entry;
202 struct acmp_ep *ep;
203 struct acmp_dest *dest;
204 struct ibv_ah *ah;
205 void *context;
206 void (*resp_handler)(struct acmp_send_msg *req,
207 struct ibv_wc *wc, struct acm_mad *resp);
208 struct acmp_send_queue *req_queue;
209 struct ibv_mr *mr;
210 struct ibv_send_wr wr;
211 struct ibv_sge sge;
212 uint64_t expires;
213 int tries;
214 uint8_t data[ACM_SEND_SIZE];
215 };
216
217 struct acmp_request {
218 uint64_t id;
219 struct list_node entry;
220 struct acm_msg msg;
221 struct acmp_ep *ep;
222 };
223
224 static int acmp_open_dev(const struct acm_device *device, void **dev_context);
225 static void acmp_close_dev(void *dev_context);
226 static int acmp_open_port(const struct acm_port *port, void *dev_context,
227 void **port_context);
228 static void acmp_close_port(void *port_context);
229 static int acmp_open_endpoint(const struct acm_endpoint *endpoint,
230 void *port_context, void **ep_context);
231 static void acmp_close_endpoint(void *ep_context);
232 static int acmp_add_addr(const struct acm_address *addr, void *ep_context,
233 void **addr_context);
234 static void acmp_remove_addr(void *addr_context);
235 static int acmp_resolve(void *addr_context, struct acm_msg *msg, uint64_t id);
236 static int acmp_query(void *addr_context, struct acm_msg *msg, uint64_t id);
237 static int acmp_handle_event(void *port_context, enum ibv_event_type type);
238 static void acmp_query_perf(void *ep_context, uint64_t *values, uint8_t *cnt);
239
240 static struct acm_provider def_prov = {
241 .size = sizeof(struct acm_provider),
242 .version = ACM_PROV_VERSION,
243 .name = "ibacmp",
244 .open_device = acmp_open_dev,
245 .close_device = acmp_close_dev,
246 .open_port = acmp_open_port,
247 .close_port = acmp_close_port,
248 .open_endpoint = acmp_open_endpoint,
249 .close_endpoint = acmp_close_endpoint,
250 .add_address = acmp_add_addr,
251 .remove_address = acmp_remove_addr,
252 .resolve = acmp_resolve,
253 .query = acmp_query,
254 .handle_event = acmp_handle_event,
255 .query_perf = acmp_query_perf,
256 };
257
258 static LIST_HEAD(acmp_dev_list);
259 static pthread_mutex_t acmp_dev_lock;
260
261 static atomic_t g_tid;
262 static LIST_HEAD(timeout_list);
263 static event_t timeout_event;
264 static atomic_t wait_cnt;
265 static pthread_t retry_thread_id;
266 static int retry_thread_started = 0;
267
268 static __thread char log_data[ACM_MAX_ADDRESS];
269
270 /*
271 * Service options - may be set through ibacm_opts.cfg file.
272 */
273 static char route_data_file[128] = ACM_CONF_DIR "/ibacm_route.data";
274 static char addr_data_file[128] = ACM_CONF_DIR "/ibacm_hosts.data";
275 static enum acmp_addr_prot addr_prot = ACMP_ADDR_PROT_ACM;
276 static int addr_timeout = 1440;
277 static enum acmp_route_prot route_prot = ACMP_ROUTE_PROT_SA;
278 static int route_timeout = -1;
279 static enum acmp_loopback_prot loopback_prot = ACMP_LOOPBACK_PROT_LOCAL;
280 static int timeout = 2000;
281 static int retries = 2;
282 static int resolve_depth = 1;
283 static int send_depth = 1;
284 static int recv_depth = 1024;
285 static uint8_t min_mtu = IBV_MTU_2048;
286 static uint8_t min_rate = IBV_RATE_10_GBPS;
287 static enum acmp_route_preload route_preload;
288 static enum acmp_addr_preload addr_preload;
289
290 static int acmp_initialized = 0;
291
292 static int acmp_compare_dest(const void *dest1, const void *dest2)
293 {
294 return memcmp(dest1, dest2, ACM_MAX_ADDRESS);
295 }
296
297 static void
298 acmp_set_dest_addr(struct acmp_dest *dest, uint8_t addr_type,
299 const uint8_t *addr, size_t size)
300 {
301 memcpy(dest->address, addr, size);
302 dest->addr_type = addr_type;
303 acm_format_name(0, dest->name, sizeof dest->name, addr_type, addr, size);
304 }
305
306 static void
307 acmp_init_dest(struct acmp_dest *dest, uint8_t addr_type,
308 const uint8_t *addr, size_t size)
309 {
310 list_head_init(&dest->req_queue);
311 atomic_init(&dest->refcnt);
312 atomic_set(&dest->refcnt, 1);
313 pthread_mutex_init(&dest->lock, NULL);
314 if (size)
315 acmp_set_dest_addr(dest, addr_type, addr, size);
316 dest->state = ACMP_INIT;
317 }
318
319 static struct acmp_dest *
320 acmp_alloc_dest(uint8_t addr_type, const uint8_t *addr)
321 {
322 struct acmp_dest *dest;
323
324 dest = calloc(1, sizeof *dest);
325 if (!dest) {
326 acm_log(0, "ERROR - unable to allocate dest\n");
327 return NULL;
328 }
329
330 acmp_init_dest(dest, addr_type, addr, ACM_MAX_ADDRESS);
331 acm_log(1, "%s\n", dest->name);
332 return dest;
333 }
334
335 /* Caller must hold ep lock. */
336 static struct acmp_dest *
337 acmp_get_dest(struct acmp_ep *ep, uint8_t addr_type, const uint8_t *addr)
338 {
339 struct acmp_dest *dest, **tdest;
340
341 tdest = tfind(addr, &ep->dest_map[addr_type - 1], acmp_compare_dest);
342 if (tdest) {
343 dest = *tdest;
344 (void) atomic_inc(&dest->refcnt);
345 acm_log(2, "%s\n", dest->name);
346 } else {
347 dest = NULL;
348 acm_format_name(2, log_data, sizeof log_data,
349 addr_type, addr, ACM_MAX_ADDRESS);
350 acm_log(2, "%s not found\n", log_data);
351 }
352 return dest;
353 }
354
355 static void
356 acmp_put_dest(struct acmp_dest *dest)
357 {
358 acm_log(2, "%s\n", dest->name);
359 if (atomic_dec(&dest->refcnt) == 0) {
360 free(dest);
361 }
362 }
363
364 /* Caller must hold ep lock. */
365 static void
366 acmp_remove_dest(struct acmp_ep *ep, struct acmp_dest *dest)
367 {
368 acm_log(2, "%s\n", dest->name);
369 if (!tdelete(dest->address, &ep->dest_map[dest->addr_type - 1],
370 acmp_compare_dest))
371 acm_log(0, "ERROR: %s not found!!\n", dest->name);
372
373 acmp_put_dest(dest);
374 }
375
376 static struct acmp_dest *
377 acmp_acquire_dest(struct acmp_ep *ep, uint8_t addr_type, const uint8_t *addr)
378 {
379 struct acmp_dest *dest;
380 int64_t rec_expr_minutes;
381
382 acm_format_name(2, log_data, sizeof log_data,
383 addr_type, addr, ACM_MAX_ADDRESS);
384 acm_log(2, "%s\n", log_data);
385 pthread_mutex_lock(&ep->lock);
386 dest = acmp_get_dest(ep, addr_type, addr);
387 if (dest && dest->state == ACMP_READY &&
388 dest->addr_timeout != (uint64_t)~0ULL) {
389 rec_expr_minutes = dest->addr_timeout - time_stamp_min();
390 if (rec_expr_minutes <= 0) {
391 acm_log(2, "Record expired\n");
392 acmp_remove_dest(ep, dest);
393 dest = NULL;
394 } else {
395 acm_log(2, "Record valid for the next %" PRId64 " minute(s)\n",
396 rec_expr_minutes);
397 }
398 }
399 if (!dest) {
400 dest = acmp_alloc_dest(addr_type, addr);
401 if (dest) {
402 dest->ep = ep;
403 tsearch(dest, &ep->dest_map[addr_type - 1], acmp_compare_dest);
404 (void) atomic_inc(&dest->refcnt);
405 }
406 }
407 pthread_mutex_unlock(&ep->lock);
408 return dest;
409 }
410
411 static struct acmp_request *acmp_alloc_req(uint64_t id, struct acm_msg *msg)
412 {
413 struct acmp_request *req;
414
415 req = calloc(1, sizeof *req);
416 if (!req) {
417 acm_log(0, "ERROR - unable to alloc client request\n");
418 return NULL;
419 }
420
421 req->id = id;
422 memcpy(&req->msg, msg, sizeof(req->msg));
423 acm_log(2, "id %" PRIu64 ", req %p\n", id, req);
424 return req;
425 }
426
427 static void acmp_free_req(struct acmp_request *req)
428 {
429 acm_log(2, "%p\n", req);
430 free(req);
431 }
432
433 static struct acmp_send_msg *
434 acmp_alloc_send(struct acmp_ep *ep, struct acmp_dest *dest, size_t size)
435 {
436 struct acmp_send_msg *msg;
437
438 msg = (struct acmp_send_msg *) calloc(1, sizeof *msg);
439 if (!msg) {
440 acm_log(0, "ERROR - unable to allocate send buffer\n");
441 return NULL;
442 }
443
444 msg->ep = ep;
445 msg->mr = ibv_reg_mr(ep->port->dev->pd, msg->data, size, 0);
446 if (!msg->mr) {
447 acm_log(0, "ERROR - failed to register send buffer\n");
448 goto err1;
449 }
450
451 if (!dest->ah) {
452 msg->ah = ibv_create_ah(ep->port->dev->pd, &dest->av);
453 if (!msg->ah) {
454 acm_log(0, "ERROR - unable to create ah\n");
455 goto err2;
456 }
457 msg->wr.wr.ud.ah = msg->ah;
458 } else {
459 msg->wr.wr.ud.ah = dest->ah;
460 }
461
462 acm_log(2, "get dest %s\n", dest->name);
463 (void) atomic_inc(&dest->refcnt);
464 msg->dest = dest;
465
466 msg->wr.next = NULL;
467 msg->wr.sg_list = &msg->sge;
468 msg->wr.num_sge = 1;
469 msg->wr.opcode = IBV_WR_SEND;
470 msg->wr.send_flags = IBV_SEND_SIGNALED;
471 msg->wr.wr_id = (uintptr_t) msg;
472 msg->wr.wr.ud.remote_qpn = dest->remote_qpn;
473 msg->wr.wr.ud.remote_qkey = ACM_QKEY;
474
475 msg->sge.length = size;
476 msg->sge.lkey = msg->mr->lkey;
477 msg->sge.addr = (uintptr_t) msg->data;
478 acm_log(2, "%p\n", msg);
479 return msg;
480
481 err2:
482 ibv_dereg_mr(msg->mr);
483 err1:
484 free(msg);
485 return NULL;
486 }
487
488 static void
489 acmp_init_send_req(struct acmp_send_msg *msg, void *context,
490 void (*resp_handler)(struct acmp_send_msg *req,
491 struct ibv_wc *wc, struct acm_mad *resp))
492 {
493 acm_log(2, "%p\n", msg);
494 msg->tries = retries + 1;
495 msg->context = context;
496 msg->resp_handler = resp_handler;
497 }
498
499 static void acmp_free_send(struct acmp_send_msg *msg)
500 {
501 acm_log(2, "%p\n", msg);
502 if (msg->ah)
503 ibv_destroy_ah(msg->ah);
504 ibv_dereg_mr(msg->mr);
505 acmp_put_dest(msg->dest);
506 free(msg);
507 }
508
509 static void acmp_post_send(struct acmp_send_queue *queue, struct acmp_send_msg *msg)
510 {
511 struct acmp_ep *ep = msg->ep;
512 struct ibv_send_wr *bad_wr;
513
514 msg->req_queue = queue;
515 pthread_mutex_lock(&ep->lock);
516 if (queue->credits) {
517 acm_log(2, "posting send to QP\n");
518 queue->credits--;
519 list_add_tail(&ep->active_queue, &msg->entry);
520 ibv_post_send(ep->qp, &msg->wr, &bad_wr);
521 } else {
522 acm_log(2, "no sends available, queuing message\n");
523 list_add_tail(&queue->pending, &msg->entry);
524 }
525 pthread_mutex_unlock(&ep->lock);
526 }
527
528 static void acmp_post_recv(struct acmp_ep *ep, uint64_t address)
529 {
530 struct ibv_recv_wr wr, *bad_wr;
531 struct ibv_sge sge;
532
533 wr.next = NULL;
534 wr.sg_list = &sge;
535 wr.num_sge = 1;
536 wr.wr_id = address;
537
538 sge.length = ACM_RECV_SIZE;
539 sge.lkey = ep->mr->lkey;
540 sge.addr = address;
541
542 ibv_post_recv(ep->qp, &wr, &bad_wr);
543 }
544
545 /* Caller must hold ep lock */
546 static void acmp_send_available(struct acmp_ep *ep, struct acmp_send_queue *queue)
547 {
548 struct acmp_send_msg *msg;
549 struct ibv_send_wr *bad_wr;
550
551 msg = list_pop(&queue->pending, struct acmp_send_msg, entry);
552 if (msg) {
553 acm_log(2, "posting queued send message\n");
554 list_add_tail(&ep->active_queue, &msg->entry);
555 ibv_post_send(ep->qp, &msg->wr, &bad_wr);
556 } else {
557 queue->credits++;
558 }
559 }
560
561 static void acmp_complete_send(struct acmp_send_msg *msg)
562 {
563 struct acmp_ep *ep = msg->ep;
564
565 pthread_mutex_lock(&ep->lock);
566 list_del(&msg->entry);
567 if (msg->tries) {
568 acm_log(2, "waiting for response\n");
569 msg->expires = time_stamp_ms() + ep->port->subnet_timeout + timeout;
570 list_add_tail(&ep->wait_queue, &msg->entry);
571 if (atomic_inc(&wait_cnt) == 1)
572 event_signal(&timeout_event);
573 } else {
574 acm_log(2, "freeing\n");
575 acmp_send_available(ep, msg->req_queue);
576 acmp_free_send(msg);
577 }
578 pthread_mutex_unlock(&ep->lock);
579 }
580
581 static struct acmp_send_msg *acmp_get_request(struct acmp_ep *ep, __be64 tid, int *free)
582 {
583 struct acmp_send_msg *msg, *next, *req = NULL;
584 struct acm_mad *mad;
585
586 acm_log(2, "\n");
587 pthread_mutex_lock(&ep->lock);
588 list_for_each_safe(&ep->wait_queue, msg, next, entry) {
589 mad = (struct acm_mad *) msg->data;
590 if (mad->tid == tid) {
591 acm_log(2, "match found in wait queue\n");
592 req = msg;
593 list_del(&msg->entry);
594 (void) atomic_dec(&wait_cnt);
595 acmp_send_available(ep, msg->req_queue);
596 *free = 1;
597 goto unlock;
598 }
599 }
600
601 list_for_each(&ep->active_queue, msg, entry) {
602 mad = (struct acm_mad *) msg->data;
603 if (mad->tid == tid && msg->tries) {
604 acm_log(2, "match found in active queue\n");
605 req = msg;
606 req->tries = 0;
607 *free = 0;
608 break;
609 }
610 }
611 unlock:
612 pthread_mutex_unlock(&ep->lock);
613 return req;
614 }
615
616 static int acmp_mc_index(struct acmp_ep *ep, union ibv_gid *gid)
617 {
618 int i;
619
620 for (i = 0; i < ep->mc_cnt; i++) {
621 if (!memcmp(&ep->mc_dest[i].address, gid, sizeof(*gid)))
622 return i;
623 }
624 return -1;
625 }
626
627 /* Multicast groups are ordered lowest to highest preference. */
628 static int acmp_best_mc_index(struct acmp_ep *ep, struct acm_resolve_rec *rec)
629 {
630 int i, index;
631
632 for (i = min_t(int, rec->gid_cnt, ACM_MAX_GID_COUNT) - 1; i >= 0; i--) {
633 index = acmp_mc_index(ep, &rec->gid[i]);
634 if (index >= 0) {
635 return index;
636 }
637 }
638 return -1;
639 }
640
641 static void
642 acmp_record_mc_av(struct acmp_port *port, struct ib_mc_member_rec *mc_rec,
643 struct acmp_dest *dest)
644 {
645 uint32_t sl_flow_hop;
646
647 sl_flow_hop = be32toh(mc_rec->sl_flow_hop);
648
649 dest->av.dlid = be16toh(mc_rec->mlid);
650 dest->av.sl = (uint8_t) (sl_flow_hop >> 28);
651 dest->av.src_path_bits = port->sa_dest.av.src_path_bits;
652 dest->av.static_rate = mc_rec->rate & 0x3F;
653 dest->av.port_num = port->port_num;
654
655 dest->av.is_global = 1;
656 dest->av.grh.dgid = mc_rec->mgid;
657 dest->av.grh.flow_label = (sl_flow_hop >> 8) & 0xFFFFF;
658 dest->av.grh.sgid_index = acm_gid_index((struct acm_port *) port->port,
659 &mc_rec->port_gid);
660 dest->av.grh.hop_limit = (uint8_t) sl_flow_hop;
661 dest->av.grh.traffic_class = mc_rec->tclass;
662
663 dest->path.dgid = mc_rec->mgid;
664 dest->path.sgid = mc_rec->port_gid;
665 dest->path.dlid = mc_rec->mlid;
666 dest->path.slid = htobe16(port->lid | port->sa_dest.av.src_path_bits);
667 dest->path.flowlabel_hoplimit = htobe32(sl_flow_hop & 0xFFFFFFF);
668 dest->path.tclass = mc_rec->tclass;
669 dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE | 1;
670 dest->path.pkey = mc_rec->pkey;
671 dest->path.qosclass_sl = htobe16((uint16_t) (sl_flow_hop >> 28));
672 dest->path.mtu = mc_rec->mtu;
673 dest->path.rate = mc_rec->rate;
674 dest->path.packetlifetime = mc_rec->packet_lifetime;
675 }
676
677 /* Always send the GRH to transfer GID data to remote side */
678 static void
679 acmp_init_path_av(struct acmp_port *port, struct acmp_dest *dest)
680 {
681 uint32_t flow_hop;
682
683 dest->av.dlid = be16toh(dest->path.dlid);
684 dest->av.sl = be16toh(dest->path.qosclass_sl) & 0xF;
685 dest->av.src_path_bits = be16toh(dest->path.slid) & 0x7F;
686 dest->av.static_rate = dest->path.rate & 0x3F;
687 dest->av.port_num = port->port_num;
688
689 flow_hop = be32toh(dest->path.flowlabel_hoplimit);
690 dest->av.is_global = 1;
691 dest->av.grh.flow_label = (flow_hop >> 8) & 0xFFFFF;
692 pthread_mutex_lock(&port->lock);
693 if (port->port)
694 dest->av.grh.sgid_index = acm_gid_index(
695 (struct acm_port *) port->port, &dest->path.sgid);
696 else
697 dest->av.grh.sgid_index = 0;
698 pthread_mutex_unlock(&port->lock);
699 dest->av.grh.hop_limit = (uint8_t) flow_hop;
700 dest->av.grh.traffic_class = dest->path.tclass;
701 }
702
703 static void acmp_process_join_resp(struct acm_sa_mad *sa_mad)
704 {
705 struct acmp_dest *dest;
706 struct ib_mc_member_rec *mc_rec;
707 struct ib_sa_mad *mad;
708 int index, ret;
709 struct acmp_ep *ep = sa_mad->context;
710
711 mad = (struct ib_sa_mad *) &sa_mad->sa_mad;
712 acm_log(1, "response status: 0x%x, mad status: 0x%x\n",
713 sa_mad->umad.status, mad->status);
714 pthread_mutex_lock(&ep->lock);
715 if (sa_mad->umad.status) {
716 acm_log(0, "ERROR - send join failed 0x%x\n", sa_mad->umad.status);
717 goto out;
718 }
719 if (mad->status) {
720 acm_log(0, "ERROR - join response status 0x%x\n", mad->status);
721 goto out;
722 }
723
724 mc_rec = (struct ib_mc_member_rec *) mad->data;
725 index = acmp_mc_index(ep, &mc_rec->mgid);
726 if (index < 0) {
727 acm_log(0, "ERROR - MGID in join response not found\n");
728 goto out;
729 }
730
731 dest = &ep->mc_dest[index];
732 dest->remote_qpn = IB_MC_QPN;
733 dest->mgid = mc_rec->mgid;
734 acmp_record_mc_av(ep->port, mc_rec, dest);
735
736 if (index == 0) {
737 dest->ah = ibv_create_ah(ep->port->dev->pd, &dest->av);
738 if (!dest->ah) {
739 acm_log(0, "ERROR - unable to create ah\n");
740 goto out;
741 }
742 ret = ibv_attach_mcast(ep->qp, &dest->mgid, dest->av.dlid);
743 if (ret) {
744 acm_log(0, "ERROR - unable to attach QP to multicast group\n");
745 ibv_destroy_ah(dest->ah);
746 dest->ah = NULL;
747 goto out;
748 }
749 ep->state = ACMP_READY;
750 }
751
752 atomic_set(&dest->refcnt, 1);
753 dest->state = ACMP_READY;
754 acm_log(1, "join successful\n");
755 out:
756 acm_free_sa_mad(sa_mad);
757 pthread_mutex_unlock(&ep->lock);
758 }
759
760 static uint8_t
761 acmp_record_acm_route(struct acmp_ep *ep, struct acmp_dest *dest)
762 {
763 int i;
764
765 acm_log(2, "\n");
766 for (i = 0; i < MAX_EP_MC; i++) {
767 if (!memcmp(&dest->mgid, &ep->mc_dest[i].mgid, sizeof dest->mgid))
768 break;
769 }
770 if (i == MAX_EP_MC) {
771 acm_log(0, "ERROR - cannot match mgid\n");
772 return ACM_STATUS_EINVAL;
773 }
774
775 dest->path = ep->mc_dest[i].path;
776 dest->path.dgid = dest->av.grh.dgid;
777 dest->path.dlid = htobe16(dest->av.dlid);
778 dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
779 dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
780 dest->state = ACMP_READY;
781 return ACM_STATUS_SUCCESS;
782 }
783
784 static void acmp_init_path_query(struct ib_sa_mad *mad)
785 {
786 acm_log(2, "\n");
787 mad->base_version = 1;
788 mad->mgmt_class = IB_MGMT_CLASS_SA;
789 mad->class_version = 2;
790 mad->method = IB_METHOD_GET;
791 mad->tid = htobe64((uint64_t) atomic_inc(&g_tid));
792 mad->attr_id = IB_SA_ATTR_PATH_REC;
793 }
794
795 /* Caller must hold dest lock */
796 static uint8_t acmp_resolve_path_sa(struct acmp_ep *ep, struct acmp_dest *dest,
797 void (*handler)(struct acm_sa_mad *))
798 {
799 struct ib_sa_mad *mad;
800 uint8_t ret;
801 struct acm_sa_mad *sa_mad;
802
803 acm_log(2, "%s\n", dest->name);
804
805 sa_mad = acm_alloc_sa_mad(ep->endpoint, dest, handler);
806 if (!sa_mad) {
807 acm_log(0, "Error - failed to allocate sa_mad\n");
808 ret = ACM_STATUS_ENOMEM;
809 goto err;
810 }
811
812 mad = (struct ib_sa_mad *) &sa_mad->sa_mad;
813 acmp_init_path_query(mad);
814
815 memcpy(mad->data, &dest->path, sizeof(dest->path));
816 mad->comp_mask = acm_path_comp_mask(&dest->path);
817
818 acm_increment_counter(ACM_CNTR_ROUTE_QUERY);
819 atomic_inc(&ep->counters[ACM_CNTR_ROUTE_QUERY]);
820 dest->state = ACMP_QUERY_ROUTE;
821 if (acm_send_sa_mad(sa_mad)) {
822 acm_log(0, "Error - Failed to send sa mad\n");
823 ret = ACM_STATUS_ENODATA;
824 goto free_mad;
825 }
826 return ACM_STATUS_SUCCESS;
827 free_mad:
828 acm_free_sa_mad(sa_mad);
829 err:
830 dest->state = ACMP_INIT;
831 return ret;
832 }
833
834 static uint8_t
835 acmp_record_acm_addr(struct acmp_ep *ep, struct acmp_dest *dest, struct ibv_wc *wc,
836 struct acm_resolve_rec *rec)
837 {
838 int index;
839
840 acm_log(2, "%s\n", dest->name);
841 index = acmp_best_mc_index(ep, rec);
842 if (index < 0) {
843 acm_log(0, "ERROR - no shared multicast groups\n");
844 dest->state = ACMP_INIT;
845 return ACM_STATUS_ENODATA;
846 }
847
848 acm_log(2, "selecting MC group at index %d\n", index);
849 dest->av = ep->mc_dest[index].av;
850 dest->av.dlid = wc->slid;
851 dest->av.src_path_bits = wc->dlid_path_bits;
852 dest->av.grh.dgid = ((struct ibv_grh *) (uintptr_t) wc->wr_id)->sgid;
853
854 dest->mgid = ep->mc_dest[index].mgid;
855 dest->path.sgid = ep->mc_dest[index].path.sgid;
856 dest->path.dgid = dest->av.grh.dgid;
857 dest->path.tclass = ep->mc_dest[index].path.tclass;
858 dest->path.pkey = ep->mc_dest[index].path.pkey;
859 dest->remote_qpn = wc->src_qp;
860
861 dest->state = ACMP_ADDR_RESOLVED;
862 return ACM_STATUS_SUCCESS;
863 }
864
865 static void
866 acmp_record_path_addr(struct acmp_ep *ep, struct acmp_dest *dest,
867 struct ibv_path_record *path)
868 {
869 acm_log(2, "%s\n", dest->name);
870 dest->path.pkey = htobe16(ep->pkey);
871 dest->path.dgid = path->dgid;
872 if (path->slid) {
873 dest->path.slid = path->slid;
874 } else {
875 dest->path.slid = htobe16(ep->port->lid);
876 }
877 if (!ib_any_gid(&path->sgid)) {
878 dest->path.sgid = path->sgid;
879 } else {
880 dest->path.sgid = ep->mc_dest[0].path.sgid;
881 }
882 dest->path.dlid = path->dlid;
883 dest->state = ACMP_ADDR_RESOLVED;
884 }
885
886 static uint8_t acmp_validate_addr_req(struct acm_mad *mad)
887 {
888 struct acm_resolve_rec *rec;
889
890 if (mad->method != IB_METHOD_GET) {
891 acm_log(0, "ERROR - invalid method 0x%x\n", mad->method);
892 return ACM_STATUS_EINVAL;
893 }
894
895 rec = (struct acm_resolve_rec *) mad->data;
896 if (!rec->src_type || rec->src_type >= ACM_ADDRESS_RESERVED) {
897 acm_log(0, "ERROR - unknown src type 0x%x\n", rec->src_type);
898 return ACM_STATUS_EINVAL;
899 }
900
901 return ACM_STATUS_SUCCESS;
902 }
903
904 static void
905 acmp_send_addr_resp(struct acmp_ep *ep, struct acmp_dest *dest)
906 {
907 struct acm_resolve_rec *rec;
908 struct acmp_send_msg *msg;
909 struct acm_mad *mad;
910
911 acm_log(2, "%s\n", dest->name);
912 msg = acmp_alloc_send(ep, dest, sizeof (*mad));
913 if (!msg) {
914 acm_log(0, "ERROR - failed to allocate message\n");
915 return;
916 }
917
918 mad = (struct acm_mad *) msg->data;
919 rec = (struct acm_resolve_rec *) mad->data;
920
921 mad->base_version = 1;
922 mad->mgmt_class = ACM_MGMT_CLASS;
923 mad->class_version = 1;
924 mad->method = IB_METHOD_GET | IB_METHOD_RESP;
925 mad->status = ACM_STATUS_SUCCESS;
926 mad->control = ACM_CTRL_RESOLVE;
927 mad->tid = dest->req_id;
928 rec->gid_cnt = 1;
929 memcpy(rec->gid, dest->mgid.raw, sizeof(union ibv_gid));
930
931 acmp_post_send(&ep->resp_queue, msg);
932 }
933
934 static int
935 acmp_resolve_response(uint64_t id, struct acm_msg *req_msg,
936 struct acmp_dest *dest, uint8_t status)
937 {
938 struct acm_msg msg;
939
940 acm_log(2, "client %" PRIu64 ", status 0x%x\n", id, status);
941 memset(&msg, 0, sizeof msg);
942
943 if (dest) {
944 if (status == ACM_STATUS_ENODATA)
945 atomic_inc(&dest->ep->counters[ACM_CNTR_NODATA]);
946 else if (status)
947 atomic_inc(&dest->ep->counters[ACM_CNTR_ERROR]);
948 }
949 msg.hdr = req_msg->hdr;
950 msg.hdr.status = status;
951 msg.hdr.length = ACM_MSG_HDR_LENGTH;
952 memset(msg.hdr.data, 0, sizeof(msg.hdr.data));
953
954 if (status == ACM_STATUS_SUCCESS) {
955 msg.hdr.length += ACM_MSG_EP_LENGTH;
956 msg.resolve_data[0].flags = IBV_PATH_FLAG_GMP |
957 IBV_PATH_FLAG_PRIMARY | IBV_PATH_FLAG_BIDIRECTIONAL;
958 msg.resolve_data[0].type = ACM_EP_INFO_PATH;
959 msg.resolve_data[0].info.path = dest->path;
960
961 if (req_msg->hdr.src_out) {
962 msg.hdr.length += ACM_MSG_EP_LENGTH;
963 memcpy(&msg.resolve_data[1],
964 &req_msg->resolve_data[req_msg->hdr.src_index],
965 ACM_MSG_EP_LENGTH);
966 }
967 }
968
969 return acm_resolve_response(id, &msg);
970 }
971
972 static void
973 acmp_complete_queued_req(struct acmp_dest *dest, uint8_t status)
974 {
975 struct acmp_request *req;
976
977 acm_log(2, "status %d\n", status);
978 pthread_mutex_lock(&dest->lock);
979 while ((req = list_pop(&dest->req_queue, struct acmp_request, entry))) {
980 pthread_mutex_unlock(&dest->lock);
981
982 acm_log(2, "completing request, client %" PRIu64 "\n", req->id);
983 acmp_resolve_response(req->id, &req->msg, dest, status);
984 acmp_free_req(req);
985
986 pthread_mutex_lock(&dest->lock);
987 }
988 pthread_mutex_unlock(&dest->lock);
989 }
990
991 static void
992 acmp_dest_sa_resp(struct acm_sa_mad *mad)
993 {
994 struct acmp_dest *dest = (struct acmp_dest *) mad->context;
995 struct ib_sa_mad *sa_mad = (struct ib_sa_mad *) &mad->sa_mad;
996 uint8_t status;
997
998 if (!mad->umad.status) {
999 status = (uint8_t) (be16toh(sa_mad->status) >> 8);
1000 } else {
1001 status = ACM_STATUS_ETIMEDOUT;
1002 }
1003 acm_log(2, "%s status=0x%x\n", dest->name, status);
1004
1005 pthread_mutex_lock(&dest->lock);
1006 if (dest->state != ACMP_QUERY_ROUTE) {
1007 acm_log(1, "notice - discarding SA response\n");
1008 pthread_mutex_unlock(&dest->lock);
1009 goto out;
1010 }
1011
1012 if (!status) {
1013 memcpy(&dest->path, sa_mad->data, sizeof(dest->path));
1014 acmp_init_path_av(dest->ep->port, dest);
1015 dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
1016 dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
1017 acm_log(2, "timeout addr %" PRIu64 " route %" PRIu64 "\n",
1018 dest->addr_timeout, dest->route_timeout);
1019 dest->state = ACMP_READY;
1020 } else {
1021 dest->state = ACMP_INIT;
1022 }
1023 pthread_mutex_unlock(&dest->lock);
1024
1025 acmp_complete_queued_req(dest, status);
1026 out:
1027 acm_free_sa_mad(mad);
1028 }
1029
1030 static void
1031 acmp_resolve_sa_resp(struct acm_sa_mad *mad)
1032 {
1033 struct acmp_dest *dest = (struct acmp_dest *) mad->context;
1034 int send_resp;
1035
1036 acm_log(2, "\n");
1037 acmp_dest_sa_resp(mad);
1038
1039 pthread_mutex_lock(&dest->lock);
1040 send_resp = (dest->state == ACMP_READY);
1041 pthread_mutex_unlock(&dest->lock);
1042
1043 if (send_resp)
1044 acmp_send_addr_resp(dest->ep, dest);
1045 }
1046
1047 static struct acmp_addr *
1048 acmp_addr_lookup(struct acmp_ep *ep, uint8_t *addr, uint16_t type)
1049 {
1050 struct acmp_addr *ret = NULL;
1051 int i;
1052
1053 pthread_rwlock_rdlock(&ep->rwlock);
1054 for (i = 0; i < ep->nmbr_ep_addrs; i++) {
1055 if (ep->addr_info[i].type != type)
1056 continue;
1057
1058 if ((type == ACM_ADDRESS_NAME &&
1059 !strncasecmp((char *) ep->addr_info[i].info.name,
1060 (char *) addr, ACM_MAX_ADDRESS)) ||
1061 !memcmp(ep->addr_info[i].info.addr, addr,
1062 ACM_MAX_ADDRESS)) {
1063 ret = ep->addr_info + i;
1064 break;
1065 }
1066 }
1067 pthread_rwlock_unlock(&ep->rwlock);
1068
1069 return ret;
1070 }
1071
1072 static void
1073 acmp_process_addr_req(struct acmp_ep *ep, struct ibv_wc *wc, struct acm_mad *mad)
1074 {
1075 struct acm_resolve_rec *rec;
1076 struct acmp_dest *dest;
1077 uint8_t status;
1078 struct acmp_addr *addr;
1079
1080 acm_log(2, "\n");
1081 if ((status = acmp_validate_addr_req(mad))) {
1082 acm_log(0, "ERROR - invalid request\n");
1083 return;
1084 }
1085
1086 rec = (struct acm_resolve_rec *) mad->data;
1087 dest = acmp_acquire_dest(ep, rec->src_type, rec->src);
1088 if (!dest) {
1089 acm_log(0, "ERROR - unable to add source\n");
1090 return;
1091 }
1092
1093 addr = acmp_addr_lookup(ep, rec->dest, rec->dest_type);
1094 if (addr)
1095 dest->req_id = mad->tid;
1096
1097 pthread_mutex_lock(&dest->lock);
1098 acm_log(2, "dest state %d\n", dest->state);
1099 switch (dest->state) {
1100 case ACMP_READY:
1101 if (dest->remote_qpn == wc->src_qp)
1102 break;
1103
1104 acm_log(2, "src service has new qp, resetting\n");
1105 /* fall through */
1106 case ACMP_INIT:
1107 case ACMP_QUERY_ADDR:
1108 status = acmp_record_acm_addr(ep, dest, wc, rec);
1109 if (status)
1110 break;
1111 /* fall through */
1112 case ACMP_ADDR_RESOLVED:
1113 if (route_prot == ACMP_ROUTE_PROT_ACM) {
1114 status = acmp_record_acm_route(ep, dest);
1115 break;
1116 }
1117 if (addr || !list_empty(&dest->req_queue)) {
1118 status = acmp_resolve_path_sa(ep, dest, acmp_resolve_sa_resp);
1119 if (status)
1120 break;
1121 }
1122 /* fall through */
1123 default:
1124 pthread_mutex_unlock(&dest->lock);
1125 acmp_put_dest(dest);
1126 return;
1127 }
1128 pthread_mutex_unlock(&dest->lock);
1129 acmp_complete_queued_req(dest, status);
1130
1131 if (addr && !status) {
1132 acmp_send_addr_resp(ep, dest);
1133 }
1134 acmp_put_dest(dest);
1135 }
1136
1137 static void
1138 acmp_process_addr_resp(struct acmp_send_msg *msg, struct ibv_wc *wc, struct acm_mad *mad)
1139 {
1140 struct acm_resolve_rec *resp_rec;
1141 struct acmp_dest *dest = (struct acmp_dest *) msg->context;
1142 uint8_t status;
1143
1144 if (mad) {
1145 status = acm_class_status(mad->status);
1146 resp_rec = (struct acm_resolve_rec *) mad->data;
1147 } else {
1148 status = ACM_STATUS_ETIMEDOUT;
1149 resp_rec = NULL;
1150 }
1151 acm_log(2, "resp status 0x%x\n", status);
1152
1153 pthread_mutex_lock(&dest->lock);
1154 if (dest->state != ACMP_QUERY_ADDR) {
1155 pthread_mutex_unlock(&dest->lock);
1156 goto put;
1157 }
1158
1159 if (!status) {
1160 status = acmp_record_acm_addr(msg->ep, dest, wc, resp_rec);
1161 if (!status) {
1162 if (route_prot == ACMP_ROUTE_PROT_ACM) {
1163 status = acmp_record_acm_route(msg->ep, dest);
1164 } else {
1165 status = acmp_resolve_path_sa(msg->ep, dest, acmp_dest_sa_resp);
1166 if (!status) {
1167 pthread_mutex_unlock(&dest->lock);
1168 goto put;
1169 }
1170 }
1171 }
1172 } else {
1173 dest->state = ACMP_INIT;
1174 }
1175 pthread_mutex_unlock(&dest->lock);
1176
1177 acmp_complete_queued_req(dest, status);
1178 put:
1179 acmp_put_dest(dest);
1180 }
1181
1182 static void acmp_process_acm_recv(struct acmp_ep *ep, struct ibv_wc *wc, struct acm_mad *mad)
1183 {
1184 struct acmp_send_msg *req;
1185 struct acm_resolve_rec *rec;
1186 int free;
1187
1188 acm_log(2, "\n");
1189 if (mad->base_version != 1 || mad->class_version != 1) {
1190 acm_log(0, "ERROR - invalid version %d %d\n",
1191 mad->base_version, mad->class_version);
1192 return;
1193 }
1194
1195 if (mad->control != ACM_CTRL_RESOLVE) {
1196 acm_log(0, "ERROR - invalid control 0x%x\n", mad->control);
1197 return;
1198 }
1199
1200 rec = (struct acm_resolve_rec *) mad->data;
1201 acm_format_name(2, log_data, sizeof log_data,
1202 rec->src_type, rec->src, sizeof rec->src);
1203 acm_log(2, "src %s\n", log_data);
1204 acm_format_name(2, log_data, sizeof log_data,
1205 rec->dest_type, rec->dest, sizeof rec->dest);
1206 acm_log(2, "dest %s\n", log_data);
1207 if (mad->method & IB_METHOD_RESP) {
1208 acm_log(2, "received response\n");
1209 req = acmp_get_request(ep, mad->tid, &free);
1210 if (!req) {
1211 acm_log(1, "notice - response did not match active request\n");
1212 return;
1213 }
1214 acm_log(2, "found matching request\n");
1215 req->resp_handler(req, wc, mad);
1216 if (free)
1217 acmp_free_send(req);
1218 } else {
1219 acm_log(2, "unsolicited request\n");
1220 acmp_process_addr_req(ep, wc, mad);
1221 }
1222 }
1223
1224 static void
1225 acmp_sa_resp(struct acm_sa_mad *mad)
1226 {
1227 struct acmp_request *req = (struct acmp_request *) mad->context;
1228 struct ib_sa_mad *sa_mad = (struct ib_sa_mad *) &mad->sa_mad;
1229
1230 req->msg.hdr.opcode |= ACM_OP_ACK;
1231 if (!mad->umad.status) {
1232 struct acm_ep_addr_data *resolve_data = req->msg.resolve_data;
1233
1234 req->msg.hdr.status = (uint8_t) (be16toh(sa_mad->status) >> 8);
1235 memcpy(&resolve_data->info.path, sa_mad->data,
1236 sizeof(struct ibv_path_record));
1237 } else {
1238 req->msg.hdr.status = ACM_STATUS_ETIMEDOUT;
1239 }
1240 acm_log(2, "status 0x%x\n", req->msg.hdr.status);
1241
1242 if (req->msg.hdr.status)
1243 atomic_inc(&req->ep->counters[ACM_CNTR_ERROR]);
1244 acm_query_response(req->id, &req->msg);
1245 acm_free_sa_mad(mad);
1246 acmp_free_req(req);
1247 }
1248
1249 static void acmp_process_sa_recv(struct acmp_ep *ep, struct ibv_wc *wc, struct acm_mad *mad)
1250 {
1251 struct ib_sa_mad *sa_mad = (struct ib_sa_mad *) mad;
1252 struct acmp_send_msg *req;
1253 int free;
1254
1255 acm_log(2, "\n");
1256 if (mad->base_version != 1 || mad->class_version != 2 ||
1257 !(mad->method & IB_METHOD_RESP) || sa_mad->attr_id != IB_SA_ATTR_PATH_REC) {
1258 acm_log(0, "ERROR - unexpected SA MAD %d %d\n",
1259 mad->base_version, mad->class_version);
1260 return;
1261 }
1262
1263 req = acmp_get_request(ep, mad->tid, &free);
1264 if (!req) {
1265 acm_log(1, "notice - response did not match active request\n");
1266 return;
1267 }
1268 acm_log(2, "found matching request\n");
1269 req->resp_handler(req, wc, mad);
1270 if (free)
1271 acmp_free_send(req);
1272 }
1273
1274 static void acmp_process_recv(struct acmp_ep *ep, struct ibv_wc *wc)
1275 {
1276 struct acm_mad *mad;
1277
1278 acm_log(2, "base endpoint name %s\n", ep->id_string);
1279 mad = (struct acm_mad *) (uintptr_t) (wc->wr_id + sizeof(struct ibv_grh));
1280 switch (mad->mgmt_class) {
1281 case IB_MGMT_CLASS_SA:
1282 acmp_process_sa_recv(ep, wc, mad);
1283 break;
1284 case ACM_MGMT_CLASS:
1285 acmp_process_acm_recv(ep, wc, mad);
1286 break;
1287 default:
1288 acm_log(0, "ERROR - invalid mgmt class 0x%x\n", mad->mgmt_class);
1289 break;
1290 }
1291
1292 acmp_post_recv(ep, wc->wr_id);
1293 }
1294
1295 static void acmp_process_comp(struct acmp_ep *ep, struct ibv_wc *wc)
1296 {
1297 if (wc->status) {
1298 acm_log(0, "ERROR - work completion error\n"
1299 "\topcode %d, completion status %d\n",
1300 wc->opcode, wc->status);
1301 return;
1302 }
1303
1304 if (wc->opcode & IBV_WC_RECV)
1305 acmp_process_recv(ep, wc);
1306 else
1307 acmp_complete_send((struct acmp_send_msg *) (uintptr_t) wc->wr_id);
1308 }
1309
1310 static void *acmp_comp_handler(void *context)
1311 {
1312 struct acmp_device *dev = (struct acmp_device *) context;
1313 struct acmp_ep *ep;
1314 struct ibv_cq *cq;
1315 struct ibv_wc wc;
1316 int cnt;
1317
1318 acm_log(1, "started\n");
1319
1320 if (pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL)) {
1321 acm_log(0, "Error: failed to set cancel type for dev %s\n",
1322 dev->verbs->device->name);
1323 pthread_exit(NULL);
1324 }
1325
1326 if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL)) {
1327 acm_log(0, "Error: failed to set cancel state for dev %s\n",
1328 dev->verbs->device->name);
1329 pthread_exit(NULL);
1330 }
1331 while (1) {
1332 pthread_testcancel();
1333 ibv_get_cq_event(dev->channel, &cq, (void *) &ep);
1334
1335 cnt = 0;
1336 while (ibv_poll_cq(cq, 1, &wc) > 0) {
1337 cnt++;
1338 acmp_process_comp(ep, &wc);
1339 }
1340
1341 ibv_req_notify_cq(cq, 0);
1342 while (ibv_poll_cq(cq, 1, &wc) > 0) {
1343 cnt++;
1344 acmp_process_comp(ep, &wc);
1345 }
1346
1347 ibv_ack_cq_events(cq, cnt);
1348 }
1349
1350 return NULL;
1351 }
1352
1353 static void acmp_format_mgid(union ibv_gid *mgid, uint16_t pkey, uint8_t tos,
1354 uint8_t rate, uint8_t mtu)
1355 {
1356 mgid->raw[0] = 0xFF;
1357 mgid->raw[1] = 0x10 | 0x05;
1358 mgid->raw[2] = 0x40;
1359 mgid->raw[3] = 0x01;
1360 mgid->raw[4] = (uint8_t) (pkey >> 8);
1361 mgid->raw[5] = (uint8_t) pkey;
1362 mgid->raw[6] = tos;
1363 mgid->raw[7] = rate;
1364 mgid->raw[8] = mtu;
1365 mgid->raw[9] = 0;
1366 mgid->raw[10] = 0;
1367 mgid->raw[11] = 0;
1368 mgid->raw[12] = 0;
1369 mgid->raw[13] = 0;
1370 mgid->raw[14] = 0;
1371 mgid->raw[15] = 0;
1372 }
1373
1374 static void acmp_init_join(struct ib_sa_mad *mad, union ibv_gid *port_gid,
1375 uint16_t pkey, uint8_t tos, uint8_t tclass, uint8_t sl, uint8_t rate, uint8_t mtu)
1376 {
1377 struct ib_mc_member_rec *mc_rec;
1378
1379 acm_log(2, "\n");
1380 mad->base_version = 1;
1381 mad->mgmt_class = IB_MGMT_CLASS_SA;
1382 mad->class_version = 2;
1383 mad->method = IB_METHOD_SET;
1384 mad->tid = htobe64((uint64_t) atomic_inc(&g_tid));
1385 mad->attr_id = IB_SA_ATTR_MC_MEMBER_REC;
1386 mad->comp_mask =
1387 IB_COMP_MASK_MC_MGID | IB_COMP_MASK_MC_PORT_GID |
1388 IB_COMP_MASK_MC_QKEY | IB_COMP_MASK_MC_MTU_SEL| IB_COMP_MASK_MC_MTU |
1389 IB_COMP_MASK_MC_TCLASS | IB_COMP_MASK_MC_PKEY | IB_COMP_MASK_MC_RATE_SEL |
1390 IB_COMP_MASK_MC_RATE | IB_COMP_MASK_MC_SL | IB_COMP_MASK_MC_FLOW |
1391 IB_COMP_MASK_MC_SCOPE | IB_COMP_MASK_MC_JOIN_STATE;
1392
1393 mc_rec = (struct ib_mc_member_rec *) mad->data;
1394 acmp_format_mgid(&mc_rec->mgid, pkey | IB_PKEY_FULL_MEMBER, tos, rate, mtu);
1395 mc_rec->port_gid = *port_gid;
1396 mc_rec->qkey = htobe32(ACM_QKEY);
1397 mc_rec->mtu = umad_sa_set_rate_mtu_or_life(UMAD_SA_SELECTOR_EXACTLY, mtu);
1398 mc_rec->tclass = tclass;
1399 mc_rec->pkey = htobe16(pkey);
1400 mc_rec->rate = umad_sa_set_rate_mtu_or_life(UMAD_SA_SELECTOR_EXACTLY, rate);
1401 mc_rec->sl_flow_hop = umad_sa_mcm_set_sl_flow_hop(sl, 0, 0);
1402 mc_rec->scope_state = umad_sa_mcm_set_scope_state(UMAD_SA_MCM_ADDR_SCOPE_SITE_LOCAL,
1403 UMAD_SA_MCM_JOIN_STATE_FULL_MEMBER);
1404 }
1405
1406 static void acmp_join_group(struct acmp_ep *ep, union ibv_gid *port_gid,
1407 uint8_t tos, uint8_t tclass, uint8_t sl, uint8_t rate, uint8_t mtu)
1408 {
1409 struct ib_sa_mad *mad;
1410 struct ib_mc_member_rec *mc_rec;
1411 struct acm_sa_mad *sa_mad;
1412
1413 acm_log(2, "\n");
1414 sa_mad = acm_alloc_sa_mad(ep->endpoint, ep, acmp_process_join_resp);
1415 if (!sa_mad) {
1416 acm_log(0, "Error - failed to allocate sa_mad\n");
1417 return;
1418 }
1419
1420 acm_log(0, "%s %d pkey 0x%x, sl 0x%x, rate 0x%x, mtu 0x%x\n",
1421 ep->port->dev->verbs->device->name,
1422 ep->port->port_num, ep->pkey, sl, rate, mtu);
1423 mad = (struct ib_sa_mad *) &sa_mad->sa_mad;
1424 acmp_init_join(mad, port_gid, ep->pkey, tos, tclass, sl, rate, mtu);
1425 mc_rec = (struct ib_mc_member_rec *) mad->data;
1426 acmp_set_dest_addr(&ep->mc_dest[ep->mc_cnt++], ACM_ADDRESS_GID,
1427 mc_rec->mgid.raw, sizeof(mc_rec->mgid));
1428 ep->mc_dest[ep->mc_cnt - 1].state = ACMP_INIT;
1429
1430 if (acm_send_sa_mad(sa_mad)) {
1431 acm_log(0, "Error - Failed to send sa mad\n");
1432 acm_free_sa_mad(sa_mad);
1433 }
1434 }
1435
1436 static void acmp_ep_join(struct acmp_ep *ep)
1437 {
1438 struct acmp_port *port;
1439 union ibv_gid gid;
1440
1441 port = ep->port;
1442 acm_log(1, "%s\n", ep->id_string);
1443
1444 if (ep->mc_dest[0].state == ACMP_READY && ep->mc_dest[0].ah) {
1445 ibv_detach_mcast(ep->qp, &ep->mc_dest[0].mgid,
1446 ep->mc_dest[0].av.dlid);
1447 ibv_destroy_ah(ep->mc_dest[0].ah);
1448 ep->mc_dest[0].ah = NULL;
1449 }
1450 ep->mc_cnt = 0;
1451 ep->state = ACMP_INIT;
1452 acm_get_gid((struct acm_port *)ep->port->port, 0, &gid);
1453 acmp_join_group(ep, &gid, 0, 0, 0, min_rate, min_mtu);
1454
1455 if ((route_prot == ACMP_ROUTE_PROT_ACM) &&
1456 (port->rate != min_rate || port->mtu != min_mtu))
1457 acmp_join_group(ep, &gid, 0, 0, 0, port->rate, port->mtu);
1458
1459 acm_log(1, "join for %s complete\n", ep->id_string);
1460 }
1461
1462 static int acmp_port_join(void *port_context)
1463 {
1464 struct acmp_ep *ep;
1465 struct acmp_port *port = port_context;
1466
1467 acm_log(1, "device %s port %d\n", port->dev->verbs->device->name,
1468 port->port_num);
1469
1470 list_for_each(&port->ep_list, ep, entry) {
1471 if (!ep->endpoint) {
1472 /* Stale endpoint */
1473 continue;
1474 }
1475 acmp_ep_join(ep);
1476 }
1477 acm_log(1, "joins for device %s port %d complete\n",
1478 port->dev->verbs->device->name, port->port_num);
1479
1480 return 0;
1481 }
1482
1483 static int acmp_handle_event(void *port_context, enum ibv_event_type type)
1484 {
1485 int ret = 0;
1486
1487 acm_log(2, "event %s\n", ibv_event_type_str(type));
1488
1489 switch (type) {
1490 case IBV_EVENT_CLIENT_REREGISTER:
1491 ret = acmp_port_join(port_context);
1492 break;
1493 default:
1494 break;
1495 }
1496 return ret;
1497 }
1498
1499 static void acmp_process_timeouts(void)
1500 {
1501 struct acmp_send_msg *msg;
1502 struct acm_resolve_rec *rec;
1503 struct acm_mad *mad;
1504
1505 while ((msg = list_pop(&timeout_list, struct acmp_send_msg, entry))) {
1506 mad = (struct acm_mad *) &msg->data[0];
1507 rec = (struct acm_resolve_rec *) mad->data;
1508
1509 acm_format_name(0, log_data, sizeof log_data,
1510 rec->dest_type, rec->dest, sizeof rec->dest);
1511 acm_log(0, "notice - dest %s\n", log_data);
1512
1513 msg->resp_handler(msg, NULL, NULL);
1514 acmp_free_send(msg);
1515 }
1516 }
1517
1518 static void acmp_process_wait_queue(struct acmp_ep *ep, uint64_t *next_expire)
1519 {
1520 struct acmp_send_msg *msg, *next;
1521 struct ibv_send_wr *bad_wr;
1522
1523 list_for_each_safe(&ep->wait_queue, msg, next, entry) {
1524 if (msg->expires <= time_stamp_ms()) {
1525 list_del(&msg->entry);
1526 (void) atomic_dec(&wait_cnt);
1527 if (--msg->tries) {
1528 acm_log(1, "notice - retrying request\n");
1529 list_add_tail(&ep->active_queue, &msg->entry);
1530 ibv_post_send(ep->qp, &msg->wr, &bad_wr);
1531 } else {
1532 acm_log(0, "notice - failing request\n");
1533 acmp_send_available(ep, msg->req_queue);
1534 list_add_tail(&timeout_list, &msg->entry);
1535 }
1536 } else {
1537 *next_expire = min(*next_expire, msg->expires);
1538 break;
1539 }
1540 }
1541 }
1542
1543 /* While the device/port/ep will not be freed, we need to be careful of
1544 * their addition while walking the link lists. Therefore, we need to acquire
1545 * the appropriate locks.
1546 */
1547 static void *acmp_retry_handler(void *context)
1548 {
1549 struct acmp_device *dev;
1550 struct acmp_port *port;
1551 struct acmp_ep *ep;
1552 uint64_t next_expire;
1553 int i, wait;
1554
1555 acm_log(0, "started\n");
1556 if (pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL)) {
1557 acm_log(0, "Error: failed to set cancel type \n");
1558 pthread_exit(NULL);
1559 }
1560 if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL)) {
1561 acm_log(0, "Error: failed to set cancel state\n");
1562 pthread_exit(NULL);
1563 }
1564 retry_thread_started = 1;
1565
1566 while (1) {
1567 while (!atomic_get(&wait_cnt)) {
1568 pthread_testcancel();
1569 event_wait(&timeout_event, -1);
1570 }
1571
1572 next_expire = -1;
1573 pthread_mutex_lock(&acmp_dev_lock);
1574 list_for_each(&acmp_dev_list, dev, entry) {
1575 pthread_mutex_unlock(&acmp_dev_lock);
1576
1577 for (i = 0; i < dev->port_cnt; i++) {
1578 port = &dev->port[i];
1579
1580 pthread_mutex_lock(&port->lock);
1581 list_for_each(&port->ep_list, ep, entry) {
1582 pthread_mutex_unlock(&port->lock);
1583 pthread_mutex_lock(&ep->lock);
1584 if (!list_empty(&ep->wait_queue))
1585 acmp_process_wait_queue(ep, &next_expire);
1586 pthread_mutex_unlock(&ep->lock);
1587 pthread_mutex_lock(&port->lock);
1588 }
1589 pthread_mutex_unlock(&port->lock);
1590 }
1591 pthread_mutex_lock(&acmp_dev_lock);
1592 }
1593 pthread_mutex_unlock(&acmp_dev_lock);
1594
1595 acmp_process_timeouts();
1596 if (next_expire != -1) {
1597 wait = (int) (next_expire - time_stamp_ms());
1598 if (wait > 0 && atomic_get(&wait_cnt)) {
1599 pthread_testcancel();
1600 event_wait(&timeout_event, wait);
1601 }
1602 }
1603 }
1604
1605 retry_thread_started = 0;
1606 return NULL;
1607 }
1608
1609 /* rwlock must be held read-locked */
1610 static int
1611 __acmp_query(struct acmp_ep *ep, struct acm_msg *msg, uint64_t id)
1612 {
1613 struct acmp_request *req;
1614 struct ib_sa_mad *mad;
1615 uint8_t status;
1616 struct acm_sa_mad *sa_mad;
1617
1618 if (ep->state != ACMP_READY) {
1619 status = ACM_STATUS_ENODATA;
1620 goto resp;
1621 }
1622
1623 req = acmp_alloc_req(id, msg);
1624 if (!req) {
1625 status = ACM_STATUS_ENOMEM;
1626 goto resp;
1627 }
1628 req->ep = ep;
1629
1630 sa_mad = acm_alloc_sa_mad(ep->endpoint, req, acmp_sa_resp);
1631 if (!sa_mad) {
1632 acm_log(0, "Error - failed to allocate sa_mad\n");
1633 status = ACM_STATUS_ENOMEM;
1634 goto free_req;
1635 }
1636
1637 mad = (struct ib_sa_mad *) &sa_mad->sa_mad;
1638 acmp_init_path_query(mad);
1639
1640 memcpy(mad->data, &msg->resolve_data[0].info.path,
1641 sizeof(struct ibv_path_record));
1642 mad->comp_mask = acm_path_comp_mask(&msg->resolve_data[0].info.path);
1643
1644 acm_increment_counter(ACM_CNTR_ROUTE_QUERY);
1645 atomic_inc(&ep->counters[ACM_CNTR_ROUTE_QUERY]);
1646 if (acm_send_sa_mad(sa_mad)) {
1647 acm_log(0, "Error - Failed to send sa mad\n");
1648 status = ACM_STATUS_ENODATA;
1649 goto free_mad;
1650 }
1651 return ACM_STATUS_SUCCESS;
1652
1653 free_mad:
1654 acm_free_sa_mad(sa_mad);
1655 free_req:
1656 acmp_free_req(req);
1657 resp:
1658 msg->hdr.opcode |= ACM_OP_ACK;
1659 msg->hdr.status = status;
1660 if (status == ACM_STATUS_ENODATA)
1661 atomic_inc(&ep->counters[ACM_CNTR_NODATA]);
1662 else
1663 atomic_inc(&ep->counters[ACM_CNTR_ERROR]);
1664 return acm_query_response(id, msg);
1665 }
1666
1667 static int
1668 acmp_query(void *addr_context, struct acm_msg *msg, uint64_t id)
1669 {
1670 struct acmp_addr_ctx *addr_ctx = addr_context;
1671 struct acmp_addr *address;
1672 int ret;
1673
1674 pthread_rwlock_rdlock(&addr_ctx->ep->rwlock);
1675 address = addr_ctx->ep->addr_info + addr_ctx->addr_inx;
1676 ret = __acmp_query(address->ep, msg, id);
1677 pthread_rwlock_unlock(&addr_ctx->ep->rwlock);
1678
1679 return ret;
1680 }
1681
1682 static uint8_t
1683 acmp_send_resolve(struct acmp_ep *ep, struct acmp_dest *dest,
1684 struct acm_ep_addr_data *saddr)
1685 {
1686 struct acmp_send_msg *msg;
1687 struct acm_mad *mad;
1688 struct acm_resolve_rec *rec;
1689 int i;
1690
1691 acm_log(2, "\n");
1692 msg = acmp_alloc_send(ep, &ep->mc_dest[0], sizeof(*mad));
1693 if (!msg) {
1694 acm_log(0, "ERROR - cannot allocate send msg\n");
1695 return ACM_STATUS_ENOMEM;
1696 }
1697
1698 acmp_init_send_req(msg, (void *) dest, acmp_process_addr_resp);
1699 (void) atomic_inc(&dest->refcnt);
1700
1701 mad = (struct acm_mad *) msg->data;
1702 mad->base_version = 1;
1703 mad->mgmt_class = ACM_MGMT_CLASS;
1704 mad->class_version = 1;
1705 mad->method = IB_METHOD_GET;
1706 mad->control = ACM_CTRL_RESOLVE;
1707 mad->tid = htobe64((uint64_t) atomic_inc(&g_tid));
1708
1709 rec = (struct acm_resolve_rec *) mad->data;
1710 rec->src_type = (uint8_t) saddr->type;
1711 rec->src_length = ACM_MAX_ADDRESS;
1712 memcpy(rec->src, saddr->info.addr, ACM_MAX_ADDRESS);
1713 rec->dest_type = dest->addr_type;
1714 rec->dest_length = ACM_MAX_ADDRESS;
1715 memcpy(rec->dest, dest->address, ACM_MAX_ADDRESS);
1716
1717 rec->gid_cnt = (uint8_t) ep->mc_cnt;
1718 for (i = 0; i < ep->mc_cnt; i++)
1719 memcpy(&rec->gid[i], ep->mc_dest[i].address, 16);
1720
1721 acm_increment_counter(ACM_CNTR_ADDR_QUERY);
1722 atomic_inc(&ep->counters[ACM_CNTR_ADDR_QUERY]);
1723 acmp_post_send(&ep->resolve_queue, msg);
1724 return 0;
1725 }
1726
1727 /* Caller must hold dest lock */
1728 static uint8_t acmp_queue_req(struct acmp_dest *dest, uint64_t id, struct acm_msg *msg)
1729 {
1730 struct acmp_request *req;
1731
1732 acm_log(2, "id %" PRIu64 "\n", id);
1733 req = acmp_alloc_req(id, msg);
1734 if (!req) {
1735 return ACM_STATUS_ENOMEM;
1736 }
1737 req->ep = dest->ep;
1738
1739 list_add_tail(&dest->req_queue, &req->entry);
1740 return ACM_STATUS_SUCCESS;
1741 }
1742
1743 static int acmp_dest_timeout(struct acmp_dest *dest)
1744 {
1745 uint64_t timestamp = time_stamp_min();
1746
1747 if (timestamp > dest->addr_timeout) {
1748 acm_log(2, "%s address timed out\n", dest->name);
1749 dest->state = ACMP_INIT;
1750 return 1;
1751 } else if (timestamp > dest->route_timeout) {
1752 acm_log(2, "%s route timed out\n", dest->name);
1753 dest->state = ACMP_ADDR_RESOLVED;
1754 return 1;
1755 }
1756 return 0;
1757 }
1758
1759 static int
1760 acmp_check_addr_match(struct ifaddrs *iap, struct acm_ep_addr_data *saddr,
1761 unsigned int d_family)
1762 {
1763 char sip[INET6_ADDRSTRLEN] = {0};
1764 char dip[INET6_ADDRSTRLEN] = {0};
1765 const char *tmp;
1766 size_t sock_size;
1767 unsigned int s_family;
1768 int ret;
1769
1770 s_family = iap->ifa_addr->sa_family;
1771
1772 if (!(iap->ifa_flags & IFF_UP) ||
1773 (s_family != d_family))
1774 return -1;
1775
1776 sock_size = (s_family == AF_INET) ? sizeof(struct sockaddr_in) :
1777 sizeof(struct sockaddr_in6);
1778
1779 ret = getnameinfo(iap->ifa_addr, sock_size,
1780 sip, sizeof(sip),
1781 NULL, 0, NI_NUMERICHOST);
1782
1783 if (ret)
1784 return ret;
1785
1786 tmp = inet_ntop(d_family, (void *)saddr->info.addr, dip,
1787 sizeof(dip));
1788 if (!tmp)
1789 return -1;
1790 ret = memcmp(sip, dip, strlen(dip));
1791 return ret;
1792 }
1793
1794 static void
1795 acmp_acquire_sgid(struct acm_ep_addr_data *saddr,
1796 struct acmp_dest *dest)
1797 {
1798 struct ifaddrs *addrs, *iap;
1799 unsigned int d_family;
1800 int ret;
1801
1802 if (!ib_any_gid(&dest->path.sgid))
1803 return;
1804
1805 if (dest->addr_type != ACM_ADDRESS_IP6 &&
1806 dest->addr_type != ACM_ADDRESS_IP)
1807 return;
1808
1809 if (getifaddrs(&addrs))
1810 return;
1811
1812 d_family = (dest->addr_type == ACM_ADDRESS_IP) ? AF_INET : AF_INET6;
1813
1814 for (iap = addrs; iap != NULL; iap = iap->ifa_next) {
1815 ret = acmp_check_addr_match(iap, saddr, d_family);
1816 if (!ret) {
1817 ret = acm_if_get_sgid(iap->ifa_name,
1818 &dest->path.sgid);
1819 if (!ret)
1820 break;
1821 }
1822 }
1823 freeifaddrs(addrs);
1824 }
1825
1826 static int
1827 acmp_resolve_dest(struct acmp_ep *ep, struct acm_msg *msg, uint64_t id)
1828 {
1829 struct acmp_dest *dest;
1830 struct acm_ep_addr_data *saddr, *daddr;
1831 uint8_t status;
1832 int ret;
1833
1834 saddr = &msg->resolve_data[msg->hdr.src_index];
1835 daddr = &msg->resolve_data[msg->hdr.dst_index];
1836 acm_format_name(2, log_data, sizeof log_data,
1837 daddr->type, daddr->info.addr, sizeof daddr->info.addr);
1838 acm_log(2, "dest %s\n", log_data);
1839
1840 dest = acmp_acquire_dest(ep, daddr->type, daddr->info.addr);
1841 if (!dest) {
1842 acm_log(0, "ERROR - unable to allocate destination in request\n");
1843 atomic_inc(&ep->counters[ACM_CNTR_ERROR]);
1844 return acmp_resolve_response(id, msg, NULL, ACM_STATUS_ENOMEM);
1845 }
1846
1847 pthread_mutex_lock(&dest->lock);
1848 test:
1849 switch (dest->state) {
1850 case ACMP_READY:
1851 if (acmp_dest_timeout(dest))
1852 goto test;
1853 acm_log(2, "request satisfied from local cache\n");
1854 acm_increment_counter(ACM_CNTR_ROUTE_CACHE);
1855 atomic_inc(&ep->counters[ACM_CNTR_ROUTE_CACHE]);
1856 status = ACM_STATUS_SUCCESS;
1857 break;
1858 case ACMP_ADDR_RESOLVED:
1859 acm_log(2, "have address, resolving route\n");
1860 acm_increment_counter(ACM_CNTR_ADDR_CACHE);
1861 atomic_inc(&ep->counters[ACM_CNTR_ADDR_CACHE]);
1862 acmp_acquire_sgid(saddr, dest);
1863 status = acmp_resolve_path_sa(ep, dest, acmp_dest_sa_resp);
1864 if (status) {
1865 break;
1866 }
1867 goto queue;
1868 case ACMP_INIT:
1869 acm_log(2, "sending resolve msg to dest\n");
1870 status = acmp_send_resolve(ep, dest, saddr);
1871 if (status) {
1872 break;
1873 }
1874 dest->state = ACMP_QUERY_ADDR;
1875 /* fall through */
1876 default:
1877 queue:
1878 if (daddr->flags & ACM_FLAGS_NODELAY) {
1879 acm_log(2, "lookup initiated, but client wants no delay\n");
1880 status = ACM_STATUS_ENODATA;
1881 break;
1882 }
1883 status = acmp_queue_req(dest, id, msg);
1884 if (status) {
1885 break;
1886 }
1887 ret = 0;
1888 pthread_mutex_unlock(&dest->lock);
1889 goto put;
1890 }
1891 pthread_mutex_unlock(&dest->lock);
1892 ret = acmp_resolve_response(id, msg, dest, status);
1893 put:
1894 acmp_put_dest(dest);
1895 return ret;
1896 }
1897
1898 static int
1899 acmp_resolve_path(struct acmp_ep *ep, struct acm_msg *msg, uint64_t id)
1900 {
1901 struct acmp_dest *dest;
1902 struct ibv_path_record *path;
1903 uint8_t *addr;
1904 uint8_t status;
1905 int ret;
1906
1907 path = &msg->resolve_data[0].info.path;
1908 addr = msg->resolve_data[1].info.addr;
1909 memset(addr, 0, ACM_MAX_ADDRESS);
1910 if (path->dlid) {
1911 * ((__be16 *) addr) = path->dlid;
1912 dest = acmp_acquire_dest(ep, ACM_ADDRESS_LID, addr);
1913 } else {
1914 memcpy(addr, &path->dgid, sizeof path->dgid);
1915 dest = acmp_acquire_dest(ep, ACM_ADDRESS_GID, addr);
1916 }
1917 if (!dest) {
1918 acm_log(0, "ERROR - unable to allocate destination in request\n");
1919 atomic_inc(&ep->counters[ACM_CNTR_ERROR]);
1920 return acmp_resolve_response(id, msg, NULL, ACM_STATUS_ENOMEM);
1921 }
1922
1923 pthread_mutex_lock(&dest->lock);
1924 test:
1925 switch (dest->state) {
1926 case ACMP_READY:
1927 if (acmp_dest_timeout(dest))
1928 goto test;
1929 acm_log(2, "request satisfied from local cache\n");
1930 acm_increment_counter(ACM_CNTR_ROUTE_CACHE);
1931 atomic_inc(&ep->counters[ACM_CNTR_ROUTE_CACHE]);
1932 status = ACM_STATUS_SUCCESS;
1933 break;
1934 case ACMP_INIT:
1935 acm_log(2, "have path, bypassing address resolution\n");
1936 acmp_record_path_addr(ep, dest, path);
1937 /* fall through */
1938 case ACMP_ADDR_RESOLVED:
1939 acm_log(2, "have address, resolving route\n");
1940 status = acmp_resolve_path_sa(ep, dest, acmp_dest_sa_resp);
1941 if (status) {
1942 break;
1943 }
1944 /* fall through */
1945 default:
1946 if (msg->resolve_data[0].flags & ACM_FLAGS_NODELAY) {
1947 acm_log(2, "lookup initiated, but client wants no delay\n");
1948 status = ACM_STATUS_ENODATA;
1949 break;
1950 }
1951 status = acmp_queue_req(dest, id, msg);
1952 if (status) {
1953 break;
1954 }
1955 ret = 0;
1956 pthread_mutex_unlock(&dest->lock);
1957 goto put;
1958 }
1959 pthread_mutex_unlock(&dest->lock);
1960 ret = acmp_resolve_response(id, msg, dest, status);
1961 put:
1962 acmp_put_dest(dest);
1963 return ret;
1964 }
1965
1966 static int
1967 acmp_resolve(void *addr_context, struct acm_msg *msg, uint64_t id)
1968 {
1969 struct acmp_addr_ctx *addr_ctx = addr_context;
1970 struct acmp_addr *address = addr_ctx->ep->addr_info + addr_ctx->addr_inx;
1971 struct acmp_ep *ep = address->ep;
1972
1973 if (ep->state != ACMP_READY) {
1974 atomic_inc(&ep->counters[ACM_CNTR_NODATA]);
1975 return acmp_resolve_response(id, msg, NULL, ACM_STATUS_ENODATA);
1976 }
1977
1978 atomic_inc(&ep->counters[ACM_CNTR_RESOLVE]);
1979 if (msg->resolve_data[0].type == ACM_EP_INFO_PATH)
1980 return acmp_resolve_path(ep, msg, id);
1981 else
1982 return acmp_resolve_dest(ep, msg, id);
1983 }
1984
1985 static void acmp_query_perf(void *ep_context, uint64_t *values, uint8_t *cnt)
1986 {
1987 struct acmp_ep *ep = ep_context;
1988 int i;
1989
1990 for (i = 0; i < ACM_MAX_COUNTER; i++)
1991 values[i] = htobe64((uint64_t) atomic_get(&ep->counters[i]));
1992 *cnt = ACM_MAX_COUNTER;
1993 }
1994
1995 static enum acmp_addr_prot acmp_convert_addr_prot(char *param)
1996 {
1997 if (!strcasecmp("acm", param))
1998 return ACMP_ADDR_PROT_ACM;
1999
2000 return addr_prot;
2001 }
2002
2003 static enum acmp_route_prot acmp_convert_route_prot(char *param)
2004 {
2005 if (!strcasecmp("acm", param))
2006 return ACMP_ROUTE_PROT_ACM;
2007 else if (!strcasecmp("sa", param))
2008 return ACMP_ROUTE_PROT_SA;
2009
2010 return route_prot;
2011 }
2012
2013 static enum acmp_loopback_prot acmp_convert_loopback_prot(char *param)
2014 {
2015 if (!strcasecmp("none", param))
2016 return ACMP_LOOPBACK_PROT_NONE;
2017 else if (!strcasecmp("local", param))
2018 return ACMP_LOOPBACK_PROT_LOCAL;
2019
2020 return loopback_prot;
2021 }
2022
2023 static enum acmp_route_preload acmp_convert_route_preload(char *param)
2024 {
2025 if (!strcasecmp("none", param) || !strcasecmp("no", param))
2026 return ACMP_ROUTE_PRELOAD_NONE;
2027 else if (!strcasecmp("opensm_full_v1", param))
2028 return ACMP_ROUTE_PRELOAD_OSM_FULL_V1;
2029
2030 return route_preload;
2031 }
2032
2033 static enum acmp_addr_preload acmp_convert_addr_preload(char *param)
2034 {
2035 if (!strcasecmp("none", param) || !strcasecmp("no", param))
2036 return ACMP_ADDR_PRELOAD_NONE;
2037 else if (!strcasecmp("acm_hosts", param))
2038 return ACMP_ADDR_PRELOAD_HOSTS;
2039
2040 return addr_preload;
2041 }
2042
2043 static int acmp_post_recvs(struct acmp_ep *ep)
2044 {
2045 int i, size;
2046
2047 size = recv_depth * ACM_RECV_SIZE;
2048 ep->recv_bufs = malloc(size);
2049 if (!ep->recv_bufs) {
2050 acm_log(0, "ERROR - unable to allocate receive buffer\n");
2051 return ACM_STATUS_ENOMEM;
2052 }
2053
2054 ep->mr = ibv_reg_mr(ep->port->dev->pd, ep->recv_bufs, size,
2055 IBV_ACCESS_LOCAL_WRITE);
2056 if (!ep->mr) {
2057 acm_log(0, "ERROR - unable to register receive buffer\n");
2058 goto err;
2059 }
2060
2061 for (i = 0; i < recv_depth; i++) {
2062 acmp_post_recv(ep, (uintptr_t) (ep->recv_bufs + ACM_RECV_SIZE * i));
2063 }
2064 return 0;
2065
2066 err:
2067 free(ep->recv_bufs);
2068 return -1;
2069 }
2070
2071 /* Parse "opensm full v1" file to build LID to GUID table */
2072 static void acmp_parse_osm_fullv1_lid2guid(FILE *f, __be64 *lid2guid)
2073 {
2074 char s[128];
2075 char *p, *ptr, *p_guid, *p_lid;
2076 uint64_t guid;
2077 uint16_t lid;
2078
2079 while (fgets(s, sizeof s, f)) {
2080 if (s[0] == '#')
2081 continue;
2082 if (!(p = strtok_r(s, " \n", &ptr)))
2083 continue; /* ignore blank lines */
2084
2085 if (strncmp(p, "Switch", sizeof("Switch") - 1) &&
2086 strncmp(p, "Channel", sizeof("Channel") - 1) &&
2087 strncmp(p, "Router", sizeof("Router") - 1))
2088 continue;
2089
2090 if (!strncmp(p, "Channel", sizeof("Channel") - 1)) {
2091 p = strtok_r(NULL, " ", &ptr); /* skip 'Adapter' */
2092 if (!p)
2093 continue;
2094 }
2095
2096 p_guid = strtok_r(NULL, ",", &ptr);
2097 if (!p_guid)
2098 continue;
2099
2100 guid = (uint64_t) strtoull(p_guid, NULL, 16);
2101
2102 ptr = strstr(ptr, "base LID");
2103 if (!ptr)
2104 continue;
2105 ptr += sizeof("base LID");
2106 p_lid = strtok_r(NULL, ",", &ptr);
2107 if (!p_lid)
2108 continue;
2109
2110 lid = (uint16_t) strtoul(p_lid, NULL, 0);
2111 if (lid >= IB_LID_MCAST_START)
2112 continue;
2113 if (lid2guid[lid])
2114 acm_log(0, "ERROR - duplicate lid %u\n", lid);
2115 else
2116 lid2guid[lid] = htobe64(guid);
2117 }
2118 }
2119
2120 /* Parse 'opensm full v1' file to populate PR cache */
2121 static int acmp_parse_osm_fullv1_paths(FILE *f, __be64 *lid2guid, struct acmp_ep *ep)
2122 {
2123 union ibv_gid sgid, dgid;
2124 struct ibv_port_attr attr = {};
2125 struct acmp_dest *dest;
2126 char s[128];
2127 char *p, *ptr, *p_guid, *p_lid;
2128 uint64_t guid;
2129 uint16_t lid, dlid;
2130 __be16 net_dlid;
2131 int sl, mtu, rate;
2132 int ret = 1, i;
2133 uint8_t addr[ACM_MAX_ADDRESS];
2134 uint8_t addr_type;
2135
2136 acm_get_gid((struct acm_port *)ep->port->port, 0, &sgid);
2137
2138 /* Search for endpoint's SLID */
2139 while (fgets(s, sizeof s, f)) {
2140 if (s[0] == '#')
2141 continue;
2142 if (!(p = strtok_r(s, " \n", &ptr)))
2143 continue; /* ignore blank lines */
2144
2145 if (strncmp(p, "Switch", sizeof("Switch") - 1) &&
2146 strncmp(p, "Channel", sizeof("Channel") - 1) &&
2147 strncmp(p, "Router", sizeof("Router") - 1))
2148 continue;
2149
2150 if (!strncmp(p, "Channel", sizeof("Channel") - 1)) {
2151 p = strtok_r(NULL, " ", &ptr); /* skip 'Adapter' */
2152 if (!p)
2153 continue;
2154 }
2155
2156 p_guid = strtok_r(NULL, ",", &ptr);
2157 if (!p_guid)
2158 continue;
2159
2160 guid = (uint64_t) strtoull(p_guid, NULL, 16);
2161 if (guid != be64toh(sgid.global.interface_id))
2162 continue;
2163
2164 ptr = strstr(ptr, "base LID");
2165 if (!ptr)
2166 continue;
2167 ptr += sizeof("base LID");
2168 p_lid = strtok_r(NULL, ",", &ptr);
2169 if (!p_lid)
2170 continue;
2171
2172 lid = (uint16_t) strtoul(p_lid, NULL, 0);
2173 if (lid != ep->port->lid)
2174 continue;
2175
2176 ibv_query_port(ep->port->dev->verbs, ep->port->port_num, &attr);
2177 ret = 0;
2178 break;
2179 }
2180
2181 while (fgets(s, sizeof s, f)) {
2182 if (s[0] == '#')
2183 continue;
2184 if (!(p = strtok_r(s, " \n", &ptr)))
2185 continue; /* ignore blank lines */
2186
2187 if (!strncmp(p, "Switch", sizeof("Switch") - 1) ||
2188 !strncmp(p, "Channel", sizeof("Channel") - 1) ||
2189 !strncmp(p, "Router", sizeof("Router") - 1))
2190 break;
2191
2192 dlid = strtoul(p, NULL, 0);
2193 net_dlid = htobe16(dlid);
2194
2195 p = strtok_r(NULL, ":", &ptr);
2196 if (!p)
2197 continue;
2198 if (strcmp(p, "UNREACHABLE") == 0)
2199 continue;
2200 sl = atoi(p);
2201
2202 p = strtok_r(NULL, ":", &ptr);
2203 if (!p)
2204 continue;
2205 mtu = atoi(p);
2206
2207 p = strtok_r(NULL, ":", &ptr);
2208 if (!p)
2209 continue;
2210 rate = atoi(p);
2211
2212 if (!lid2guid[dlid]) {
2213 acm_log(0, "ERROR - dlid %u not found in lid2guid table\n", dlid);
2214 continue;
2215 }
2216
2217 dgid.global.subnet_prefix = sgid.global.subnet_prefix;
2218 dgid.global.interface_id = lid2guid[dlid];
2219
2220 for (i = 0; i < 2; i++) {
2221 memset(addr, 0, ACM_MAX_ADDRESS);
2222 if (i == 0) {
2223 addr_type = ACM_ADDRESS_LID;
2224 memcpy(addr, &net_dlid, sizeof net_dlid);
2225 } else {
2226 addr_type = ACM_ADDRESS_GID;
2227 memcpy(addr, &dgid, sizeof(dgid));
2228 }
2229 dest = acmp_acquire_dest(ep, addr_type, addr);
2230 if (!dest) {
2231 acm_log(0, "ERROR - unable to create dest\n");
2232 break;
2233 }
2234
2235 dest->path.sgid = sgid;
2236 dest->path.slid = htobe16(ep->port->lid);
2237 dest->path.dgid = dgid;
2238 dest->path.dlid = net_dlid;
2239 dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
2240 dest->path.pkey = htobe16(ep->pkey);
2241 dest->path.mtu = (uint8_t) mtu;
2242 dest->path.rate = (uint8_t) rate;
2243 dest->path.qosclass_sl = htobe16((uint16_t) sl & 0xF);
2244 if (dlid == ep->port->lid) {
2245 dest->path.packetlifetime = 0;
2246 dest->addr_timeout = (uint64_t)~0ULL;
2247 dest->route_timeout = (uint64_t)~0ULL;
2248 } else {
2249 dest->path.packetlifetime = attr.subnet_timeout;
2250 dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
2251 dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
2252 }
2253 dest->remote_qpn = 1;
2254 dest->state = ACMP_READY;
2255 acmp_put_dest(dest);
2256 acm_log(1, "added cached dest %s\n", dest->name);
2257 }
2258 }
2259 return ret;
2260 }
2261
2262 static int acmp_parse_osm_fullv1(struct acmp_ep *ep)
2263 {
2264 FILE *f;
2265 __be64 *lid2guid;
2266 int ret = 1;
2267
2268 if (!(f = fopen(route_data_file, "r"))) {
2269 acm_log(0, "ERROR - couldn't open %s\n", route_data_file);
2270 return ret;
2271 }
2272
2273 lid2guid = calloc(IB_LID_MCAST_START, sizeof(*lid2guid));
2274 if (!lid2guid) {
2275 acm_log(0, "ERROR - no memory for path record parsing\n");
2276 goto err;
2277 }
2278
2279 acmp_parse_osm_fullv1_lid2guid(f, lid2guid);
2280 rewind(f);
2281 ret = acmp_parse_osm_fullv1_paths(f, lid2guid, ep);
2282 free(lid2guid);
2283 err:
2284 fclose(f);
2285 return ret;
2286 }
2287
2288 static void acmp_parse_hosts_file(struct acmp_ep *ep)
2289 {
2290 FILE *f;
2291 char s[120];
2292 char addr[INET6_ADDRSTRLEN], gid[INET6_ADDRSTRLEN];
2293 uint8_t name[ACM_MAX_ADDRESS];
2294 struct in6_addr ip_addr, ib_addr;
2295 struct acmp_dest *dest, *gid_dest;
2296 uint8_t addr_type;
2297
2298 if (!(f = fopen(addr_data_file, "r"))) {
2299 acm_log(0, "ERROR - couldn't open %s\n", addr_data_file);
2300 return;
2301 }
2302
2303 while (fgets(s, sizeof s, f)) {
2304 if (s[0] == '#')
2305 continue;
2306
2307 if (sscanf(s, "%45s%45s", addr, gid) != 2)
2308 continue;
2309
2310 acm_log(2, "%s", s);
2311 if (inet_pton(AF_INET6, gid, &ib_addr) <= 0) {
2312 acm_log(0, "ERROR - %s is not IB GID\n", gid);
2313 continue;
2314 }
2315 memset(name, 0, ACM_MAX_ADDRESS);
2316 if (inet_pton(AF_INET, addr, &ip_addr) > 0) {
2317 addr_type = ACM_ADDRESS_IP;
2318 memcpy(name, &ip_addr, 4);
2319 } else if (inet_pton(AF_INET6, addr, &ip_addr) > 0) {
2320 addr_type = ACM_ADDRESS_IP6;
2321 memcpy(name, &ip_addr, sizeof(ip_addr));
2322 } else {
2323 addr_type = ACM_ADDRESS_NAME;
2324 strncpy((char *)name, addr, ACM_MAX_ADDRESS);
2325 }
2326
2327 dest = acmp_acquire_dest(ep, addr_type, name);
2328 if (!dest) {
2329 acm_log(0, "ERROR - unable to create dest %s\n", addr);
2330 continue;
2331 }
2332
2333 memset(name, 0, ACM_MAX_ADDRESS);
2334 memcpy(name, &ib_addr, sizeof(ib_addr));
2335 gid_dest = acmp_get_dest(ep, ACM_ADDRESS_GID, name);
2336 if (gid_dest) {
2337 dest->path = gid_dest->path;
2338 dest->state = ACMP_READY;
2339 acmp_put_dest(gid_dest);
2340 } else {
2341 memcpy(&dest->path.dgid, &ib_addr, 16);
2342 //ibv_query_gid(ep->port->dev->verbs, ep->port->port_num,
2343 // 0, &dest->path.sgid);
2344 dest->path.slid = htobe16(ep->port->lid);
2345 dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
2346 dest->path.pkey = htobe16(ep->pkey);
2347 dest->state = ACMP_ADDR_RESOLVED;
2348 }
2349
2350 dest->remote_qpn = 1;
2351 dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
2352 dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
2353 acmp_put_dest(dest);
2354 acm_log(1, "added host %s address type %d IB GID %s\n",
2355 addr, addr_type, gid);
2356 }
2357
2358 fclose(f);
2359 }
2360
2361 /*
2362 * We currently require that the routing data be preloaded in order to
2363 * load the address data. This is backwards from normal operation, which
2364 * usually resolves the address before the route.
2365 */
2366 static void acmp_ep_preload(struct acmp_ep *ep)
2367 {
2368 switch (route_preload) {
2369 case ACMP_ROUTE_PRELOAD_OSM_FULL_V1:
2370 if (acmp_parse_osm_fullv1(ep))
2371 acm_log(0, "ERROR - failed to preload EP\n");
2372 break;
2373 default:
2374 break;
2375 }
2376
2377 switch (addr_preload) {
2378 case ACMP_ADDR_PRELOAD_HOSTS:
2379 acmp_parse_hosts_file(ep);
2380 break;
2381 default:
2382 break;
2383 }
2384 }
2385
2386 /* rwlock must be held write-locked */
2387 static int __acmp_add_addr(const struct acm_address *addr, struct acmp_ep *ep,
2388 void **addr_context)
2389 {
2390 struct acmp_dest *dest;
2391 struct acmp_addr_ctx *addr_ctx;
2392 int i;
2393
2394 for (i = 0; (i < ep->nmbr_ep_addrs) &&
2395 (ep->addr_info[i].type != ACM_ADDRESS_INVALID); i++)
2396 ;
2397
2398 if (i == ep->nmbr_ep_addrs) {
2399 struct acmp_addr *new_info;
2400
2401 new_info = realloc(ep->addr_info, (i + 1) * sizeof(*ep->addr_info));
2402 if (!new_info) {
2403 acm_log(0, "ERROR - no more space for local address\n");
2404 return -1;
2405 }
2406 ep->addr_info = new_info;
2407 /* Added memory is not initialized */
2408 memset(ep->addr_info + i, 0, sizeof(*ep->addr_info));
2409 ++ep->nmbr_ep_addrs;
2410 }
2411 ep->addr_info[i].type = addr->type;
2412 memcpy(&ep->addr_info[i].info, &addr->info, sizeof(addr->info));
2413 memcpy(&ep->addr_info[i].addr, addr, sizeof(*addr));
2414 ep->addr_info[i].ep = ep;
2415
2416 addr_ctx = malloc(sizeof(*addr_ctx));
2417 if (!addr_ctx) {
2418 acm_log(0, "ERROR - unable to alloc address context struct\n");
2419 return -1;
2420 }
2421 addr_ctx->ep = ep;
2422 addr_ctx->addr_inx = i;
2423
2424 if (loopback_prot != ACMP_LOOPBACK_PROT_LOCAL) {
2425 *addr_context = addr_ctx;
2426 return 0;
2427 }
2428
2429 dest = acmp_acquire_dest(ep, addr->type, (uint8_t *)addr->info.addr);
2430 if (!dest) {
2431 acm_log(0, "ERROR - unable to create loopback dest %s\n",
2432 addr->id_string);
2433 memset(&ep->addr_info[i], 0, sizeof(ep->addr_info[i]));
2434 free(addr_ctx);
2435 return -1;
2436 }
2437
2438 acm_get_gid((struct acm_port *) ep->port->port, 0, &dest->path.sgid);
2439 dest->path.dgid = dest->path.sgid;
2440 dest->path.dlid = dest->path.slid = htobe16(ep->port->lid);
2441 dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
2442 dest->path.pkey = htobe16(ep->pkey);
2443 dest->path.mtu = (uint8_t) ep->port->mtu;
2444 dest->path.rate = (uint8_t) ep->port->rate;
2445
2446 dest->remote_qpn = ep->qp->qp_num;
2447 dest->addr_timeout = (uint64_t) ~0ULL;
2448 dest->route_timeout = (uint64_t) ~0ULL;
2449 dest->state = ACMP_READY;
2450 acmp_put_dest(dest);
2451 *addr_context = addr_ctx;
2452 acm_log(1, "added loopback dest %s\n", dest->name);
2453
2454 return 0;
2455 }
2456
2457 static int acmp_add_addr(const struct acm_address *addr, void *ep_context,
2458 void **addr_context)
2459 {
2460 struct acmp_ep *ep = ep_context;
2461 int ret;
2462
2463 acm_log(2, "\n");
2464
2465 pthread_rwlock_wrlock(&ep->rwlock);
2466 ret = __acmp_add_addr(addr, ep, addr_context);
2467 pthread_rwlock_unlock(&ep->rwlock);
2468
2469 return ret;
2470 }
2471
2472 static void acmp_remove_addr(void *addr_context)
2473 {
2474 struct acmp_addr_ctx *addr_ctx = addr_context;
2475 struct acmp_addr *address = addr_ctx->ep->addr_info + addr_ctx->addr_inx;
2476 struct acmp_device *dev;
2477 struct acmp_dest *dest;
2478 struct acmp_ep *ep;
2479 int i;
2480
2481 acm_log(2, "\n");
2482
2483 /*
2484 * The address may be a local destination address. If so,
2485 * delete it from the cache.
2486 */
2487
2488 pthread_mutex_lock(&acmp_dev_lock);
2489 list_for_each(&acmp_dev_list, dev, entry) {
2490 pthread_mutex_unlock(&acmp_dev_lock);
2491
2492 for (i = 0; i < dev->port_cnt; i++) {
2493 struct acmp_port *port = &dev->port[i];
2494
2495 pthread_mutex_lock(&port->lock);
2496 list_for_each(&port->ep_list, ep, entry) {
2497 pthread_mutex_unlock(&port->lock);
2498 dest = acmp_get_dest(ep, address->type, address->addr.info.addr);
2499 if (dest) {
2500 acm_log(2, "Found a dest addr, deleting it\n");
2501 pthread_mutex_lock(&ep->lock);
2502 acmp_remove_dest(ep, dest);
2503 pthread_mutex_unlock(&ep->lock);
2504 }
2505 pthread_mutex_lock(&port->lock);
2506 }
2507 pthread_mutex_unlock(&port->lock);
2508 }
2509 pthread_mutex_lock(&acmp_dev_lock);
2510 }
2511 pthread_mutex_unlock(&acmp_dev_lock);
2512
2513 memset(address, 0, sizeof(*address));
2514 free(addr_ctx);
2515 }
2516
2517 static struct acmp_port *acmp_get_port(struct acm_endpoint *endpoint)
2518 {
2519 struct acmp_device *dev;
2520
2521 acm_log(1, "dev 0x%" PRIx64 " port %d pkey 0x%x\n",
2522 be64toh(endpoint->port->dev->dev_guid),
2523 endpoint->port->port_num, endpoint->pkey);
2524
2525 list_for_each(&acmp_dev_list, dev, entry) {
2526 if (dev->guid == endpoint->port->dev->dev_guid)
2527 return &dev->port[endpoint->port->port_num - 1];
2528 }
2529
2530 return NULL;
2531 }
2532
2533 static struct acmp_ep *
2534 acmp_get_ep(struct acmp_port *port, struct acm_endpoint *endpoint)
2535 {
2536 struct acmp_ep *ep;
2537
2538 acm_log(1, "dev 0x%" PRIx64 " port %d pkey 0x%x\n",
2539 be64toh(endpoint->port->dev->dev_guid),
2540 endpoint->port->port_num, endpoint->pkey);
2541
2542 list_for_each(&port->ep_list, ep, entry) {
2543 if (ep->pkey == endpoint->pkey)
2544 return ep;
2545 }
2546
2547 return NULL;
2548 }
2549
2550 static uint16_t acmp_get_pkey_index(struct acm_endpoint *endpoint)
2551 {
2552 struct acmp_port *port;
2553 int i;
2554
2555 port = acmp_get_port(endpoint);
2556 if (!port)
2557 return 0;
2558 i = ibv_get_pkey_index(port->dev->verbs, port->port_num,
2559 htobe16(endpoint->pkey));
2560 if (i < 0)
2561 return 0;
2562 return i;
2563 }
2564
2565 static void acmp_close_endpoint(void *ep_context)
2566 {
2567
2568 struct acmp_ep *ep = ep_context;
2569
2570 acm_log(1, "%s %d pkey 0x%04x\n",
2571 ep->port->dev->verbs->device->name,
2572 ep->port->port_num, ep->pkey);
2573
2574 ep->endpoint = NULL;
2575 }
2576
2577 static struct acmp_ep *
2578 acmp_alloc_ep(struct acmp_port *port, struct acm_endpoint *endpoint)
2579 {
2580 struct acmp_ep *ep;
2581 int i;
2582
2583 acm_log(1, "\n");
2584 ep = calloc(1, sizeof *ep);
2585 if (!ep)
2586 return NULL;
2587
2588 ep->port = port;
2589 ep->endpoint = endpoint;
2590 ep->pkey = endpoint->pkey;
2591 ep->resolve_queue.credits = resolve_depth;
2592 ep->resp_queue.credits = send_depth;
2593 list_head_init(&ep->resolve_queue.pending);
2594 list_head_init(&ep->resp_queue.pending);
2595 list_head_init(&ep->active_queue);
2596 list_head_init(&ep->wait_queue);
2597 pthread_mutex_init(&ep->lock, NULL);
2598 sprintf(ep->id_string, "%s-%d-0x%x", port->dev->verbs->device->name,
2599 port->port_num, endpoint->pkey);
2600
2601 if (pthread_rwlock_init(&ep->rwlock, NULL)) {
2602 free(ep);
2603 return NULL;
2604 }
2605 ep->addr_info = NULL;
2606 ep->nmbr_ep_addrs = 0;
2607
2608 for (i = 0; i < ACM_MAX_COUNTER; i++)
2609 atomic_init(&ep->counters[i]);
2610
2611 return ep;
2612 }
2613
2614 static int acmp_open_endpoint(const struct acm_endpoint *endpoint,
2615 void *port_context, void **ep_context)
2616 {
2617 struct acmp_port *port = port_context;
2618 struct acmp_ep *ep;
2619 struct ibv_qp_init_attr init_attr;
2620 struct ibv_qp_attr attr;
2621 int ret, sq_size;
2622
2623 ep = acmp_get_ep(port, (struct acm_endpoint *) endpoint);
2624 if (ep) {
2625 acm_log(2, "endpoint for pkey 0x%x already exists\n", endpoint->pkey);
2626 pthread_mutex_lock(&ep->lock);
2627 ep->endpoint = (struct acm_endpoint *) endpoint;
2628 pthread_mutex_unlock(&ep->lock);
2629 *ep_context = (void *) ep;
2630 return 0;
2631 }
2632
2633 acm_log(2, "creating endpoint for pkey 0x%x\n", endpoint->pkey);
2634 ep = acmp_alloc_ep(port, (struct acm_endpoint *) endpoint);
2635 if (!ep)
2636 return -1;
2637
2638 sprintf(ep->id_string, "%s-%d-0x%x",
2639 port->dev->verbs->device->name,
2640 port->port_num, endpoint->pkey);
2641
2642 sq_size = resolve_depth + send_depth;
2643 ep->cq = ibv_create_cq(port->dev->verbs, sq_size + recv_depth,
2644 ep, port->dev->channel, 0);
2645 if (!ep->cq) {
2646 acm_log(0, "ERROR - failed to create CQ\n");
2647 goto err0;
2648 }
2649
2650 ret = ibv_req_notify_cq(ep->cq, 0);
2651 if (ret) {
2652 acm_log(0, "ERROR - failed to arm CQ\n");
2653 goto err1;
2654 }
2655
2656 memset(&init_attr, 0, sizeof init_attr);
2657 init_attr.cap.max_send_wr = sq_size;
2658 init_attr.cap.max_recv_wr = recv_depth;
2659 init_attr.cap.max_send_sge = 1;
2660 init_attr.cap.max_recv_sge = 1;
2661 init_attr.qp_context = ep;
2662 init_attr.sq_sig_all = 1;
2663 init_attr.qp_type = IBV_QPT_UD;
2664 init_attr.send_cq = ep->cq;
2665 init_attr.recv_cq = ep->cq;
2666 ep->qp = ibv_create_qp(ep->port->dev->pd, &init_attr);
2667 if (!ep->qp) {
2668 acm_log(0, "ERROR - failed to create QP\n");
2669 goto err1;
2670 }
2671
2672 attr.qp_state = IBV_QPS_INIT;
2673 attr.port_num = port->port_num;
2674 attr.pkey_index = acmp_get_pkey_index((struct acm_endpoint *) endpoint);
2675 attr.qkey = ACM_QKEY;
2676 ret = ibv_modify_qp(ep->qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX |
2677 IBV_QP_PORT | IBV_QP_QKEY);
2678 if (ret) {
2679 acm_log(0, "ERROR - failed to modify QP to init\n");
2680 goto err2;
2681 }
2682
2683 attr.qp_state = IBV_QPS_RTR;
2684 ret = ibv_modify_qp(ep->qp, &attr, IBV_QP_STATE);
2685 if (ret) {
2686 acm_log(0, "ERROR - failed to modify QP to rtr\n");
2687 goto err2;
2688 }
2689
2690 attr.qp_state = IBV_QPS_RTS;
2691 attr.sq_psn = 0;
2692 ret = ibv_modify_qp(ep->qp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
2693 if (ret) {
2694 acm_log(0, "ERROR - failed to modify QP to rts\n");
2695 goto err2;
2696 }
2697
2698 ret = acmp_post_recvs(ep);
2699 if (ret)
2700 goto err2;
2701
2702 pthread_mutex_lock(&port->lock);
2703 list_add(&port->ep_list, &ep->entry);
2704 pthread_mutex_unlock(&port->lock);
2705 acmp_ep_preload(ep);
2706 acmp_ep_join(ep);
2707 *ep_context = (void *) ep;
2708 return 0;
2709
2710 err2:
2711 ibv_destroy_qp(ep->qp);
2712 err1:
2713 ibv_destroy_cq(ep->cq);
2714 err0:
2715 free(ep);
2716 return -1;
2717 }
2718
2719 static void acmp_port_up(struct acmp_port *port)
2720 {
2721 struct ibv_port_attr attr;
2722 uint16_t pkey;
2723 __be16 pkey_be;
2724 __be16 sm_lid;
2725 int i, ret;
2726 int instance;
2727
2728 acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
2729 ret = ibv_query_port(port->dev->verbs, port->port_num, &attr);
2730 if (ret) {
2731 acm_log(0, "ERROR - unable to get port attribute\n");
2732 return;
2733 }
2734
2735 port->mtu = attr.active_mtu;
2736 port->rate = acm_get_rate(attr.active_width, attr.active_speed);
2737 if (attr.subnet_timeout >= 8)
2738 port->subnet_timeout = 1 << (attr.subnet_timeout - 8);
2739
2740 port->lid = attr.lid;
2741 port->lid_mask = 0xffff - ((1 << attr.lmc) - 1);
2742
2743 port->sa_dest.av.src_path_bits = 0;
2744 port->sa_dest.av.dlid = attr.sm_lid;
2745 port->sa_dest.av.sl = attr.sm_sl;
2746 port->sa_dest.av.port_num = port->port_num;
2747 port->sa_dest.remote_qpn = 1;
2748 sm_lid = htobe16(attr.sm_lid);
2749 acmp_set_dest_addr(&port->sa_dest, ACM_ADDRESS_LID,
2750 (uint8_t *) &sm_lid, sizeof(sm_lid));
2751
2752 instance = atomic_inc(&port->sa_dest.refcnt) - 1;
2753 port->sa_dest.state = ACMP_READY;
2754 for (i = 0; i < attr.pkey_tbl_len; i++) {
2755 ret = ibv_query_pkey(port->dev->verbs, port->port_num, i, &pkey_be);
2756 if (ret)
2757 continue;
2758 pkey = be16toh(pkey_be);
2759 if (!(pkey & 0x7fff))
2760 continue;
2761
2762 /* Determine pkey index for default partition with preference
2763 * for full membership
2764 */
2765 if ((pkey & 0x7fff) == 0x7fff) {
2766 port->default_pkey_ix = i;
2767 break;
2768 }
2769 }
2770
2771 port->state = IBV_PORT_ACTIVE;
2772 acm_log(1, "%s %d %d is up\n", port->dev->verbs->device->name, port->port_num, instance);
2773 }
2774
2775 static void acmp_port_down(struct acmp_port *port)
2776 {
2777 int instance;
2778
2779 acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
2780 pthread_mutex_lock(&port->lock);
2781 port->state = IBV_PORT_DOWN;
2782 pthread_mutex_unlock(&port->lock);
2783
2784 /*
2785 * We wait for the SA destination to be released. We could use an
2786 * event instead of a sleep loop, but it's not worth it given how
2787 * infrequently we should be processing a port down event in practice.
2788 */
2789 instance = atomic_dec(&port->sa_dest.refcnt);
2790 if (instance == 1) {
2791 pthread_mutex_lock(&port->sa_dest.lock);
2792 port->sa_dest.state = ACMP_INIT;
2793 pthread_mutex_unlock(&port->sa_dest.lock);
2794 }
2795 acm_log(1, "%s %d %d is down\n", port->dev->verbs->device->name, port->port_num, instance);
2796 }
2797
2798 static int acmp_open_port(const struct acm_port *cport, void *dev_context,
2799 void **port_context)
2800 {
2801 struct acmp_device *dev = dev_context;
2802 struct acmp_port *port;
2803
2804 if (cport->port_num < 1 || cport->port_num > dev->port_cnt) {
2805 acm_log(0, "Error: port_num %d is out of range (max %d)\n",
2806 cport->port_num, dev->port_cnt);
2807 return -1;
2808 }
2809
2810 port = &dev->port[cport->port_num - 1];
2811 pthread_mutex_lock(&port->lock);
2812 port->port = cport;
2813 port->state = IBV_PORT_DOWN;
2814 pthread_mutex_unlock(&port->lock);
2815 acmp_port_up(port);
2816 *port_context = port;
2817 return 0;
2818 }
2819
2820 static void acmp_close_port(void *port_context)
2821 {
2822 struct acmp_port *port = port_context;
2823
2824 acmp_port_down(port);
2825 pthread_mutex_lock(&port->lock);
2826 port->port = NULL;
2827 pthread_mutex_unlock(&port->lock);
2828 }
2829
2830 static void acmp_init_port(struct acmp_port *port, struct acmp_device *dev,
2831 uint8_t port_num)
2832 {
2833 acm_log(1, "%s %d\n", dev->verbs->device->name, port_num);
2834 port->dev = dev;
2835 port->port_num = port_num;
2836 pthread_mutex_init(&port->lock, NULL);
2837 list_head_init(&port->ep_list);
2838 acmp_init_dest(&port->sa_dest, ACM_ADDRESS_LID, NULL, 0);
2839 port->state = IBV_PORT_DOWN;
2840 }
2841
2842 static int acmp_open_dev(const struct acm_device *device, void **dev_context)
2843 {
2844 struct acmp_device *dev;
2845 size_t size;
2846 struct ibv_device_attr attr;
2847 int i, ret;
2848 struct ibv_context *verbs;
2849
2850 acm_log(1, "dev_guid 0x%" PRIx64 " %s\n", be64toh(device->dev_guid),
2851 device->verbs->device->name);
2852
2853 list_for_each(&acmp_dev_list, dev, entry) {
2854 if (dev->guid == device->dev_guid) {
2855 acm_log(2, "dev_guid 0x%" PRIx64 " already exits\n",
2856 be64toh(device->dev_guid));
2857 *dev_context = dev;
2858 dev->device = device;
2859 return 0;
2860 }
2861 }
2862
2863 /* We need to release the core device structure when device close is
2864 * called. But this provider does not support dynamic add/removal of
2865 * devices/ports/endpoints. To avoid use-after-free issues, we open
2866 * our own verbs context, rather than using the one in the core
2867 * device structure.
2868 */
2869 verbs = ibv_open_device(device->verbs->device);
2870 if (!verbs) {
2871 acm_log(0, "ERROR - opening device %s\n",
2872 device->verbs->device->name);
2873 goto err;
2874 }
2875
2876 ret = ibv_query_device(verbs, &attr);
2877 if (ret) {
2878 acm_log(0, "ERROR - ibv_query_device (%s) %d\n",
2879 verbs->device->name, ret);
2880 goto err;
2881 }
2882
2883 size = sizeof(*dev) + sizeof(struct acmp_port) * attr.phys_port_cnt;
2884 dev = (struct acmp_device *) calloc(1, size);
2885 if (!dev)
2886 goto err;
2887
2888 dev->verbs = verbs;
2889 dev->device = device;
2890 dev->port_cnt = attr.phys_port_cnt;
2891
2892 dev->pd = ibv_alloc_pd(dev->verbs);
2893 if (!dev->pd) {
2894 acm_log(0, "ERROR - unable to allocate PD\n");
2895 goto err1;
2896 }
2897
2898 dev->channel = ibv_create_comp_channel(dev->verbs);
2899 if (!dev->channel) {
2900 acm_log(0, "ERROR - unable to create comp channel\n");
2901 goto err2;
2902 }
2903
2904 for (i = 0; i < dev->port_cnt; i++) {
2905 acmp_init_port(&dev->port[i], dev, i + 1);
2906 }
2907
2908 if (pthread_create(&dev->comp_thread_id, NULL, acmp_comp_handler, dev)) {
2909 acm_log(0, "Error -- failed to create the comp thread for dev %s",
2910 dev->verbs->device->name);
2911 goto err3;
2912 }
2913
2914 pthread_mutex_lock(&acmp_dev_lock);
2915 list_add(&acmp_dev_list, &dev->entry);
2916 pthread_mutex_unlock(&acmp_dev_lock);
2917 dev->guid = device->dev_guid;
2918 *dev_context = dev;
2919
2920 acm_log(1, "%s opened\n", dev->verbs->device->name);
2921 return 0;
2922
2923 err3:
2924 ibv_destroy_comp_channel(dev->channel);
2925 err2:
2926 ibv_dealloc_pd(dev->pd);
2927 err1:
2928 free(dev);
2929 err:
2930 return -1;
2931 }
2932
2933 static void acmp_close_dev(void *dev_context)
2934 {
2935 struct acmp_device *dev = dev_context;
2936
2937 acm_log(1, "dev_guid 0x%" PRIx64 "\n", be64toh(dev->device->dev_guid));
2938 dev->device = NULL;
2939 }
2940
2941 static void acmp_set_options(void)
2942 {
2943 FILE *f;
2944 char s[120];
2945 char opt[32], value[256];
2946 const char *opts_file = acm_get_opts_file();
2947
2948 if (!(f = fopen(opts_file, "r")))
2949 return;
2950
2951 while (fgets(s, sizeof s, f)) {
2952 if (s[0] == '#')
2953 continue;
2954
2955 if (sscanf(s, "%31s%255s", opt, value) != 2)
2956 continue;
2957
2958 if (!strcasecmp("addr_prot", opt))
2959 addr_prot = acmp_convert_addr_prot(value);
2960 else if (!strcasecmp("addr_timeout", opt))
2961 addr_timeout = atoi(value);
2962 else if (!strcasecmp("route_prot", opt))
2963 route_prot = acmp_convert_route_prot(value);
2964 else if (!strcmp("route_timeout", opt))
2965 route_timeout = atoi(value);
2966 else if (!strcasecmp("loopback_prot", opt))
2967 loopback_prot = acmp_convert_loopback_prot(value);
2968 else if (!strcasecmp("timeout", opt))
2969 timeout = atoi(value);
2970 else if (!strcasecmp("retries", opt))
2971 retries = atoi(value);
2972 else if (!strcasecmp("resolve_depth", opt))
2973 resolve_depth = atoi(value);
2974 else if (!strcasecmp("send_depth", opt))
2975 send_depth = atoi(value);
2976 else if (!strcasecmp("recv_depth", opt))
2977 recv_depth = atoi(value);
2978 else if (!strcasecmp("min_mtu", opt))
2979 min_mtu = acm_convert_mtu(atoi(value));
2980 else if (!strcasecmp("min_rate", opt))
2981 min_rate = acm_convert_rate(atoi(value));
2982 else if (!strcasecmp("route_preload", opt))
2983 route_preload = acmp_convert_route_preload(value);
2984 else if (!strcasecmp("route_data_file", opt))
2985 strcpy(route_data_file, value);
2986 else if (!strcasecmp("addr_preload", opt))
2987 addr_preload = acmp_convert_addr_preload(value);
2988 else if (!strcasecmp("addr_data_file", opt))
2989 strcpy(addr_data_file, value);
2990 }
2991
2992 fclose(f);
2993 }
2994
2995 static void acmp_log_options(void)
2996 {
2997 acm_log(0, "address resolution %d\n", addr_prot);
2998 acm_log(0, "address timeout %d\n", addr_timeout);
2999 acm_log(0, "route resolution %d\n", route_prot);
3000 acm_log(0, "route timeout %d\n", route_timeout);
3001 acm_log(0, "loopback resolution %d\n", loopback_prot);
3002 acm_log(0, "timeout %d ms\n", timeout);
3003 acm_log(0, "retries %d\n", retries);
3004 acm_log(0, "resolve depth %d\n", resolve_depth);
3005 acm_log(0, "send depth %d\n", send_depth);
3006 acm_log(0, "receive depth %d\n", recv_depth);
3007 acm_log(0, "minimum mtu %d\n", min_mtu);
3008 acm_log(0, "minimum rate %d\n", min_rate);
3009 acm_log(0, "route preload %d\n", route_preload);
3010 acm_log(0, "route data file %s\n", route_data_file);
3011 acm_log(0, "address preload %d\n", addr_preload);
3012 acm_log(0, "address data file %s\n", addr_data_file);
3013 }
3014
3015 static void __attribute__((constructor)) acmp_init(void)
3016 {
3017 acmp_set_options();
3018
3019 acmp_log_options();
3020
3021 atomic_init(&g_tid);
3022 atomic_init(&wait_cnt);
3023 pthread_mutex_init(&acmp_dev_lock, NULL);
3024 event_init(&timeout_event);
3025
3026 umad_init();
3027
3028 acm_log(1, "starting timeout/retry thread\n");
3029 if (pthread_create(&retry_thread_id, NULL, acmp_retry_handler, NULL)) {
3030 acm_log(0, "Error: failed to create the retry thread");
3031 retry_thread_started = 0;
3032 return;
3033 }
3034
3035 acmp_initialized = 1;
3036 }
3037
3038 int provider_query(struct acm_provider **provider, uint32_t *version)
3039 {
3040 acm_log(1, "\n");
3041
3042 if (!acmp_initialized)
3043 return -1;
3044
3045 if (provider)
3046 *provider = &def_prov;
3047 if (version)
3048 *version = ACM_PROV_VERSION;
3049
3050 return 0;
3051 }
3052