Debian Codegraph @

prov/verbs/src/windows/verbs_nd_ibv.c

raw
1 /*
2 * Copyright (c) 2022 Canon Medical Research USA, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <assert.h>
34 #include "ndspi.h"
35 #include "verbs_nd.h"
36
37 static enum ibv_wc_opcode vrb_ndrequest2opcode(enum ND2_REQUEST_TYPE rq)
38 {
39 static enum ibv_wc_opcode opcodes[] = {
40 [Nd2RequestTypeReceive] = IBV_WC_RECV,
41 [Nd2RequestTypeSend] = IBV_WC_SEND,
42 [Nd2RequestTypeRead] = IBV_WC_RDMA_READ,
43 [Nd2RequestTypeWrite] = IBV_WC_RDMA_WRITE
44 };
45 return (rq < ARRAY_SIZE(opcodes)) ? opcodes[rq] : IBV_WC_SEND;
46 }
47
48 struct ibv_device **ibv_get_device_list(int *num_devices)
49 {
50 VRB_TRACE(FI_LOG_FABRIC, "\n");
51
52 if (num_devices) {
53 assert(nd_infra.adapters.count <= INT_MAX);
54 *num_devices = (int)nd_infra.adapters.count;
55 }
56 return nd_infra.adapters.device_list;
57 }
58
59 void ibv_free_device_list(struct ibv_device **list)
60 {
61 VRB_TRACE(FI_LOG_FABRIC, "\n");
62 // Since we are keeping state statically, there is nothing to free.
63 }
64
65 struct ibv_context *ibv_open_device(struct ibv_device *device)
66 {
67 VRB_TRACE(FI_LOG_FABRIC, "\n");
68 return nd_get_context_by_device(device);
69 }
70
71 int ibv_close_device(struct ibv_context *context)
72 {
73 VRB_TRACE(FI_LOG_FABRIC, "\n");
74 // Since we are keeping state statically, there is nothing to close.
75 return 0;
76 }
77
78 // The IND2Adapter::Query method doesn't provide data for a number of fields
79 // in the ibv_device_attr structure. The following definitions were inferred
80 // from what ibv_query_device returned on a Linux implementation.
81 #define ND_MAX_CQ_CNT (1 << 24)
82 #define ND_MAX_QP_CNT (1 << 18)
83 #define ND_MAX_SRQ_CNT (1 << 23)
84 #define ND_MAX_MR_CNT (1 << 24)
85
86 int ibv_query_device(struct ibv_context *context,
87 struct ibv_device_attr *device_attr)
88 {
89 struct nd_adapter *adapter;
90 ND2_ADAPTER_INFO *nd_attr;
91
92 VRB_TRACE(FI_LOG_FABRIC, "\n");
93
94 if ((!context) || (!device_attr)) {
95 errno = EINVAL;
96 return errno;
97 }
98
99 adapter = nd_get_adapter_by_context(context);
100 if (!adapter) {
101 errno = ENODEV;
102 return errno;
103 }
104
105 nd_attr = &adapter->info;
106 memset(device_attr, 0, sizeof(struct ibv_device_attr));
107 strcpy(device_attr->fw_ver, "(null)");
108 device_attr->node_guid = nd_attr->AdapterId;
109 device_attr->sys_image_guid = nd_attr->AdapterId;
110 device_attr->max_mr_size = nd_attr->MaxRegistrationSize;
111 device_attr->vendor_id = nd_attr->VendorId;
112 device_attr->vendor_part_id = nd_attr->DeviceId;
113 device_attr->max_qp = ND_MAX_QP_CNT;
114 device_attr->max_qp_wr = nd_attr->MaxInitiatorQueueDepth;
115 device_attr->max_sge = nd_attr->MaxInitiatorSge;
116 device_attr->max_sge_rd = nd_attr->MaxReadSge;
117 device_attr->max_cq = ND_MAX_CQ_CNT;
118 device_attr->max_cqe = nd_attr->MaxCompletionQueueDepth;
119 device_attr->max_mr = ND_MAX_MR_CNT;
120 device_attr->max_qp_rd_atom = nd_attr->MaxInboundReadLimit;
121 device_attr->max_qp_init_rd_atom = nd_attr->MaxOutboundReadLimit;
122 device_attr->max_srq =
123 nd_attr->MaxSharedReceiveQueueDepth ? ND_MAX_SRQ_CNT : 0;
124 device_attr->max_srq_wr = nd_attr->MaxSharedReceiveQueueDepth;
125 device_attr->phys_port_cnt = 1;
126
127 return 0;
128 }
129
130 const char *ibv_get_device_name(struct ibv_device *device)
131 {
132 VRB_TRACE(FI_LOG_FABRIC, "\n");
133
134 if (!device)
135 return NULL;
136
137 return device->name;
138 }
139
140 // infiniband/verbs.h defines ibv_query_port to be ___ibv_query_port
141 int ___ibv_query_port(struct ibv_context *context, uint8_t port_num,
142 struct ibv_port_attr *port_attr)
143 {
144 struct nd_adapter *adapter;
145
146 VRB_TRACE(FI_LOG_FABRIC, "\n");
147
148 if ((!context) || (!port_attr)) {
149 errno = EINVAL;
150 return errno;
151 }
152
153 adapter = nd_get_adapter_by_context(context);
154 if (!adapter) {
155 errno = ENODEV;
156 return errno;
157 }
158
159 memset(port_attr, 0, sizeof(struct ibv_port_attr));
160 port_attr->state = IBV_PORT_ACTIVE;
161 port_attr->max_mtu = IBV_MTU_4096;
162 port_attr->active_mtu = IBV_MTU_4096;
163 port_attr->gid_tbl_len = 0;
164 port_attr->max_msg_sz = adapter->info.MaxTransferLength;
165 port_attr->pkey_tbl_len = 0;
166 port_attr->lid = 0;
167 port_attr->sm_sl = 0;
168 port_attr->active_width = 2; // 4x
169 port_attr->active_speed = 32; // 25 Gbps
170 port_attr->phys_state =
171 5; // IB_PORT_PHYS_STATE_LINKUP from "infiniband/iba/ib_types.h"
172 port_attr->link_layer = IBV_LINK_LAYER_ETHERNET;
173
174 return 0;
175 }
176
177 int ibv_query_gid(struct ibv_context *context, uint8_t port_num, int index,
178 union ibv_gid *gid)
179 {
180 VRB_TRACE(FI_LOG_FABRIC, "\n");
181
182 // little endian representation of link-local address prefix.
183 gid->global.subnet_prefix = 0x80fe;
184 gid->global.interface_id = 0;
185
186 return 0;
187 }
188
189 int ibv_query_pkey(struct ibv_context *context, uint8_t port_num, int index,
190 __be16 *pkey)
191 {
192 VRB_TRACE(FI_LOG_FABRIC, "\n");
193
194 *pkey = 0xffff;
195
196 return 0;
197 }
198
199 struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
200 {
201 struct ibv_pd *pd;
202
203 VRB_TRACE(FI_LOG_FABRIC, "\n");
204
205 if (!context) {
206 errno = EINVAL;
207 return NULL;
208 }
209
210 pd = calloc(1, sizeof(*pd));
211 if (!pd) {
212 errno = ENOMEM;
213 return NULL;
214 }
215
216 pd->context = context;
217
218 return pd;
219 }
220
221 int ibv_dealloc_pd(struct ibv_pd *pd)
222 {
223 VRB_TRACE(FI_LOG_FABRIC, "\n");
224
225 if (!pd) {
226 errno = EINVAL;
227 return errno;
228 }
229
230 free(pd);
231 return 0;
232 }
233
234 struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context)
235 {
236 struct nd_comp_channel *channel_nd;
237 int ret;
238
239 VRB_TRACE(FI_LOG_FABRIC, "\n");
240
241 if (!context) {
242 errno = EINVAL;
243 return NULL;
244 }
245
246 channel_nd = calloc(1, sizeof(*channel_nd));
247 if (!channel_nd) {
248 errno = ENOMEM;
249 return NULL;
250 }
251
252 ret = dlistfd_head_init(&channel_nd->q);
253 if (ret) {
254 free(channel_nd);
255 errno = ret;
256 return NULL;
257 }
258 ofi_mutex_init(&channel_nd->q_lock);
259
260 channel_nd->channel.context = context;
261 channel_nd->channel.fd = fd_signal_get(&channel_nd->q.signal);
262
263 return &channel_nd->channel;
264 }
265
266 int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
267 {
268 struct nd_comp_channel *channel_nd;
269
270 VRB_TRACE(FI_LOG_FABRIC, "\n");
271
272 if (!channel) {
273 errno = EINVAL;
274 return errno;
275 }
276
277 if (channel->refcnt) {
278 errno = EBUSY;
279 return -1;
280 }
281
282 channel_nd = container_of(channel, struct nd_comp_channel, channel);
283
284 dlistfd_head_free(&channel_nd->q);
285 ofi_mutex_destroy(&channel_nd->q_lock);
286 free(channel_nd);
287 return 0;
288 }
289
290 struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
291 void *cq_context, struct ibv_comp_channel *channel,
292 int comp_vector)
293 {
294 struct nd_adapter *adapter;
295 struct nd_cq *cq_nd;
296 struct ibv_cq *cq;
297 HRESULT hr;
298
299 VRB_TRACE(FI_LOG_FABRIC, "\n");
300
301 if (!context) {
302 errno = EINVAL;
303 return NULL;
304 }
305
306 adapter = nd_get_adapter_by_context(context);
307 if (!adapter) {
308 errno = ENODEV;
309 return NULL;
310 }
311
312 cq_nd = calloc(1, sizeof(*cq_nd));
313 if (!cq_nd) {
314 errno = ENOMEM;
315 return NULL;
316 }
317
318 if (channel)
319 ++channel->refcnt;
320
321 ofi_mutex_init(&cq_nd->notification.lock);
322 pthread_cond_init(&cq_nd->notification.cond, 0);
323 cq_nd->notification.event_cb = nd_cq_notify_event;
324 cq_nd->notification.error_cb = nd_cq_notify_error;
325
326 cq = &cq_nd->cq;
327 cq->context = context;
328 cq->channel = channel;
329 cq->cq_context = cq_context;
330 cq->cqe = cqe;
331
332 hr = adapter->adapter->lpVtbl->CreateOverlappedFile(adapter->adapter,
333 &cq_nd->ov_file);
334 VRB_DBG(FI_LOG_CQ, "IND2Adapter::CreateOverlappedFile: hr=0x%08lx\n",
335 hr);
336 if (FAILED(hr)) {
337 errno = hresult2fi(hr);
338 goto err1;
339 }
340
341 assert(cq_nd->ov_file && cq_nd->ov_file != INVALID_HANDLE_VALUE);
342
343 BindIoCompletionCallback(cq_nd->ov_file, nd_io_cb, 0);
344
345 hr = adapter->adapter->lpVtbl->CreateCompletionQueue(
346 adapter->adapter, &IID_IND2CompletionQueue, cq_nd->ov_file, cqe,
347 0, 0, &cq_nd->nd2cq);
348 VRB_DBG(FI_LOG_CQ, "IND2Adapter::CreateCompletionQueue: hr=0x%08lx\n",
349 hr);
350 if (FAILED(hr)) {
351 errno = hresult2fi(hr);
352 goto err2;
353 }
354 assert(cq_nd->nd2cq);
355
356 return cq;
357 err2:
358 CloseHandle(cq_nd->ov_file);
359 err1:
360 free(cq_nd);
361 return NULL;
362 }
363
364 int ibv_destroy_cq(struct ibv_cq *cq)
365 {
366 struct nd_cq *cq_nd;
367
368 VRB_TRACE(FI_LOG_FABRIC, "\n");
369
370 if (!cq) {
371 errno = EINVAL;
372 return errno;
373 }
374
375 cq_nd = container_of(cq, struct nd_cq, cq);
376 if (nd_cancel_pending(&cq_nd->notification,
377 (IND2Overlapped *)cq_nd->nd2cq)) {
378 return errno;
379 }
380
381 if (!CloseHandle(cq_nd->ov_file)) {
382 errno = GetLastError();
383 return errno;
384 }
385
386 // There appears to be a bug related to the reference counting for
387 // ComplettionQueue objects in the Mellanox ND driver. CompletionQueue
388 // reference counts are incremented when they are used in the creation
389 // of a QueuePair, but are not always decremented when the QueuePair is
390 // Released. Ideally, we'd assert that that the reference count is zero
391 // here, but we cannot because of that bug.
392 cq_nd->nd2cq->lpVtbl->Release(cq_nd->nd2cq);
393
394 if (cq_nd->cq.channel)
395 --cq_nd->cq.channel->refcnt;
396
397 pthread_cond_destroy(&cq_nd->notification.cond);
398 ofi_mutex_destroy(&cq_nd->notification.lock);
399 free(cq_nd);
400
401 return 0;
402 }
403
404 int ibv_poll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc)
405 {
406 struct nd_cq *cq_nd;
407 int num_results;
408 ND2_RESULT result;
409 ULONG nResults;
410
411 VRB_TRACE(FI_LOG_FABRIC, "\n");
412
413 if (!cq || !wc) {
414 errno = EINVAL;
415 return -1;
416 }
417
418 cq_nd = container_of(cq, struct nd_cq, cq);
419 for (num_results = 0; num_results < num_entries; ++num_results) {
420 nResults = cq_nd->nd2cq->lpVtbl->GetResults(cq_nd->nd2cq,
421 &result, 1);
422
423 if (nResults == 0)
424 break;
425
426 memset(&wc[num_results], 0, sizeof(wc[num_results]));
427 wc[num_results].wr_id = (uint64_t)result.RequestContext;
428 wc[num_results].byte_len = result.BytesTransferred;
429 wc[num_results].status = result.Status;
430 wc[num_results].opcode = vrb_ndrequest2opcode(result.RequestType);
431 FI_LOG(&vrb_prov, result.Status ? FI_LOG_WARN : FI_LOG_DEBUG,
432 FI_LOG_CQ,
433 "ibv_poll_cq: context=0x%016llx, numBytes=%d, "
434 "status=0x%08lx, request=0x%08lx\n",
435 (uint64_t)result.RequestContext, result.BytesTransferred,
436 result.Status, result.RequestType);
437 }
438
439 return num_results;
440 }
441
442 int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only)
443 {
444 struct nd_cq *cq_nd;
445 HRESULT hr;
446 int ret = 0;
447
448 VRB_TRACE(FI_LOG_FABRIC, "\n");
449
450 if (!cq) {
451 errno = EINVAL;
452 return errno;
453 }
454
455 cq_nd = container_of(cq, struct nd_cq, cq);
456
457 ofi_mutex_lock(&cq_nd->notification.lock);
458 if (!cq_nd->notification.cb_pending) {
459 cq_nd->notification.cb_pending = 1;
460 hr = cq_nd->nd2cq->lpVtbl->Notify(
461 cq_nd->nd2cq,
462 solicited_only ? ND_CQ_NOTIFY_SOLICITED :
463 ND_CQ_NOTIFY_ANY,
464 &cq_nd->notification.ov);
465 VRB_DBG(FI_LOG_CQ,
466 "IND2CompletionQueue::Notify: hr=0x%08lx; ov=%p\n", hr,
467 &cq_nd->notification.ov);
468 if (FAILED(hr)) {
469 cq_nd->notification.cb_pending = 0;
470 errno = hresult2fi(hr);
471 ret = errno;
472 }
473 }
474 ofi_mutex_unlock(&cq_nd->notification.lock);
475
476 return ret;
477 }
478
479 int ibv_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
480 void **cq_context)
481 {
482 int ret = 0;
483 struct nd_comp_channel *channel_nd;
484 struct nd_cq *cq_nd = NULL;
485
486 VRB_TRACE(FI_LOG_FABRIC, "\n");
487
488 if ((!channel) || (!cq) || (!cq_context)) {
489 errno = EINVAL;
490 return -1;
491 }
492
493 channel_nd = container_of(channel, struct nd_comp_channel, channel);
494
495 ofi_mutex_lock(&channel_nd->q_lock);
496 if (dlistfd_empty(&channel_nd->q)) {
497 ret = -1;
498 } else {
499 cq_nd = container_of(channel_nd->q.list.next, struct nd_cq,
500 entry);
501 *cq = &cq_nd->cq;
502 *cq_context = cq_nd->cq.cq_context;
503 dlistfd_remove(channel_nd->q.list.next, &channel_nd->q);
504 }
505 ofi_mutex_unlock(&channel_nd->q_lock);
506
507 return ret;
508 }
509
510 void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
511 {
512 VRB_TRACE(FI_LOG_FABRIC, "\n");
513 // N/A
514 }
515
516 struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
517 struct ibv_qp_init_attr *qp_init_attr)
518 {
519 static uint32_t unique_qp_num = 2; // QPN 0 & 1 are reserved by IB spec.
520 struct nd_adapter *adapter;
521 struct nd_qp *qp_nd;
522 struct ibv_qp *qp;
523 HRESULT hr;
524
525 VRB_TRACE(FI_LOG_FABRIC, "\n");
526
527 if ((!pd) || (!qp_init_attr)) {
528 errno = EINVAL;
529 return NULL;
530 }
531
532 adapter = nd_get_adapter_by_context(pd->context);
533 if (qp_init_attr->cap.max_inline_data >
534 adapter->info.MaxInlineDataSize) {
535 errno = ENODEV;
536 return NULL;
537 }
538
539 qp_nd = calloc(1, sizeof(*qp_nd));
540 if (!qp_nd) {
541 errno = ENOMEM;
542 return NULL;
543 }
544
545 qp = &qp_nd->qp;
546 qp->context = pd->context;
547 qp->qp_context = qp_init_attr->qp_context;
548 qp->pd = pd;
549 qp->send_cq = qp_init_attr->send_cq;
550 qp->recv_cq = qp_init_attr->recv_cq;
551 qp->srq = qp_init_attr->srq;
552 qp->qp_num = unique_qp_num++;
553 qp->state = IBV_QPS_RESET;
554 qp->qp_type = qp_init_attr->qp_type;
555 qp_nd->max_inline_data = qp_init_attr->cap.max_inline_data;
556
557 hr = adapter->adapter->lpVtbl->CreateQueuePair(
558 adapter->adapter, &IID_IND2QueuePair,
559 (IUnknown *)((struct nd_cq *)qp_init_attr->recv_cq)->nd2cq,
560 (IUnknown *)((struct nd_cq *)qp_init_attr->send_cq)->nd2cq,
561 qp_init_attr->qp_context, qp_init_attr->cap.max_recv_wr,
562 qp_init_attr->cap.max_send_wr, qp_init_attr->cap.max_recv_sge,
563 qp_init_attr->cap.max_send_sge,
564 qp_init_attr->cap.max_inline_data, (void **)&qp_nd->nd2qp);
565 VRB_DBG(FI_LOG_EP_CTRL, "IND2Adapter::CreateQueuePair: hr=0x%08lx\n",
566 hr);
567 if (FAILED(hr)) {
568 free(qp_nd);
569 errno = hresult2fi(hr);
570 return NULL;
571 }
572
573 return qp;
574 }
575
576 int ibv_destroy_qp(struct ibv_qp *qp)
577 {
578 struct nd_qp *qp_nd;
579 ULONG refcnt;
580
581 VRB_TRACE(FI_LOG_FABRIC, "\n");
582
583 if (!qp) {
584 errno = EINVAL;
585 return errno;
586 }
587
588 qp_nd = container_of(qp, struct nd_qp, qp);
589 refcnt = qp_nd->nd2qp->lpVtbl->Release(qp_nd->nd2qp);
590 assert(!refcnt);
591 free(qp_nd);
592 return 0;
593 }
594
595 int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
596 {
597 VRB_TRACE(FI_LOG_FABRIC, "\n");
598 return 0;
599 }
600
601 int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask,
602 struct ibv_qp_init_attr *init_attr)
603 {
604 VRB_TRACE(FI_LOG_FABRIC, "\n");
605 return 0;
606 }
607
608 // infiniband/verbs.h defines ibv_reg_mr as a macro to call ibv_reg_mr_iova2.
609 struct ibv_mr *ibv_reg_mr_iova2(struct ibv_pd *pd, void *addr, size_t length,
610 uint64_t iova, unsigned int access)
611 {
612 struct nd_adapter *adapter;
613 struct nd_mr *mr_nd;
614 struct ibv_mr *mr;
615 OVERLAPPED ov = { 0 };
616 HRESULT hr;
617 int ret;
618 int flags = 0;
619
620 VRB_TRACE(FI_LOG_FABRIC, "\n");
621
622 if ((!pd) || (!addr)) {
623 errno = EINVAL;
624 return NULL;
625 }
626
627 adapter = nd_get_adapter_by_context(pd->context);
628 if (!adapter) {
629 errno = ENODEV;
630 return NULL;
631 }
632
633 mr_nd = calloc(1, sizeof(*mr_nd));
634 if (!mr_nd) {
635 errno = ENOMEM;
636 goto err1;
637 }
638
639 hr = adapter->adapter->lpVtbl->CreateOverlappedFile(adapter->adapter,
640 &mr_nd->ov_file);
641 VRB_DBG(FI_LOG_MR, "IND2Adapter::CreateOverlappedFile: hr=0x%08lx\n",
642 hr);
643 if (FAILED(hr)) {
644 errno = hresult2fi(hr);
645 goto err2;
646 }
647
648 assert(mr_nd->ov_file && mr_nd->ov_file != INVALID_HANDLE_VALUE);
649
650 hr = adapter->adapter->lpVtbl->CreateMemoryRegion(adapter->adapter,
651 &IID_IND2MemoryRegion,
652 mr_nd->ov_file,
653 &mr_nd->region);
654 VRB_DBG(FI_LOG_MR, "IND2Adapter::CreateMemoryRegion: hr=0x%08lx\n", hr);
655 if (FAILED(hr)) {
656 errno = hresult2fi(hr);
657 goto err3;
658 }
659
660 mr = &mr_nd->mr;
661 mr->context = pd->context;
662 mr->pd = pd;
663 mr->addr = addr;
664 mr->length = length;
665
666 if (access & IBV_ACCESS_LOCAL_WRITE)
667 flags |= ND_MR_FLAG_ALLOW_LOCAL_WRITE;
668 if (access & IBV_ACCESS_REMOTE_READ)
669 flags |= ND_MR_FLAG_ALLOW_REMOTE_READ;
670 if (access & IBV_ACCESS_REMOTE_WRITE)
671 flags |= ND_MR_FLAG_ALLOW_REMOTE_WRITE;
672
673 hr = mr_nd->region->lpVtbl->Register(mr_nd->region, addr, length, flags,
674 &ov);
675 VRB_DBG(FI_LOG_MR,
676 "IND2MemoryRegion::Register: hr=0x%08lx; ov=%p, "
677 "addr=0x%016llx, len=0x%016llx\n",
678 hr, &ov, addr, length);
679 if (FAILED(hr)) {
680 errno = hresult2fi(hr);
681 goto err4;
682 }
683 hr = mr_nd->region->lpVtbl->GetOverlappedResult(mr_nd->region, &ov,
684 TRUE);
685 VRB_DBG(FI_LOG_MR,
686 "IND2MemoryRegion::GetOverlappedResult: hr=0x%08lx; ov=%p\n",
687 hr, &ov);
688 if (FAILED(hr)) {
689 errno = hresult2fi(hr);
690 goto err5;
691 }
692
693 mr->lkey = mr_nd->region->lpVtbl->GetLocalToken(mr_nd->region);
694 mr->rkey = mr_nd->region->lpVtbl->GetRemoteToken(mr_nd->region);
695
696 return mr;
697 err5:
698 mr_nd->region->lpVtbl->CancelOverlappedRequests(mr_nd->region);
699 err4:
700 ret = mr_nd->region->lpVtbl->Release(mr_nd->region);
701 assert(!ret);
702 err3:
703 ret = CloseHandle(mr_nd->ov_file);
704 assert(ret);
705 err2:
706 free(mr_nd);
707 err1:
708 return NULL;
709 }
710
711 int ibv_dereg_mr(struct ibv_mr *mr)
712 {
713 OVERLAPPED ov = { 0 };
714 int ret = 0;
715 struct nd_mr *mr_nd;
716 HRESULT hr;
717 int tmp;
718
719 VRB_TRACE(FI_LOG_FABRIC, "\n");
720
721 if (!mr) {
722 errno = EINVAL;
723 return errno;
724 }
725
726 mr_nd = container_of(mr, struct nd_mr, mr);
727 hr = mr_nd->region->lpVtbl->Deregister(mr_nd->region, &ov);
728 VRB_DBG(FI_LOG_MR, "IND2MemoryRegion::Deregister: hr=0x%08lx; ov=%p\n",
729 hr, &ov);
730 if (FAILED(hr)) {
731 ret = hresult2fi(hr);
732 goto err1;
733 }
734 hr = mr_nd->region->lpVtbl->GetOverlappedResult(mr_nd->region, &ov,
735 TRUE);
736 VRB_DBG(FI_LOG_MR,
737 "IND2MemoryRegion::GetOverlappedResult: hr=0x%08lx; ov=%p\n",
738 hr, &ov);
739 if (FAILED(hr)) {
740 ret = hresult2fi(hr);
741 mr_nd->region->lpVtbl->CancelOverlappedRequests(mr_nd->region);
742 }
743 err1:
744 tmp = CloseHandle(mr_nd->ov_file);
745 assert(tmp);
746 tmp = mr_nd->region->lpVtbl->Release(mr_nd->region);
747 assert(!tmp);
748 free(mr_nd);
749 if (ret)
750 errno = ret;
751 return ret;
752 }
753
754 int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
755 struct ibv_recv_wr **bad_wr)
756 {
757 struct nd_qp *qp_nd;
758 HRESULT hr;
759
760 VRB_TRACE(FI_LOG_FABRIC, "\n");
761
762 if ((!qp) || (!wr) || (!bad_wr)) {
763 errno = EINVAL;
764 return errno;
765 }
766
767 *bad_wr = NULL;
768
769 qp_nd = container_of(qp, struct nd_qp, qp);
770 while (wr) {
771 hr = qp_nd->nd2qp->lpVtbl->Receive(qp_nd->nd2qp,
772 (void *)wr->wr_id,
773 (ND2_SGE *)wr->sg_list,
774 wr->num_sge);
775 VRB_DBG(FI_LOG_EP_DATA,
776 "IND2QueuePair::Receive: hr=0x%08lx, "
777 "context=0x%016llx\n",
778 hr, wr->wr_id);
779 if (FAILED(hr)) {
780 errno = hresult2fi(hr);
781 *bad_wr = wr;
782 return errno;
783 }
784
785 wr = wr->next;
786 }
787
788 return 0;
789 }
790
791 int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr,
792 struct ibv_send_wr **bad_wr)
793 {
794 struct nd_qp *qp_nd;
795 uint32_t flags = 0;
796 HRESULT hr;
797
798 VRB_TRACE(FI_LOG_FABRIC, "\n");
799
800 if ((!qp) || (!wr) || (!bad_wr)) {
801 errno = EINVAL;
802 return errno;
803 }
804
805 *bad_wr = NULL;
806
807 qp_nd = container_of(qp, struct nd_qp, qp);
808 while (wr) {
809 switch (wr->opcode) {
810 case IBV_WR_SEND:
811 flags = (wr->send_flags & IBV_SEND_INLINE) ?
812 ND_OP_FLAG_INLINE :
813 0;
814 hr = qp_nd->nd2qp->lpVtbl->Send(qp_nd->nd2qp,
815 (void *)wr->wr_id,
816 (ND2_SGE *)wr->sg_list,
817 wr->num_sge, flags);
818 VRB_DBG(FI_LOG_EP_DATA,
819 "IND2QueuePair::Send: hr=0x%08lx, nsge=%d, "
820 "addr=0x%016llx, length=%d, lkey=0x%08lx\n",
821 hr, wr->num_sge, wr->sg_list[0].addr,
822 wr->sg_list[0].length, wr->sg_list[0].lkey);
823 break;
824
825 case IBV_WR_RDMA_READ:
826 hr = qp_nd->nd2qp->lpVtbl->Read(
827 qp_nd->nd2qp, (void *)wr->wr_id,
828 (ND2_SGE *)wr->sg_list, wr->num_sge,
829 wr->wr.rdma.remote_addr, wr->wr.rdma.rkey,
830 flags);
831 VRB_DBG(FI_LOG_EP_DATA,
832 "IND2QueuePair::Read: hr=0x%08lx, nsge=%d, "
833 "addr=0x%016llx, raddr=0x%016llx, "
834 "rkey=0x%08lx, lkey=0x%08lx\n",
835 hr, wr->num_sge, wr->sg_list[0].addr,
836 wr->wr.rdma.remote_addr, wr->wr.rdma.rkey,
837 wr->sg_list[0].lkey);
838 break;
839
840 case IBV_WR_RDMA_WRITE:
841 flags = (wr->send_flags & IBV_SEND_INLINE) ?
842 ND_OP_FLAG_INLINE :
843 0;
844 hr = qp_nd->nd2qp->lpVtbl->Write(
845 qp_nd->nd2qp, (void *)wr->wr_id,
846 (ND2_SGE *)wr->sg_list, wr->num_sge,
847 wr->wr.rdma.remote_addr, wr->wr.rdma.rkey,
848 flags);
849 VRB_DBG(FI_LOG_EP_DATA,
850 "IND2QueuePair::Write: hr=0x%08lx, nsge=%d, "
851 "addr=0x%016llx, raddr=0x%016llx, "
852 "rkey=0x%08lx, lkey=0x%08lx\n",
853 hr, wr->num_sge, wr->sg_list[0].addr,
854 wr->wr.rdma.remote_addr, wr->wr.rdma.rkey,
855 wr->sg_list[0].lkey);
856 break;
857
858 default:
859 // TODO: Add support for other transfer methods.
860 VRB_WARN(FI_LOG_EP_DATA, "Unsupported opcode: %d\n",
861 wr->opcode);
862 assert(false);
863 errno = EINVAL;
864 return errno;
865 }
866
867 if (FAILED(hr)) {
868 errno = hresult2fi(hr);
869 *bad_wr = wr;
870 return errno;
871 }
872
873 wr = wr->next;
874 }
875
876 return 0;
877 }
878
879 const char *ibv_wc_status_str(enum ibv_wc_status status)
880 {
881 VRB_TRACE(FI_LOG_FABRIC, "\n");
882
883 static const char *const wc_status_str[] = {
884 [IBV_WC_SUCCESS] = "success",
885 [IBV_WC_LOC_LEN_ERR] = "local length error",
886 [IBV_WC_LOC_QP_OP_ERR] = "local QP operation error",
887 [IBV_WC_LOC_EEC_OP_ERR] = "local EE context operation error",
888 [IBV_WC_LOC_PROT_ERR] = "local protection error",
889 [IBV_WC_WR_FLUSH_ERR] = "Work Request Flushed Error",
890 [IBV_WC_MW_BIND_ERR] = "memory management operation error",
891 [IBV_WC_BAD_RESP_ERR] = "bad response error",
892 [IBV_WC_LOC_ACCESS_ERR] = "local access error",
893 [IBV_WC_REM_INV_REQ_ERR] = "remote invalid request error",
894 [IBV_WC_REM_ACCESS_ERR] = "remote access error",
895 [IBV_WC_REM_OP_ERR] = "remote operation error",
896 [IBV_WC_RETRY_EXC_ERR] = "transport retry counter exceeded",
897 [IBV_WC_RNR_RETRY_EXC_ERR] = "RNR retry counter exceeded",
898 [IBV_WC_LOC_RDD_VIOL_ERR] = "local RDD violation error",
899 [IBV_WC_REM_INV_RD_REQ_ERR] = "remote invalid RD request",
900 [IBV_WC_REM_ABORT_ERR] = "aborted error",
901 [IBV_WC_INV_EECN_ERR] = "invalid EE context number",
902 [IBV_WC_INV_EEC_STATE_ERR] = "invalid EE context state",
903 [IBV_WC_FATAL_ERR] = "fatal error",
904 [IBV_WC_RESP_TIMEOUT_ERR] = "response timeout error",
905 [IBV_WC_GENERAL_ERR] = "general error",
906 [IBV_WC_TM_ERR] = "TM error",
907 [IBV_WC_TM_RNDV_INCOMPLETE] = "TM software rendezvous",
908 };
909
910 if (status < IBV_WC_SUCCESS || status > IBV_WC_TM_RNDV_INCOMPLETE)
911 return "unknown";
912
913 return wc_status_str[status];
914 }
915
916 struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
917 struct ibv_srq_init_attr *srq_init_attr)
918 {
919 VRB_TRACE(FI_LOG_FABRIC, "\n");
920 errno = ENOSYS;
921 return NULL;
922 }
923
924 int ibv_destroy_srq(struct ibv_srq *srq)
925 {
926 VRB_TRACE(FI_LOG_FABRIC, "\n");
927 return ENOSYS;
928 }
929
930 struct ibv_ah *ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
931 {
932 VRB_TRACE(FI_LOG_FABRIC, "\n");
933 errno = ENOSYS;
934 return NULL;
935 }
936
937 int ibv_destroy_ah(struct ibv_ah *ah)
938 {
939 VRB_TRACE(FI_LOG_FABRIC, "\n");
940 return ENOSYS;
941 }
942
943 int ibv_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr,
944 struct ibv_recv_wr **bad_recv_wr)
945 {
946 VRB_TRACE(FI_LOG_FABRIC, "\n");
947 return ENOSYS;
948 }