| 1 |
/* |
| 2 |
* Copyright (c) 2022 Canon Medical Research USA, Inc. All rights reserved. |
| 3 |
* |
| 4 |
* This software is available to you under a choice of one of two |
| 5 |
* licenses. You may choose to be licensed under the terms of the GNU |
| 6 |
* General Public License (GPL) Version 2, available from the file |
| 7 |
* COPYING in the main directory of this source tree, or the |
| 8 |
* BSD license below: |
| 9 |
* |
| 10 |
* Redistribution and use in source and binary forms, with or |
| 11 |
* without modification, are permitted provided that the following |
| 12 |
* conditions are met: |
| 13 |
* |
| 14 |
* - Redistributions of source code must retain the above |
| 15 |
* copyright notice, this list of conditions and the following |
| 16 |
* disclaimer. |
| 17 |
* |
| 18 |
* - Redistributions in binary form must reproduce the above |
| 19 |
* copyright notice, this list of conditions and the following |
| 20 |
* disclaimer in the documentation and/or other materials |
| 21 |
* provided with the distribution. |
| 22 |
* |
| 23 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 |
* SOFTWARE. |
| 31 |
*/ |
| 32 |
|
| 33 |
#include <assert.h> |
| 34 |
#include "ndspi.h" |
| 35 |
#include "verbs_nd.h" |
| 36 |
|
| 37 |
static enum ibv_wc_opcode vrb_ndrequest2opcode(enum ND2_REQUEST_TYPE rq) |
| 38 |
{ |
| 39 |
static enum ibv_wc_opcode opcodes[] = { |
| 40 |
[Nd2RequestTypeReceive] = IBV_WC_RECV, |
| 41 |
[Nd2RequestTypeSend] = IBV_WC_SEND, |
| 42 |
[Nd2RequestTypeRead] = IBV_WC_RDMA_READ, |
| 43 |
[Nd2RequestTypeWrite] = IBV_WC_RDMA_WRITE |
| 44 |
}; |
| 45 |
return (rq < ARRAY_SIZE(opcodes)) ? opcodes[rq] : IBV_WC_SEND; |
| 46 |
} |
| 47 |
|
| 48 |
struct ibv_device **ibv_get_device_list(int *num_devices) |
| 49 |
{ |
| 50 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 51 |
|
| 52 |
if (num_devices) { |
| 53 |
assert(nd_infra.adapters.count <= INT_MAX); |
| 54 |
*num_devices = (int)nd_infra.adapters.count; |
| 55 |
} |
| 56 |
return nd_infra.adapters.device_list; |
| 57 |
} |
| 58 |
|
| 59 |
void ibv_free_device_list(struct ibv_device **list) |
| 60 |
{ |
| 61 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 62 |
// Since we are keeping state statically, there is nothing to free. |
| 63 |
} |
| 64 |
|
| 65 |
struct ibv_context *ibv_open_device(struct ibv_device *device) |
| 66 |
{ |
| 67 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 68 |
return nd_get_context_by_device(device); |
| 69 |
} |
| 70 |
|
| 71 |
int ibv_close_device(struct ibv_context *context) |
| 72 |
{ |
| 73 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 74 |
// Since we are keeping state statically, there is nothing to close. |
| 75 |
return 0; |
| 76 |
} |
| 77 |
|
| 78 |
// The IND2Adapter::Query method doesn't provide data for a number of fields |
| 79 |
// in the ibv_device_attr structure. The following definitions were inferred |
| 80 |
// from what ibv_query_device returned on a Linux implementation. |
| 81 |
#define ND_MAX_CQ_CNT (1 << 24) |
| 82 |
#define ND_MAX_QP_CNT (1 << 18) |
| 83 |
#define ND_MAX_SRQ_CNT (1 << 23) |
| 84 |
#define ND_MAX_MR_CNT (1 << 24) |
| 85 |
|
| 86 |
int ibv_query_device(struct ibv_context *context, |
| 87 |
struct ibv_device_attr *device_attr) |
| 88 |
{ |
| 89 |
struct nd_adapter *adapter; |
| 90 |
ND2_ADAPTER_INFO *nd_attr; |
| 91 |
|
| 92 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 93 |
|
| 94 |
if ((!context) || (!device_attr)) { |
| 95 |
errno = EINVAL; |
| 96 |
return errno; |
| 97 |
} |
| 98 |
|
| 99 |
adapter = nd_get_adapter_by_context(context); |
| 100 |
if (!adapter) { |
| 101 |
errno = ENODEV; |
| 102 |
return errno; |
| 103 |
} |
| 104 |
|
| 105 |
nd_attr = &adapter->info; |
| 106 |
memset(device_attr, 0, sizeof(struct ibv_device_attr)); |
| 107 |
strcpy(device_attr->fw_ver, "(null)"); |
| 108 |
device_attr->node_guid = nd_attr->AdapterId; |
| 109 |
device_attr->sys_image_guid = nd_attr->AdapterId; |
| 110 |
device_attr->max_mr_size = nd_attr->MaxRegistrationSize; |
| 111 |
device_attr->vendor_id = nd_attr->VendorId; |
| 112 |
device_attr->vendor_part_id = nd_attr->DeviceId; |
| 113 |
device_attr->max_qp = ND_MAX_QP_CNT; |
| 114 |
device_attr->max_qp_wr = nd_attr->MaxInitiatorQueueDepth; |
| 115 |
device_attr->max_sge = nd_attr->MaxInitiatorSge; |
| 116 |
device_attr->max_sge_rd = nd_attr->MaxReadSge; |
| 117 |
device_attr->max_cq = ND_MAX_CQ_CNT; |
| 118 |
device_attr->max_cqe = nd_attr->MaxCompletionQueueDepth; |
| 119 |
device_attr->max_mr = ND_MAX_MR_CNT; |
| 120 |
device_attr->max_qp_rd_atom = nd_attr->MaxInboundReadLimit; |
| 121 |
device_attr->max_qp_init_rd_atom = nd_attr->MaxOutboundReadLimit; |
| 122 |
device_attr->max_srq = |
| 123 |
nd_attr->MaxSharedReceiveQueueDepth ? ND_MAX_SRQ_CNT : 0; |
| 124 |
device_attr->max_srq_wr = nd_attr->MaxSharedReceiveQueueDepth; |
| 125 |
device_attr->phys_port_cnt = 1; |
| 126 |
|
| 127 |
return 0; |
| 128 |
} |
| 129 |
|
| 130 |
const char *ibv_get_device_name(struct ibv_device *device) |
| 131 |
{ |
| 132 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 133 |
|
| 134 |
if (!device) |
| 135 |
return NULL; |
| 136 |
|
| 137 |
return device->name; |
| 138 |
} |
| 139 |
|
| 140 |
// infiniband/verbs.h defines ibv_query_port to be ___ibv_query_port |
| 141 |
int ___ibv_query_port(struct ibv_context *context, uint8_t port_num, |
| 142 |
struct ibv_port_attr *port_attr) |
| 143 |
{ |
| 144 |
struct nd_adapter *adapter; |
| 145 |
|
| 146 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 147 |
|
| 148 |
if ((!context) || (!port_attr)) { |
| 149 |
errno = EINVAL; |
| 150 |
return errno; |
| 151 |
} |
| 152 |
|
| 153 |
adapter = nd_get_adapter_by_context(context); |
| 154 |
if (!adapter) { |
| 155 |
errno = ENODEV; |
| 156 |
return errno; |
| 157 |
} |
| 158 |
|
| 159 |
memset(port_attr, 0, sizeof(struct ibv_port_attr)); |
| 160 |
port_attr->state = IBV_PORT_ACTIVE; |
| 161 |
port_attr->max_mtu = IBV_MTU_4096; |
| 162 |
port_attr->active_mtu = IBV_MTU_4096; |
| 163 |
port_attr->gid_tbl_len = 0; |
| 164 |
port_attr->max_msg_sz = adapter->info.MaxTransferLength; |
| 165 |
port_attr->pkey_tbl_len = 0; |
| 166 |
port_attr->lid = 0; |
| 167 |
port_attr->sm_sl = 0; |
| 168 |
port_attr->active_width = 2; // 4x |
| 169 |
port_attr->active_speed = 32; // 25 Gbps |
| 170 |
port_attr->phys_state = |
| 171 |
5; // IB_PORT_PHYS_STATE_LINKUP from "infiniband/iba/ib_types.h" |
| 172 |
port_attr->link_layer = IBV_LINK_LAYER_ETHERNET; |
| 173 |
|
| 174 |
return 0; |
| 175 |
} |
| 176 |
|
| 177 |
int ibv_query_gid(struct ibv_context *context, uint8_t port_num, int index, |
| 178 |
union ibv_gid *gid) |
| 179 |
{ |
| 180 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 181 |
|
| 182 |
// little endian representation of link-local address prefix. |
| 183 |
gid->global.subnet_prefix = 0x80fe; |
| 184 |
gid->global.interface_id = 0; |
| 185 |
|
| 186 |
return 0; |
| 187 |
} |
| 188 |
|
| 189 |
int ibv_query_pkey(struct ibv_context *context, uint8_t port_num, int index, |
| 190 |
__be16 *pkey) |
| 191 |
{ |
| 192 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 193 |
|
| 194 |
*pkey = 0xffff; |
| 195 |
|
| 196 |
return 0; |
| 197 |
} |
| 198 |
|
| 199 |
struct ibv_pd *ibv_alloc_pd(struct ibv_context *context) |
| 200 |
{ |
| 201 |
struct ibv_pd *pd; |
| 202 |
|
| 203 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 204 |
|
| 205 |
if (!context) { |
| 206 |
errno = EINVAL; |
| 207 |
return NULL; |
| 208 |
} |
| 209 |
|
| 210 |
pd = calloc(1, sizeof(*pd)); |
| 211 |
if (!pd) { |
| 212 |
errno = ENOMEM; |
| 213 |
return NULL; |
| 214 |
} |
| 215 |
|
| 216 |
pd->context = context; |
| 217 |
|
| 218 |
return pd; |
| 219 |
} |
| 220 |
|
| 221 |
int ibv_dealloc_pd(struct ibv_pd *pd) |
| 222 |
{ |
| 223 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 224 |
|
| 225 |
if (!pd) { |
| 226 |
errno = EINVAL; |
| 227 |
return errno; |
| 228 |
} |
| 229 |
|
| 230 |
free(pd); |
| 231 |
return 0; |
| 232 |
} |
| 233 |
|
| 234 |
struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context) |
| 235 |
{ |
| 236 |
struct nd_comp_channel *channel_nd; |
| 237 |
int ret; |
| 238 |
|
| 239 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 240 |
|
| 241 |
if (!context) { |
| 242 |
errno = EINVAL; |
| 243 |
return NULL; |
| 244 |
} |
| 245 |
|
| 246 |
channel_nd = calloc(1, sizeof(*channel_nd)); |
| 247 |
if (!channel_nd) { |
| 248 |
errno = ENOMEM; |
| 249 |
return NULL; |
| 250 |
} |
| 251 |
|
| 252 |
ret = dlistfd_head_init(&channel_nd->q); |
| 253 |
if (ret) { |
| 254 |
free(channel_nd); |
| 255 |
errno = ret; |
| 256 |
return NULL; |
| 257 |
} |
| 258 |
ofi_mutex_init(&channel_nd->q_lock); |
| 259 |
|
| 260 |
channel_nd->channel.context = context; |
| 261 |
channel_nd->channel.fd = fd_signal_get(&channel_nd->q.signal); |
| 262 |
|
| 263 |
return &channel_nd->channel; |
| 264 |
} |
| 265 |
|
| 266 |
int ibv_destroy_comp_channel(struct ibv_comp_channel *channel) |
| 267 |
{ |
| 268 |
struct nd_comp_channel *channel_nd; |
| 269 |
|
| 270 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 271 |
|
| 272 |
if (!channel) { |
| 273 |
errno = EINVAL; |
| 274 |
return errno; |
| 275 |
} |
| 276 |
|
| 277 |
if (channel->refcnt) { |
| 278 |
errno = EBUSY; |
| 279 |
return -1; |
| 280 |
} |
| 281 |
|
| 282 |
channel_nd = container_of(channel, struct nd_comp_channel, channel); |
| 283 |
|
| 284 |
dlistfd_head_free(&channel_nd->q); |
| 285 |
ofi_mutex_destroy(&channel_nd->q_lock); |
| 286 |
free(channel_nd); |
| 287 |
return 0; |
| 288 |
} |
| 289 |
|
| 290 |
struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, |
| 291 |
void *cq_context, struct ibv_comp_channel *channel, |
| 292 |
int comp_vector) |
| 293 |
{ |
| 294 |
struct nd_adapter *adapter; |
| 295 |
struct nd_cq *cq_nd; |
| 296 |
struct ibv_cq *cq; |
| 297 |
HRESULT hr; |
| 298 |
|
| 299 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 300 |
|
| 301 |
if (!context) { |
| 302 |
errno = EINVAL; |
| 303 |
return NULL; |
| 304 |
} |
| 305 |
|
| 306 |
adapter = nd_get_adapter_by_context(context); |
| 307 |
if (!adapter) { |
| 308 |
errno = ENODEV; |
| 309 |
return NULL; |
| 310 |
} |
| 311 |
|
| 312 |
cq_nd = calloc(1, sizeof(*cq_nd)); |
| 313 |
if (!cq_nd) { |
| 314 |
errno = ENOMEM; |
| 315 |
return NULL; |
| 316 |
} |
| 317 |
|
| 318 |
if (channel) |
| 319 |
++channel->refcnt; |
| 320 |
|
| 321 |
ofi_mutex_init(&cq_nd->notification.lock); |
| 322 |
pthread_cond_init(&cq_nd->notification.cond, 0); |
| 323 |
cq_nd->notification.event_cb = nd_cq_notify_event; |
| 324 |
cq_nd->notification.error_cb = nd_cq_notify_error; |
| 325 |
|
| 326 |
cq = &cq_nd->cq; |
| 327 |
cq->context = context; |
| 328 |
cq->channel = channel; |
| 329 |
cq->cq_context = cq_context; |
| 330 |
cq->cqe = cqe; |
| 331 |
|
| 332 |
hr = adapter->adapter->lpVtbl->CreateOverlappedFile(adapter->adapter, |
| 333 |
&cq_nd->ov_file); |
| 334 |
VRB_DBG(FI_LOG_CQ, "IND2Adapter::CreateOverlappedFile: hr=0x%08lx\n", |
| 335 |
hr); |
| 336 |
if (FAILED(hr)) { |
| 337 |
errno = hresult2fi(hr); |
| 338 |
goto err1; |
| 339 |
} |
| 340 |
|
| 341 |
assert(cq_nd->ov_file && cq_nd->ov_file != INVALID_HANDLE_VALUE); |
| 342 |
|
| 343 |
BindIoCompletionCallback(cq_nd->ov_file, nd_io_cb, 0); |
| 344 |
|
| 345 |
hr = adapter->adapter->lpVtbl->CreateCompletionQueue( |
| 346 |
adapter->adapter, &IID_IND2CompletionQueue, cq_nd->ov_file, cqe, |
| 347 |
0, 0, &cq_nd->nd2cq); |
| 348 |
VRB_DBG(FI_LOG_CQ, "IND2Adapter::CreateCompletionQueue: hr=0x%08lx\n", |
| 349 |
hr); |
| 350 |
if (FAILED(hr)) { |
| 351 |
errno = hresult2fi(hr); |
| 352 |
goto err2; |
| 353 |
} |
| 354 |
assert(cq_nd->nd2cq); |
| 355 |
|
| 356 |
return cq; |
| 357 |
err2: |
| 358 |
CloseHandle(cq_nd->ov_file); |
| 359 |
err1: |
| 360 |
free(cq_nd); |
| 361 |
return NULL; |
| 362 |
} |
| 363 |
|
| 364 |
int ibv_destroy_cq(struct ibv_cq *cq) |
| 365 |
{ |
| 366 |
struct nd_cq *cq_nd; |
| 367 |
|
| 368 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 369 |
|
| 370 |
if (!cq) { |
| 371 |
errno = EINVAL; |
| 372 |
return errno; |
| 373 |
} |
| 374 |
|
| 375 |
cq_nd = container_of(cq, struct nd_cq, cq); |
| 376 |
if (nd_cancel_pending(&cq_nd->notification, |
| 377 |
(IND2Overlapped *)cq_nd->nd2cq)) { |
| 378 |
return errno; |
| 379 |
} |
| 380 |
|
| 381 |
if (!CloseHandle(cq_nd->ov_file)) { |
| 382 |
errno = GetLastError(); |
| 383 |
return errno; |
| 384 |
} |
| 385 |
|
| 386 |
// There appears to be a bug related to the reference counting for |
| 387 |
// ComplettionQueue objects in the Mellanox ND driver. CompletionQueue |
| 388 |
// reference counts are incremented when they are used in the creation |
| 389 |
// of a QueuePair, but are not always decremented when the QueuePair is |
| 390 |
// Released. Ideally, we'd assert that that the reference count is zero |
| 391 |
// here, but we cannot because of that bug. |
| 392 |
cq_nd->nd2cq->lpVtbl->Release(cq_nd->nd2cq); |
| 393 |
|
| 394 |
if (cq_nd->cq.channel) |
| 395 |
--cq_nd->cq.channel->refcnt; |
| 396 |
|
| 397 |
pthread_cond_destroy(&cq_nd->notification.cond); |
| 398 |
ofi_mutex_destroy(&cq_nd->notification.lock); |
| 399 |
free(cq_nd); |
| 400 |
|
| 401 |
return 0; |
| 402 |
} |
| 403 |
|
| 404 |
int ibv_poll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc) |
| 405 |
{ |
| 406 |
struct nd_cq *cq_nd; |
| 407 |
int num_results; |
| 408 |
ND2_RESULT result; |
| 409 |
ULONG nResults; |
| 410 |
|
| 411 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 412 |
|
| 413 |
if (!cq || !wc) { |
| 414 |
errno = EINVAL; |
| 415 |
return -1; |
| 416 |
} |
| 417 |
|
| 418 |
cq_nd = container_of(cq, struct nd_cq, cq); |
| 419 |
for (num_results = 0; num_results < num_entries; ++num_results) { |
| 420 |
nResults = cq_nd->nd2cq->lpVtbl->GetResults(cq_nd->nd2cq, |
| 421 |
&result, 1); |
| 422 |
|
| 423 |
if (nResults == 0) |
| 424 |
break; |
| 425 |
|
| 426 |
memset(&wc[num_results], 0, sizeof(wc[num_results])); |
| 427 |
wc[num_results].wr_id = (uint64_t)result.RequestContext; |
| 428 |
wc[num_results].byte_len = result.BytesTransferred; |
| 429 |
wc[num_results].status = result.Status; |
| 430 |
wc[num_results].opcode = vrb_ndrequest2opcode(result.RequestType); |
| 431 |
FI_LOG(&vrb_prov, result.Status ? FI_LOG_WARN : FI_LOG_DEBUG, |
| 432 |
FI_LOG_CQ, |
| 433 |
"ibv_poll_cq: context=0x%016llx, numBytes=%d, " |
| 434 |
"status=0x%08lx, request=0x%08lx\n", |
| 435 |
(uint64_t)result.RequestContext, result.BytesTransferred, |
| 436 |
result.Status, result.RequestType); |
| 437 |
} |
| 438 |
|
| 439 |
return num_results; |
| 440 |
} |
| 441 |
|
| 442 |
int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only) |
| 443 |
{ |
| 444 |
struct nd_cq *cq_nd; |
| 445 |
HRESULT hr; |
| 446 |
int ret = 0; |
| 447 |
|
| 448 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 449 |
|
| 450 |
if (!cq) { |
| 451 |
errno = EINVAL; |
| 452 |
return errno; |
| 453 |
} |
| 454 |
|
| 455 |
cq_nd = container_of(cq, struct nd_cq, cq); |
| 456 |
|
| 457 |
ofi_mutex_lock(&cq_nd->notification.lock); |
| 458 |
if (!cq_nd->notification.cb_pending) { |
| 459 |
cq_nd->notification.cb_pending = 1; |
| 460 |
hr = cq_nd->nd2cq->lpVtbl->Notify( |
| 461 |
cq_nd->nd2cq, |
| 462 |
solicited_only ? ND_CQ_NOTIFY_SOLICITED : |
| 463 |
ND_CQ_NOTIFY_ANY, |
| 464 |
&cq_nd->notification.ov); |
| 465 |
VRB_DBG(FI_LOG_CQ, |
| 466 |
"IND2CompletionQueue::Notify: hr=0x%08lx; ov=%p\n", hr, |
| 467 |
&cq_nd->notification.ov); |
| 468 |
if (FAILED(hr)) { |
| 469 |
cq_nd->notification.cb_pending = 0; |
| 470 |
errno = hresult2fi(hr); |
| 471 |
ret = errno; |
| 472 |
} |
| 473 |
} |
| 474 |
ofi_mutex_unlock(&cq_nd->notification.lock); |
| 475 |
|
| 476 |
return ret; |
| 477 |
} |
| 478 |
|
| 479 |
int ibv_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq, |
| 480 |
void **cq_context) |
| 481 |
{ |
| 482 |
int ret = 0; |
| 483 |
struct nd_comp_channel *channel_nd; |
| 484 |
struct nd_cq *cq_nd = NULL; |
| 485 |
|
| 486 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 487 |
|
| 488 |
if ((!channel) || (!cq) || (!cq_context)) { |
| 489 |
errno = EINVAL; |
| 490 |
return -1; |
| 491 |
} |
| 492 |
|
| 493 |
channel_nd = container_of(channel, struct nd_comp_channel, channel); |
| 494 |
|
| 495 |
ofi_mutex_lock(&channel_nd->q_lock); |
| 496 |
if (dlistfd_empty(&channel_nd->q)) { |
| 497 |
ret = -1; |
| 498 |
} else { |
| 499 |
cq_nd = container_of(channel_nd->q.list.next, struct nd_cq, |
| 500 |
entry); |
| 501 |
*cq = &cq_nd->cq; |
| 502 |
*cq_context = cq_nd->cq.cq_context; |
| 503 |
dlistfd_remove(channel_nd->q.list.next, &channel_nd->q); |
| 504 |
} |
| 505 |
ofi_mutex_unlock(&channel_nd->q_lock); |
| 506 |
|
| 507 |
return ret; |
| 508 |
} |
| 509 |
|
| 510 |
void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents) |
| 511 |
{ |
| 512 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 513 |
// N/A |
| 514 |
} |
| 515 |
|
| 516 |
struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, |
| 517 |
struct ibv_qp_init_attr *qp_init_attr) |
| 518 |
{ |
| 519 |
static uint32_t unique_qp_num = 2; // QPN 0 & 1 are reserved by IB spec. |
| 520 |
struct nd_adapter *adapter; |
| 521 |
struct nd_qp *qp_nd; |
| 522 |
struct ibv_qp *qp; |
| 523 |
HRESULT hr; |
| 524 |
|
| 525 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 526 |
|
| 527 |
if ((!pd) || (!qp_init_attr)) { |
| 528 |
errno = EINVAL; |
| 529 |
return NULL; |
| 530 |
} |
| 531 |
|
| 532 |
adapter = nd_get_adapter_by_context(pd->context); |
| 533 |
if (qp_init_attr->cap.max_inline_data > |
| 534 |
adapter->info.MaxInlineDataSize) { |
| 535 |
errno = ENODEV; |
| 536 |
return NULL; |
| 537 |
} |
| 538 |
|
| 539 |
qp_nd = calloc(1, sizeof(*qp_nd)); |
| 540 |
if (!qp_nd) { |
| 541 |
errno = ENOMEM; |
| 542 |
return NULL; |
| 543 |
} |
| 544 |
|
| 545 |
qp = &qp_nd->qp; |
| 546 |
qp->context = pd->context; |
| 547 |
qp->qp_context = qp_init_attr->qp_context; |
| 548 |
qp->pd = pd; |
| 549 |
qp->send_cq = qp_init_attr->send_cq; |
| 550 |
qp->recv_cq = qp_init_attr->recv_cq; |
| 551 |
qp->srq = qp_init_attr->srq; |
| 552 |
qp->qp_num = unique_qp_num++; |
| 553 |
qp->state = IBV_QPS_RESET; |
| 554 |
qp->qp_type = qp_init_attr->qp_type; |
| 555 |
qp_nd->max_inline_data = qp_init_attr->cap.max_inline_data; |
| 556 |
|
| 557 |
hr = adapter->adapter->lpVtbl->CreateQueuePair( |
| 558 |
adapter->adapter, &IID_IND2QueuePair, |
| 559 |
(IUnknown *)((struct nd_cq *)qp_init_attr->recv_cq)->nd2cq, |
| 560 |
(IUnknown *)((struct nd_cq *)qp_init_attr->send_cq)->nd2cq, |
| 561 |
qp_init_attr->qp_context, qp_init_attr->cap.max_recv_wr, |
| 562 |
qp_init_attr->cap.max_send_wr, qp_init_attr->cap.max_recv_sge, |
| 563 |
qp_init_attr->cap.max_send_sge, |
| 564 |
qp_init_attr->cap.max_inline_data, (void **)&qp_nd->nd2qp); |
| 565 |
VRB_DBG(FI_LOG_EP_CTRL, "IND2Adapter::CreateQueuePair: hr=0x%08lx\n", |
| 566 |
hr); |
| 567 |
if (FAILED(hr)) { |
| 568 |
free(qp_nd); |
| 569 |
errno = hresult2fi(hr); |
| 570 |
return NULL; |
| 571 |
} |
| 572 |
|
| 573 |
return qp; |
| 574 |
} |
| 575 |
|
| 576 |
int ibv_destroy_qp(struct ibv_qp *qp) |
| 577 |
{ |
| 578 |
struct nd_qp *qp_nd; |
| 579 |
ULONG refcnt; |
| 580 |
|
| 581 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 582 |
|
| 583 |
if (!qp) { |
| 584 |
errno = EINVAL; |
| 585 |
return errno; |
| 586 |
} |
| 587 |
|
| 588 |
qp_nd = container_of(qp, struct nd_qp, qp); |
| 589 |
refcnt = qp_nd->nd2qp->lpVtbl->Release(qp_nd->nd2qp); |
| 590 |
assert(!refcnt); |
| 591 |
free(qp_nd); |
| 592 |
return 0; |
| 593 |
} |
| 594 |
|
| 595 |
int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) |
| 596 |
{ |
| 597 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 598 |
return 0; |
| 599 |
} |
| 600 |
|
| 601 |
int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, |
| 602 |
struct ibv_qp_init_attr *init_attr) |
| 603 |
{ |
| 604 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 605 |
return 0; |
| 606 |
} |
| 607 |
|
| 608 |
// infiniband/verbs.h defines ibv_reg_mr as a macro to call ibv_reg_mr_iova2. |
| 609 |
struct ibv_mr *ibv_reg_mr_iova2(struct ibv_pd *pd, void *addr, size_t length, |
| 610 |
uint64_t iova, unsigned int access) |
| 611 |
{ |
| 612 |
struct nd_adapter *adapter; |
| 613 |
struct nd_mr *mr_nd; |
| 614 |
struct ibv_mr *mr; |
| 615 |
OVERLAPPED ov = { 0 }; |
| 616 |
HRESULT hr; |
| 617 |
int ret; |
| 618 |
int flags = 0; |
| 619 |
|
| 620 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 621 |
|
| 622 |
if ((!pd) || (!addr)) { |
| 623 |
errno = EINVAL; |
| 624 |
return NULL; |
| 625 |
} |
| 626 |
|
| 627 |
adapter = nd_get_adapter_by_context(pd->context); |
| 628 |
if (!adapter) { |
| 629 |
errno = ENODEV; |
| 630 |
return NULL; |
| 631 |
} |
| 632 |
|
| 633 |
mr_nd = calloc(1, sizeof(*mr_nd)); |
| 634 |
if (!mr_nd) { |
| 635 |
errno = ENOMEM; |
| 636 |
goto err1; |
| 637 |
} |
| 638 |
|
| 639 |
hr = adapter->adapter->lpVtbl->CreateOverlappedFile(adapter->adapter, |
| 640 |
&mr_nd->ov_file); |
| 641 |
VRB_DBG(FI_LOG_MR, "IND2Adapter::CreateOverlappedFile: hr=0x%08lx\n", |
| 642 |
hr); |
| 643 |
if (FAILED(hr)) { |
| 644 |
errno = hresult2fi(hr); |
| 645 |
goto err2; |
| 646 |
} |
| 647 |
|
| 648 |
assert(mr_nd->ov_file && mr_nd->ov_file != INVALID_HANDLE_VALUE); |
| 649 |
|
| 650 |
hr = adapter->adapter->lpVtbl->CreateMemoryRegion(adapter->adapter, |
| 651 |
&IID_IND2MemoryRegion, |
| 652 |
mr_nd->ov_file, |
| 653 |
&mr_nd->region); |
| 654 |
VRB_DBG(FI_LOG_MR, "IND2Adapter::CreateMemoryRegion: hr=0x%08lx\n", hr); |
| 655 |
if (FAILED(hr)) { |
| 656 |
errno = hresult2fi(hr); |
| 657 |
goto err3; |
| 658 |
} |
| 659 |
|
| 660 |
mr = &mr_nd->mr; |
| 661 |
mr->context = pd->context; |
| 662 |
mr->pd = pd; |
| 663 |
mr->addr = addr; |
| 664 |
mr->length = length; |
| 665 |
|
| 666 |
if (access & IBV_ACCESS_LOCAL_WRITE) |
| 667 |
flags |= ND_MR_FLAG_ALLOW_LOCAL_WRITE; |
| 668 |
if (access & IBV_ACCESS_REMOTE_READ) |
| 669 |
flags |= ND_MR_FLAG_ALLOW_REMOTE_READ; |
| 670 |
if (access & IBV_ACCESS_REMOTE_WRITE) |
| 671 |
flags |= ND_MR_FLAG_ALLOW_REMOTE_WRITE; |
| 672 |
|
| 673 |
hr = mr_nd->region->lpVtbl->Register(mr_nd->region, addr, length, flags, |
| 674 |
&ov); |
| 675 |
VRB_DBG(FI_LOG_MR, |
| 676 |
"IND2MemoryRegion::Register: hr=0x%08lx; ov=%p, " |
| 677 |
"addr=0x%016llx, len=0x%016llx\n", |
| 678 |
hr, &ov, addr, length); |
| 679 |
if (FAILED(hr)) { |
| 680 |
errno = hresult2fi(hr); |
| 681 |
goto err4; |
| 682 |
} |
| 683 |
hr = mr_nd->region->lpVtbl->GetOverlappedResult(mr_nd->region, &ov, |
| 684 |
TRUE); |
| 685 |
VRB_DBG(FI_LOG_MR, |
| 686 |
"IND2MemoryRegion::GetOverlappedResult: hr=0x%08lx; ov=%p\n", |
| 687 |
hr, &ov); |
| 688 |
if (FAILED(hr)) { |
| 689 |
errno = hresult2fi(hr); |
| 690 |
goto err5; |
| 691 |
} |
| 692 |
|
| 693 |
mr->lkey = mr_nd->region->lpVtbl->GetLocalToken(mr_nd->region); |
| 694 |
mr->rkey = mr_nd->region->lpVtbl->GetRemoteToken(mr_nd->region); |
| 695 |
|
| 696 |
return mr; |
| 697 |
err5: |
| 698 |
mr_nd->region->lpVtbl->CancelOverlappedRequests(mr_nd->region); |
| 699 |
err4: |
| 700 |
ret = mr_nd->region->lpVtbl->Release(mr_nd->region); |
| 701 |
assert(!ret); |
| 702 |
err3: |
| 703 |
ret = CloseHandle(mr_nd->ov_file); |
| 704 |
assert(ret); |
| 705 |
err2: |
| 706 |
free(mr_nd); |
| 707 |
err1: |
| 708 |
return NULL; |
| 709 |
} |
| 710 |
|
| 711 |
int ibv_dereg_mr(struct ibv_mr *mr) |
| 712 |
{ |
| 713 |
OVERLAPPED ov = { 0 }; |
| 714 |
int ret = 0; |
| 715 |
struct nd_mr *mr_nd; |
| 716 |
HRESULT hr; |
| 717 |
int tmp; |
| 718 |
|
| 719 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 720 |
|
| 721 |
if (!mr) { |
| 722 |
errno = EINVAL; |
| 723 |
return errno; |
| 724 |
} |
| 725 |
|
| 726 |
mr_nd = container_of(mr, struct nd_mr, mr); |
| 727 |
hr = mr_nd->region->lpVtbl->Deregister(mr_nd->region, &ov); |
| 728 |
VRB_DBG(FI_LOG_MR, "IND2MemoryRegion::Deregister: hr=0x%08lx; ov=%p\n", |
| 729 |
hr, &ov); |
| 730 |
if (FAILED(hr)) { |
| 731 |
ret = hresult2fi(hr); |
| 732 |
goto err1; |
| 733 |
} |
| 734 |
hr = mr_nd->region->lpVtbl->GetOverlappedResult(mr_nd->region, &ov, |
| 735 |
TRUE); |
| 736 |
VRB_DBG(FI_LOG_MR, |
| 737 |
"IND2MemoryRegion::GetOverlappedResult: hr=0x%08lx; ov=%p\n", |
| 738 |
hr, &ov); |
| 739 |
if (FAILED(hr)) { |
| 740 |
ret = hresult2fi(hr); |
| 741 |
mr_nd->region->lpVtbl->CancelOverlappedRequests(mr_nd->region); |
| 742 |
} |
| 743 |
err1: |
| 744 |
tmp = CloseHandle(mr_nd->ov_file); |
| 745 |
assert(tmp); |
| 746 |
tmp = mr_nd->region->lpVtbl->Release(mr_nd->region); |
| 747 |
assert(!tmp); |
| 748 |
free(mr_nd); |
| 749 |
if (ret) |
| 750 |
errno = ret; |
| 751 |
return ret; |
| 752 |
} |
| 753 |
|
| 754 |
int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr, |
| 755 |
struct ibv_recv_wr **bad_wr) |
| 756 |
{ |
| 757 |
struct nd_qp *qp_nd; |
| 758 |
HRESULT hr; |
| 759 |
|
| 760 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 761 |
|
| 762 |
if ((!qp) || (!wr) || (!bad_wr)) { |
| 763 |
errno = EINVAL; |
| 764 |
return errno; |
| 765 |
} |
| 766 |
|
| 767 |
*bad_wr = NULL; |
| 768 |
|
| 769 |
qp_nd = container_of(qp, struct nd_qp, qp); |
| 770 |
while (wr) { |
| 771 |
hr = qp_nd->nd2qp->lpVtbl->Receive(qp_nd->nd2qp, |
| 772 |
(void *)wr->wr_id, |
| 773 |
(ND2_SGE *)wr->sg_list, |
| 774 |
wr->num_sge); |
| 775 |
VRB_DBG(FI_LOG_EP_DATA, |
| 776 |
"IND2QueuePair::Receive: hr=0x%08lx, " |
| 777 |
"context=0x%016llx\n", |
| 778 |
hr, wr->wr_id); |
| 779 |
if (FAILED(hr)) { |
| 780 |
errno = hresult2fi(hr); |
| 781 |
*bad_wr = wr; |
| 782 |
return errno; |
| 783 |
} |
| 784 |
|
| 785 |
wr = wr->next; |
| 786 |
} |
| 787 |
|
| 788 |
return 0; |
| 789 |
} |
| 790 |
|
| 791 |
int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr, |
| 792 |
struct ibv_send_wr **bad_wr) |
| 793 |
{ |
| 794 |
struct nd_qp *qp_nd; |
| 795 |
uint32_t flags = 0; |
| 796 |
HRESULT hr; |
| 797 |
|
| 798 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 799 |
|
| 800 |
if ((!qp) || (!wr) || (!bad_wr)) { |
| 801 |
errno = EINVAL; |
| 802 |
return errno; |
| 803 |
} |
| 804 |
|
| 805 |
*bad_wr = NULL; |
| 806 |
|
| 807 |
qp_nd = container_of(qp, struct nd_qp, qp); |
| 808 |
while (wr) { |
| 809 |
switch (wr->opcode) { |
| 810 |
case IBV_WR_SEND: |
| 811 |
flags = (wr->send_flags & IBV_SEND_INLINE) ? |
| 812 |
ND_OP_FLAG_INLINE : |
| 813 |
0; |
| 814 |
hr = qp_nd->nd2qp->lpVtbl->Send(qp_nd->nd2qp, |
| 815 |
(void *)wr->wr_id, |
| 816 |
(ND2_SGE *)wr->sg_list, |
| 817 |
wr->num_sge, flags); |
| 818 |
VRB_DBG(FI_LOG_EP_DATA, |
| 819 |
"IND2QueuePair::Send: hr=0x%08lx, nsge=%d, " |
| 820 |
"addr=0x%016llx, length=%d, lkey=0x%08lx\n", |
| 821 |
hr, wr->num_sge, wr->sg_list[0].addr, |
| 822 |
wr->sg_list[0].length, wr->sg_list[0].lkey); |
| 823 |
break; |
| 824 |
|
| 825 |
case IBV_WR_RDMA_READ: |
| 826 |
hr = qp_nd->nd2qp->lpVtbl->Read( |
| 827 |
qp_nd->nd2qp, (void *)wr->wr_id, |
| 828 |
(ND2_SGE *)wr->sg_list, wr->num_sge, |
| 829 |
wr->wr.rdma.remote_addr, wr->wr.rdma.rkey, |
| 830 |
flags); |
| 831 |
VRB_DBG(FI_LOG_EP_DATA, |
| 832 |
"IND2QueuePair::Read: hr=0x%08lx, nsge=%d, " |
| 833 |
"addr=0x%016llx, raddr=0x%016llx, " |
| 834 |
"rkey=0x%08lx, lkey=0x%08lx\n", |
| 835 |
hr, wr->num_sge, wr->sg_list[0].addr, |
| 836 |
wr->wr.rdma.remote_addr, wr->wr.rdma.rkey, |
| 837 |
wr->sg_list[0].lkey); |
| 838 |
break; |
| 839 |
|
| 840 |
case IBV_WR_RDMA_WRITE: |
| 841 |
flags = (wr->send_flags & IBV_SEND_INLINE) ? |
| 842 |
ND_OP_FLAG_INLINE : |
| 843 |
0; |
| 844 |
hr = qp_nd->nd2qp->lpVtbl->Write( |
| 845 |
qp_nd->nd2qp, (void *)wr->wr_id, |
| 846 |
(ND2_SGE *)wr->sg_list, wr->num_sge, |
| 847 |
wr->wr.rdma.remote_addr, wr->wr.rdma.rkey, |
| 848 |
flags); |
| 849 |
VRB_DBG(FI_LOG_EP_DATA, |
| 850 |
"IND2QueuePair::Write: hr=0x%08lx, nsge=%d, " |
| 851 |
"addr=0x%016llx, raddr=0x%016llx, " |
| 852 |
"rkey=0x%08lx, lkey=0x%08lx\n", |
| 853 |
hr, wr->num_sge, wr->sg_list[0].addr, |
| 854 |
wr->wr.rdma.remote_addr, wr->wr.rdma.rkey, |
| 855 |
wr->sg_list[0].lkey); |
| 856 |
break; |
| 857 |
|
| 858 |
default: |
| 859 |
// TODO: Add support for other transfer methods. |
| 860 |
VRB_WARN(FI_LOG_EP_DATA, "Unsupported opcode: %d\n", |
| 861 |
wr->opcode); |
| 862 |
assert(false); |
| 863 |
errno = EINVAL; |
| 864 |
return errno; |
| 865 |
} |
| 866 |
|
| 867 |
if (FAILED(hr)) { |
| 868 |
errno = hresult2fi(hr); |
| 869 |
*bad_wr = wr; |
| 870 |
return errno; |
| 871 |
} |
| 872 |
|
| 873 |
wr = wr->next; |
| 874 |
} |
| 875 |
|
| 876 |
return 0; |
| 877 |
} |
| 878 |
|
| 879 |
const char *ibv_wc_status_str(enum ibv_wc_status status) |
| 880 |
{ |
| 881 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 882 |
|
| 883 |
static const char *const wc_status_str[] = { |
| 884 |
[IBV_WC_SUCCESS] = "success", |
| 885 |
[IBV_WC_LOC_LEN_ERR] = "local length error", |
| 886 |
[IBV_WC_LOC_QP_OP_ERR] = "local QP operation error", |
| 887 |
[IBV_WC_LOC_EEC_OP_ERR] = "local EE context operation error", |
| 888 |
[IBV_WC_LOC_PROT_ERR] = "local protection error", |
| 889 |
[IBV_WC_WR_FLUSH_ERR] = "Work Request Flushed Error", |
| 890 |
[IBV_WC_MW_BIND_ERR] = "memory management operation error", |
| 891 |
[IBV_WC_BAD_RESP_ERR] = "bad response error", |
| 892 |
[IBV_WC_LOC_ACCESS_ERR] = "local access error", |
| 893 |
[IBV_WC_REM_INV_REQ_ERR] = "remote invalid request error", |
| 894 |
[IBV_WC_REM_ACCESS_ERR] = "remote access error", |
| 895 |
[IBV_WC_REM_OP_ERR] = "remote operation error", |
| 896 |
[IBV_WC_RETRY_EXC_ERR] = "transport retry counter exceeded", |
| 897 |
[IBV_WC_RNR_RETRY_EXC_ERR] = "RNR retry counter exceeded", |
| 898 |
[IBV_WC_LOC_RDD_VIOL_ERR] = "local RDD violation error", |
| 899 |
[IBV_WC_REM_INV_RD_REQ_ERR] = "remote invalid RD request", |
| 900 |
[IBV_WC_REM_ABORT_ERR] = "aborted error", |
| 901 |
[IBV_WC_INV_EECN_ERR] = "invalid EE context number", |
| 902 |
[IBV_WC_INV_EEC_STATE_ERR] = "invalid EE context state", |
| 903 |
[IBV_WC_FATAL_ERR] = "fatal error", |
| 904 |
[IBV_WC_RESP_TIMEOUT_ERR] = "response timeout error", |
| 905 |
[IBV_WC_GENERAL_ERR] = "general error", |
| 906 |
[IBV_WC_TM_ERR] = "TM error", |
| 907 |
[IBV_WC_TM_RNDV_INCOMPLETE] = "TM software rendezvous", |
| 908 |
}; |
| 909 |
|
| 910 |
if (status < IBV_WC_SUCCESS || status > IBV_WC_TM_RNDV_INCOMPLETE) |
| 911 |
return "unknown"; |
| 912 |
|
| 913 |
return wc_status_str[status]; |
| 914 |
} |
| 915 |
|
| 916 |
struct ibv_srq *ibv_create_srq(struct ibv_pd *pd, |
| 917 |
struct ibv_srq_init_attr *srq_init_attr) |
| 918 |
{ |
| 919 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 920 |
errno = ENOSYS; |
| 921 |
return NULL; |
| 922 |
} |
| 923 |
|
| 924 |
int ibv_destroy_srq(struct ibv_srq *srq) |
| 925 |
{ |
| 926 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 927 |
return ENOSYS; |
| 928 |
} |
| 929 |
|
| 930 |
struct ibv_ah *ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) |
| 931 |
{ |
| 932 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 933 |
errno = ENOSYS; |
| 934 |
return NULL; |
| 935 |
} |
| 936 |
|
| 937 |
int ibv_destroy_ah(struct ibv_ah *ah) |
| 938 |
{ |
| 939 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 940 |
return ENOSYS; |
| 941 |
} |
| 942 |
|
| 943 |
int ibv_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr, |
| 944 |
struct ibv_recv_wr **bad_recv_wr) |
| 945 |
{ |
| 946 |
VRB_TRACE(FI_LOG_FABRIC, "\n"); |
| 947 |
return ENOSYS; |
| 948 |
} |