| 1 |
/* |
| 2 |
* Copyright (c) 2021 Intel Corporation. All rights reserved. |
| 3 |
* |
| 4 |
* This software is available to you under a choice of one of two |
| 5 |
* licenses. You may choose to be licensed under the terms of the GNU |
| 6 |
* General Public License (GPL) Version 2, available from the file |
| 7 |
* COPYING in the main directory of this source tree, or the |
| 8 |
* BSD license below: |
| 9 |
* |
| 10 |
* Redistribution and use in source and binary forms, with or |
| 11 |
* without modification, are permitted provided that the following |
| 12 |
* conditions are met: |
| 13 |
* |
| 14 |
* - Redistributions of source code must retain the above |
| 15 |
* copyright notice, this list of conditions and the following |
| 16 |
* disclaimer. |
| 17 |
* |
| 18 |
* - Redistributions in binary form must reproduce the above |
| 19 |
* copyright notice, this list of conditions and the following |
| 20 |
* disclaimer in the documentation and/or other materials |
| 21 |
* provided with the distribution. |
| 22 |
* |
| 23 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 |
* SOFTWARE. |
| 31 |
*/ |
| 32 |
|
| 33 |
#include <inttypes.h> |
| 34 |
#include <sys/ioctl.h> |
| 35 |
#include "ofi.h" |
| 36 |
#include "ofi_prov.h" |
| 37 |
#include "ofi_hmem.h" |
| 38 |
#include "hook_prov.h" |
| 39 |
#include "hook_dmabuf_peer_mem.h" |
| 40 |
#include "dmabuf_reg.h" |
| 41 |
|
| 42 |
static char *dmabuf_reg_dev_name = "/dev/" DMABUF_REG_DEV_NAME; |
| 43 |
|
| 44 |
/* |
| 45 |
* Add dmabuf info to the registry. Ok if the info matches an existing |
| 46 |
* entry, in which case the reference counter of that entry is increased. |
| 47 |
* Return 0 on success. |
| 48 |
*/ |
| 49 |
static int dmabuf_reg_add(int reg_fd, uint64_t base, uint64_t size, int fd) |
| 50 |
{ |
| 51 |
struct dmabuf_reg_param args = { |
| 52 |
.op = DMABUF_REG_ADD, |
| 53 |
.base = base, |
| 54 |
.size = size, |
| 55 |
.fd = fd, |
| 56 |
}; |
| 57 |
|
| 58 |
return ioctl(reg_fd, DMABUF_REG_IOCTL, &args) ? -errno : 0; |
| 59 |
} |
| 60 |
|
| 61 |
/* |
| 62 |
* Remove a dmabuf entry from the registry, using dmabuf fd as the key. |
| 63 |
*/ |
| 64 |
static void dmabuf_reg_remove(int reg_fd, int fd) |
| 65 |
{ |
| 66 |
struct dmabuf_reg_param args = { |
| 67 |
.op = DMABUF_REG_REMOVE_FD, |
| 68 |
.fd = fd, |
| 69 |
}; |
| 70 |
|
| 71 |
(void) ioctl(reg_fd, DMABUF_REG_IOCTL, &args); |
| 72 |
} |
| 73 |
|
| 74 |
/* |
| 75 |
* Check if <addr,size> is covered by a dmabuf entry in the registry. Entries |
| 76 |
* in the registry don't have overlapping ranges. Valid <addr,size> should |
| 77 |
* be within the range of a single entry, or not at all. |
| 78 |
* |
| 79 |
* Return value: |
| 80 |
* |
| 81 |
* 0 ---------- The range is covered, fd is set to the corresponding dmabuf fd |
| 82 |
* -ENOENT ---- The range is empty in the registry |
| 83 |
* Others ----- Various errors including: invalid range (e.g. overflow), range |
| 84 |
* partially overlapping with entries in the registry, and I/O error. |
| 85 |
*/ |
| 86 |
static int dmabuf_reg_query(int reg_fd, uint64_t addr, uint64_t size, int *fd) |
| 87 |
{ |
| 88 |
struct dmabuf_reg_param args = { |
| 89 |
.op = DMABUF_REG_QUERY, |
| 90 |
.base = addr, |
| 91 |
.size = size, |
| 92 |
}; |
| 93 |
|
| 94 |
if (ioctl(reg_fd, DMABUF_REG_IOCTL, &args)) |
| 95 |
return -errno; |
| 96 |
|
| 97 |
*fd = args.fd; |
| 98 |
return 0; |
| 99 |
} |
| 100 |
|
| 101 |
/* |
| 102 |
* IPC handles are not cached in old oneAPI L0. Each time a call is made to |
| 103 |
* get the IPC handle, a new dmabuf fd is created. In this case, the fd needs |
| 104 |
* to be explicitly closed when no longer in use to avoid running out of file |
| 105 |
* descriptors. |
| 106 |
* |
| 107 |
* In newer L0 library, IPC handles are cached. The same dmabuf fd is returned |
| 108 |
* for multiple calls to get IPC handles as long as the buffer is the same. As |
| 109 |
* a result, the fd SHOULD NOT be closed explicitly otherwise later use of the |
| 110 |
* fd will fail. |
| 111 |
* |
| 112 |
* By default assume the IPC handle is cached. Not only this is more up-to-date, |
| 113 |
* but also the side effect is less severe if the assumption turns out wrong. |
| 114 |
*/ |
| 115 |
static bool ze_ipc_handle_is_cached = true; |
| 116 |
|
| 117 |
static inline int get_dmabuf_fd(void *buf, size_t len) |
| 118 |
{ |
| 119 |
static bool first = true; |
| 120 |
void *handle; |
| 121 |
int fd, fd2; |
| 122 |
int err; |
| 123 |
|
| 124 |
err = ze_hmem_get_handle(buf, len, &handle); |
| 125 |
if (err) |
| 126 |
return err; |
| 127 |
|
| 128 |
fd = (int)(uintptr_t)handle; |
| 129 |
assert(fd >= 0); |
| 130 |
|
| 131 |
if (!first) |
| 132 |
goto end; |
| 133 |
|
| 134 |
err = ze_hmem_get_handle(buf, len, &handle); |
| 135 |
if (err) |
| 136 |
goto end; |
| 137 |
|
| 138 |
fd2 = (int)(uintptr_t)handle; |
| 139 |
ze_ipc_handle_is_cached = (fd == fd2); |
| 140 |
|
| 141 |
if (!ze_ipc_handle_is_cached) |
| 142 |
close(fd2); |
| 143 |
|
| 144 |
first = false; |
| 145 |
|
| 146 |
end: |
| 147 |
return fd; |
| 148 |
} |
| 149 |
|
| 150 |
static inline void put_dmabuf_fd(int fd) |
| 151 |
{ |
| 152 |
if (!ze_ipc_handle_is_cached) |
| 153 |
close(fd); |
| 154 |
} |
| 155 |
|
| 156 |
/* |
| 157 |
* If the MR buffer is associated with a dmabuf, get the dmabuf fd and add to |
| 158 |
* the registry. |
| 159 |
* |
| 160 |
* The assumption is that the memory region to be registered is homogeneous: |
| 161 |
* either all system memory or belong to the same dmabuf object. We only need |
| 162 |
* to check the first non-zero iov. |
| 163 |
* |
| 164 |
* Set the fd to -1 if the buffer is not associated with a dmabuf, or failed |
| 165 |
* to be added to the registry. |
| 166 |
*/ |
| 167 |
static void get_mr_fd(struct dmabuf_peer_mem_mr *mr, |
| 168 |
size_t iov_count, const struct iovec *iov) |
| 169 |
{ |
| 170 |
int fd = -1; |
| 171 |
int err; |
| 172 |
struct dmabuf_peer_mem_fabric *fab; |
| 173 |
|
| 174 |
fab = container_of(mr->mr_hook.domain->fabric, |
| 175 |
struct dmabuf_peer_mem_fabric, fabric_hook); |
| 176 |
|
| 177 |
mr->fd = -1; |
| 178 |
|
| 179 |
while (iov_count && !iov->iov_len) { |
| 180 |
iov_count--; |
| 181 |
iov++; |
| 182 |
} |
| 183 |
|
| 184 |
if (!iov_count) |
| 185 |
goto out; |
| 186 |
|
| 187 |
err = ze_hmem_get_base_addr(iov->iov_base, iov->iov_len, |
| 188 |
(void **)&mr->base, (size_t *)&mr->size); |
| 189 |
if (err) |
| 190 |
goto out; |
| 191 |
|
| 192 |
ofi_mutex_lock(&fab->mutex); |
| 193 |
|
| 194 |
err = dmabuf_reg_query(fab->dmabuf_reg_fd, mr->base, mr->size, &fd); |
| 195 |
switch (err) { |
| 196 |
case -ENOENT: |
| 197 |
/* |
| 198 |
* The region is not covered by any entry in the registry, add a |
| 199 |
* new entry to the registry now. |
| 200 |
*/ |
| 201 |
fd = get_dmabuf_fd(iov->iov_base, iov->iov_len); |
| 202 |
if (fd < 0) |
| 203 |
goto out_unlock; |
| 204 |
|
| 205 |
err = dmabuf_reg_add(fab->dmabuf_reg_fd, mr->base, mr->size, fd); |
| 206 |
if (err) { |
| 207 |
put_dmabuf_fd(fd); |
| 208 |
} else { |
| 209 |
mr->fd = fd; |
| 210 |
FI_INFO(fab->fabric_hook.hprov, FI_LOG_MR, |
| 211 |
"Add new entry: base 0x%"PRIx64" size %"PRIu64" fd %d\n", |
| 212 |
mr->base, mr->size, mr->fd); |
| 213 |
} |
| 214 |
break; |
| 215 |
|
| 216 |
case 0: |
| 217 |
/* |
| 218 |
* The region is already covered by an entry in the registry, |
| 219 |
* add a reference to it. |
| 220 |
*/ |
| 221 |
err = dmabuf_reg_add(fab->dmabuf_reg_fd, mr->base, mr->size, fd); |
| 222 |
if (!err) |
| 223 |
mr->fd = fd; |
| 224 |
break; |
| 225 |
|
| 226 |
default: |
| 227 |
/* |
| 228 |
* Error happened: range overflow, range conflict with existing |
| 229 |
* entries, or I/O error. |
| 230 |
*/ |
| 231 |
break; |
| 232 |
} |
| 233 |
|
| 234 |
out_unlock: |
| 235 |
ofi_mutex_unlock(&fab->mutex); |
| 236 |
|
| 237 |
out: |
| 238 |
return; |
| 239 |
} |
| 240 |
|
| 241 |
static void release_mr_fd(struct dmabuf_peer_mem_mr *mr) |
| 242 |
{ |
| 243 |
int fd; |
| 244 |
int err; |
| 245 |
struct dmabuf_peer_mem_fabric *fab; |
| 246 |
|
| 247 |
fab = container_of(mr->mr_hook.domain->fabric, |
| 248 |
struct dmabuf_peer_mem_fabric, fabric_hook); |
| 249 |
|
| 250 |
if (mr->fd < 0) |
| 251 |
return; |
| 252 |
|
| 253 |
/* |
| 254 |
* Remove this MR's reference to the fd in the kernel registry. The |
| 255 |
* fd would be removed from the registry if the refcnt reaches 0. |
| 256 |
* In that case, the fd is no longer used by any MR and should be |
| 257 |
* released. |
| 258 |
*/ |
| 259 |
ofi_mutex_lock(&fab->mutex); |
| 260 |
dmabuf_reg_remove(fab->dmabuf_reg_fd, mr->fd); |
| 261 |
err = dmabuf_reg_query(fab->dmabuf_reg_fd, mr->base, mr->size, &fd); |
| 262 |
if (err == -ENOENT) { |
| 263 |
FI_INFO(fab->fabric_hook.hprov, FI_LOG_MR, |
| 264 |
"Remove entry: base 0x%"PRIx64" size %"PRIu64" fd %d\n", |
| 265 |
mr->base, mr->size, mr->fd); |
| 266 |
put_dmabuf_fd(mr->fd); |
| 267 |
} |
| 268 |
ofi_mutex_unlock(&fab->mutex); |
| 269 |
} |
| 270 |
|
| 271 |
static int hook_dmabuf_peer_mem_mr_close(struct fid *fid) |
| 272 |
{ |
| 273 |
struct dmabuf_peer_mem_mr *mr; |
| 274 |
|
| 275 |
mr = container_of(fid, struct dmabuf_peer_mem_mr, mr_hook.mr.fid); |
| 276 |
|
| 277 |
release_mr_fd(mr); |
| 278 |
|
| 279 |
hook_close(fid); |
| 280 |
|
| 281 |
return FI_SUCCESS; |
| 282 |
} |
| 283 |
|
| 284 |
static struct fi_ops dmabuf_peer_mem_mr_fid_ops = { |
| 285 |
.size = sizeof(struct fi_ops), |
| 286 |
.close = hook_dmabuf_peer_mem_mr_close, |
| 287 |
.bind = hook_bind, |
| 288 |
.control = hook_control, |
| 289 |
.ops_open = hook_ops_open, |
| 290 |
}; |
| 291 |
|
| 292 |
static int hook_dmabuf_peer_mem_mr_regattr(struct fid *fid, |
| 293 |
const struct fi_mr_attr *attr, |
| 294 |
uint64_t flags, struct fid_mr **mr) |
| 295 |
{ |
| 296 |
struct hook_domain *dom; |
| 297 |
struct dmabuf_peer_mem_mr *mymr; |
| 298 |
int ret; |
| 299 |
|
| 300 |
mymr = calloc(1, sizeof *mymr); |
| 301 |
if (!mymr) |
| 302 |
return -FI_ENOMEM; |
| 303 |
|
| 304 |
dom = container_of(fid, struct hook_domain, domain.fid); |
| 305 |
|
| 306 |
mymr->mr_hook.domain = dom; |
| 307 |
mymr->mr_hook.mr.fid.fclass = FI_CLASS_MR; |
| 308 |
mymr->mr_hook.mr.fid.context = attr->context; |
| 309 |
mymr->mr_hook.mr.fid.ops = &dmabuf_peer_mem_mr_fid_ops; |
| 310 |
|
| 311 |
get_mr_fd(mymr, attr->iov_count, attr->mr_iov); |
| 312 |
|
| 313 |
if (mymr->fd != -1 && attr->iface == FI_HMEM_SYSTEM) |
| 314 |
((struct fi_mr_attr *)attr)->iface = FI_HMEM_ZE; |
| 315 |
|
| 316 |
ret = fi_mr_regattr(dom->hdomain, attr, flags, &mymr->mr_hook.hmr); |
| 317 |
if (ret) { |
| 318 |
release_mr_fd(mymr); |
| 319 |
free(mymr); |
| 320 |
} else { |
| 321 |
mymr->mr_hook.mr.mem_desc = mymr->mr_hook.hmr->mem_desc; |
| 322 |
mymr->mr_hook.mr.key = mymr->mr_hook.hmr->key; |
| 323 |
*mr = &mymr->mr_hook.mr; |
| 324 |
} |
| 325 |
|
| 326 |
return ret; |
| 327 |
} |
| 328 |
|
| 329 |
static int hook_dmabuf_peer_mem_mr_regv(struct fid *fid, |
| 330 |
const struct iovec *iov, |
| 331 |
size_t count, uint64_t access, |
| 332 |
uint64_t offset, uint64_t requested_key, |
| 333 |
uint64_t flags, struct fid_mr **mr, |
| 334 |
void *context) |
| 335 |
{ |
| 336 |
struct fi_mr_attr attr; |
| 337 |
|
| 338 |
attr.mr_iov = iov; |
| 339 |
attr.iov_count = count; |
| 340 |
attr.access = access; |
| 341 |
attr.offset = offset; |
| 342 |
attr.requested_key = requested_key; |
| 343 |
attr.context = context; |
| 344 |
attr.auth_key_size = 0; |
| 345 |
attr.auth_key = NULL; |
| 346 |
attr.iface = FI_HMEM_SYSTEM; |
| 347 |
|
| 348 |
return hook_dmabuf_peer_mem_mr_regattr(fid, &attr, flags, mr); |
| 349 |
} |
| 350 |
|
| 351 |
static int hook_dmabuf_peer_mem_mr_reg(struct fid *fid, const void *buf, |
| 352 |
size_t len, uint64_t access, |
| 353 |
uint64_t offset, uint64_t requested_key, |
| 354 |
uint64_t flags, struct fid_mr **mr, |
| 355 |
void *context) |
| 356 |
{ |
| 357 |
struct iovec iov; |
| 358 |
|
| 359 |
iov.iov_base = (void *) buf; |
| 360 |
iov.iov_len = len; |
| 361 |
return hook_dmabuf_peer_mem_mr_regv(fid, &iov, 1, access, offset, |
| 362 |
requested_key, flags, mr, context); |
| 363 |
} |
| 364 |
|
| 365 |
static struct fi_ops_mr hook_dmabuf_peer_mem_mr_ops = { |
| 366 |
.size = sizeof(struct fi_ops_mr), |
| 367 |
.reg = hook_dmabuf_peer_mem_mr_reg, |
| 368 |
.regv = hook_dmabuf_peer_mem_mr_regv, |
| 369 |
.regattr = hook_dmabuf_peer_mem_mr_regattr, |
| 370 |
}; |
| 371 |
|
| 372 |
static int hook_dmabuf_peer_mem_domain_init(struct fid *fid) |
| 373 |
{ |
| 374 |
struct fid_domain *domain = container_of(fid, struct fid_domain, fid); |
| 375 |
domain->mr = &hook_dmabuf_peer_mem_mr_ops; |
| 376 |
return 0; |
| 377 |
} |
| 378 |
|
| 379 |
static int hook_dmabuf_peer_mem_fabric_close(struct fid *fid) |
| 380 |
{ |
| 381 |
struct dmabuf_peer_mem_fabric *fab; |
| 382 |
|
| 383 |
fab = container_of(fid, struct dmabuf_peer_mem_fabric, fabric_hook); |
| 384 |
close(fab->dmabuf_reg_fd); |
| 385 |
ofi_mutex_destroy(&fab->mutex); |
| 386 |
hook_close(fid); |
| 387 |
|
| 388 |
return FI_SUCCESS; |
| 389 |
} |
| 390 |
|
| 391 |
static struct fi_ops dmabuf_peer_mem_fabric_fid_ops = { |
| 392 |
.size = sizeof(struct fi_ops), |
| 393 |
.close = hook_dmabuf_peer_mem_fabric_close, |
| 394 |
.bind = hook_bind, |
| 395 |
.control = hook_control, |
| 396 |
.ops_open = hook_ops_open, |
| 397 |
}; |
| 398 |
|
| 399 |
static int hook_dmabuf_peer_mem_fabric(struct fi_fabric_attr *attr, |
| 400 |
struct fid_fabric **fabric, |
| 401 |
void *context) |
| 402 |
{ |
| 403 |
struct fi_provider *hprov = context; |
| 404 |
struct dmabuf_peer_mem_fabric *fab; |
| 405 |
extern struct hook_prov_ctx hook_dmabuf_peer_mem_ctx; |
| 406 |
int fd; |
| 407 |
|
| 408 |
if (ofi_prov_ctx(hprov)->type != OFI_PROV_CORE) { |
| 409 |
FI_TRACE(hprov, FI_LOG_FABRIC, |
| 410 |
"Skip installing dmabuf_peer_mem hook\n"); |
| 411 |
return -FI_EINVAL; |
| 412 |
} |
| 413 |
|
| 414 |
FI_TRACE(hprov, FI_LOG_FABRIC, "Installing dmabuf_peer_mem hook\n"); |
| 415 |
|
| 416 |
fd = open(dmabuf_reg_dev_name, 0); |
| 417 |
if (fd < 0) { |
| 418 |
FI_WARN(hprov, FI_LOG_FABRIC, |
| 419 |
"Failed to install dmabuf_peer_mem hook: couldn't open %s\n", |
| 420 |
dmabuf_reg_dev_name); |
| 421 |
return -errno; |
| 422 |
} |
| 423 |
|
| 424 |
fab = calloc(1, sizeof *fab); |
| 425 |
if (!fab) { |
| 426 |
close(fd); |
| 427 |
return -FI_ENOMEM; |
| 428 |
} |
| 429 |
|
| 430 |
ofi_mutex_init(&fab->mutex); |
| 431 |
fab->dmabuf_reg_fd = fd; |
| 432 |
hook_fabric_init(&fab->fabric_hook, HOOK_DMABUF_PEER_MEM, attr->fabric, |
| 433 |
hprov, &dmabuf_peer_mem_fabric_fid_ops, |
| 434 |
&hook_dmabuf_peer_mem_ctx); |
| 435 |
*fabric = &fab->fabric_hook.fabric; |
| 436 |
return 0; |
| 437 |
} |
| 438 |
|
| 439 |
struct hook_prov_ctx hook_dmabuf_peer_mem_ctx = { |
| 440 |
.prov = { |
| 441 |
.version = OFI_VERSION_DEF_PROV, |
| 442 |
/* We're a pass-through provider, so the fi_version is always the latest */ |
| 443 |
.fi_version = OFI_VERSION_LATEST, |
| 444 |
.name = "ofi_hook_dmabuf_peer_mem", |
| 445 |
.getinfo = NULL, |
| 446 |
.fabric = hook_dmabuf_peer_mem_fabric, |
| 447 |
.cleanup = NULL, |
| 448 |
}, |
| 449 |
}; |
| 450 |
|
| 451 |
HOOK_DMABUF_PEER_MEM_INI |
| 452 |
{ |
| 453 |
#ifdef HAVE_DMABUF_PEER_MEM_DL |
| 454 |
ze_hmem_init(); |
| 455 |
#endif |
| 456 |
|
| 457 |
hook_dmabuf_peer_mem_ctx.ini_fid[FI_CLASS_DOMAIN] = |
| 458 |
hook_dmabuf_peer_mem_domain_init; |
| 459 |
return &hook_dmabuf_peer_mem_ctx.prov; |
| 460 |
} |