Debian Codegraph @

modules/libfabric/prov/hook/dmabuf_peer_mem/src/hook_dmabuf_peer_mem.c

raw
1 /*
2 * Copyright (c) 2021 Intel Corporation. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <inttypes.h>
34 #include <sys/ioctl.h>
35 #include "ofi.h"
36 #include "ofi_prov.h"
37 #include "ofi_hmem.h"
38 #include "hook_prov.h"
39 #include "hook_dmabuf_peer_mem.h"
40 #include "dmabuf_reg.h"
41
42 static char *dmabuf_reg_dev_name = "/dev/" DMABUF_REG_DEV_NAME;
43
44 /*
45 * Add dmabuf info to the registry. Ok if the info matches an existing
46 * entry, in which case the reference counter of that entry is increased.
47 * Return 0 on success.
48 */
49 static int dmabuf_reg_add(int reg_fd, uint64_t base, uint64_t size, int fd)
50 {
51 struct dmabuf_reg_param args = {
52 .op = DMABUF_REG_ADD,
53 .base = base,
54 .size = size,
55 .fd = fd,
56 };
57
58 return ioctl(reg_fd, DMABUF_REG_IOCTL, &args) ? -errno : 0;
59 }
60
61 /*
62 * Remove a dmabuf entry from the registry, using dmabuf fd as the key.
63 */
64 static void dmabuf_reg_remove(int reg_fd, int fd)
65 {
66 struct dmabuf_reg_param args = {
67 .op = DMABUF_REG_REMOVE_FD,
68 .fd = fd,
69 };
70
71 (void) ioctl(reg_fd, DMABUF_REG_IOCTL, &args);
72 }
73
74 /*
75 * Check if <addr,size> is covered by a dmabuf entry in the registry. Entries
76 * in the registry don't have overlapping ranges. Valid <addr,size> should
77 * be within the range of a single entry, or not at all.
78 *
79 * Return value:
80 *
81 * 0 ---------- The range is covered, fd is set to the corresponding dmabuf fd
82 * -ENOENT ---- The range is empty in the registry
83 * Others ----- Various errors including: invalid range (e.g. overflow), range
84 * partially overlapping with entries in the registry, and I/O error.
85 */
86 static int dmabuf_reg_query(int reg_fd, uint64_t addr, uint64_t size, int *fd)
87 {
88 struct dmabuf_reg_param args = {
89 .op = DMABUF_REG_QUERY,
90 .base = addr,
91 .size = size,
92 };
93
94 if (ioctl(reg_fd, DMABUF_REG_IOCTL, &args))
95 return -errno;
96
97 *fd = args.fd;
98 return 0;
99 }
100
101 /*
102 * IPC handles are not cached in old oneAPI L0. Each time a call is made to
103 * get the IPC handle, a new dmabuf fd is created. In this case, the fd needs
104 * to be explicitly closed when no longer in use to avoid running out of file
105 * descriptors.
106 *
107 * In newer L0 library, IPC handles are cached. The same dmabuf fd is returned
108 * for multiple calls to get IPC handles as long as the buffer is the same. As
109 * a result, the fd SHOULD NOT be closed explicitly otherwise later use of the
110 * fd will fail.
111 *
112 * By default assume the IPC handle is cached. Not only this is more up-to-date,
113 * but also the side effect is less severe if the assumption turns out wrong.
114 */
115 static bool ze_ipc_handle_is_cached = true;
116
117 static inline int get_dmabuf_fd(void *buf, size_t len)
118 {
119 static bool first = true;
120 void *handle;
121 int fd, fd2;
122 int err;
123
124 err = ze_hmem_get_handle(buf, len, &handle);
125 if (err)
126 return err;
127
128 fd = (int)(uintptr_t)handle;
129 assert(fd >= 0);
130
131 if (!first)
132 goto end;
133
134 err = ze_hmem_get_handle(buf, len, &handle);
135 if (err)
136 goto end;
137
138 fd2 = (int)(uintptr_t)handle;
139 ze_ipc_handle_is_cached = (fd == fd2);
140
141 if (!ze_ipc_handle_is_cached)
142 close(fd2);
143
144 first = false;
145
146 end:
147 return fd;
148 }
149
150 static inline void put_dmabuf_fd(int fd)
151 {
152 if (!ze_ipc_handle_is_cached)
153 close(fd);
154 }
155
156 /*
157 * If the MR buffer is associated with a dmabuf, get the dmabuf fd and add to
158 * the registry.
159 *
160 * The assumption is that the memory region to be registered is homogeneous:
161 * either all system memory or belong to the same dmabuf object. We only need
162 * to check the first non-zero iov.
163 *
164 * Set the fd to -1 if the buffer is not associated with a dmabuf, or failed
165 * to be added to the registry.
166 */
167 static void get_mr_fd(struct dmabuf_peer_mem_mr *mr,
168 size_t iov_count, const struct iovec *iov)
169 {
170 int fd = -1;
171 int err;
172 struct dmabuf_peer_mem_fabric *fab;
173
174 fab = container_of(mr->mr_hook.domain->fabric,
175 struct dmabuf_peer_mem_fabric, fabric_hook);
176
177 mr->fd = -1;
178
179 while (iov_count && !iov->iov_len) {
180 iov_count--;
181 iov++;
182 }
183
184 if (!iov_count)
185 goto out;
186
187 err = ze_hmem_get_base_addr(iov->iov_base, iov->iov_len,
188 (void **)&mr->base, (size_t *)&mr->size);
189 if (err)
190 goto out;
191
192 ofi_mutex_lock(&fab->mutex);
193
194 err = dmabuf_reg_query(fab->dmabuf_reg_fd, mr->base, mr->size, &fd);
195 switch (err) {
196 case -ENOENT:
197 /*
198 * The region is not covered by any entry in the registry, add a
199 * new entry to the registry now.
200 */
201 fd = get_dmabuf_fd(iov->iov_base, iov->iov_len);
202 if (fd < 0)
203 goto out_unlock;
204
205 err = dmabuf_reg_add(fab->dmabuf_reg_fd, mr->base, mr->size, fd);
206 if (err) {
207 put_dmabuf_fd(fd);
208 } else {
209 mr->fd = fd;
210 FI_INFO(fab->fabric_hook.hprov, FI_LOG_MR,
211 "Add new entry: base 0x%"PRIx64" size %"PRIu64" fd %d\n",
212 mr->base, mr->size, mr->fd);
213 }
214 break;
215
216 case 0:
217 /*
218 * The region is already covered by an entry in the registry,
219 * add a reference to it.
220 */
221 err = dmabuf_reg_add(fab->dmabuf_reg_fd, mr->base, mr->size, fd);
222 if (!err)
223 mr->fd = fd;
224 break;
225
226 default:
227 /*
228 * Error happened: range overflow, range conflict with existing
229 * entries, or I/O error.
230 */
231 break;
232 }
233
234 out_unlock:
235 ofi_mutex_unlock(&fab->mutex);
236
237 out:
238 return;
239 }
240
241 static void release_mr_fd(struct dmabuf_peer_mem_mr *mr)
242 {
243 int fd;
244 int err;
245 struct dmabuf_peer_mem_fabric *fab;
246
247 fab = container_of(mr->mr_hook.domain->fabric,
248 struct dmabuf_peer_mem_fabric, fabric_hook);
249
250 if (mr->fd < 0)
251 return;
252
253 /*
254 * Remove this MR's reference to the fd in the kernel registry. The
255 * fd would be removed from the registry if the refcnt reaches 0.
256 * In that case, the fd is no longer used by any MR and should be
257 * released.
258 */
259 ofi_mutex_lock(&fab->mutex);
260 dmabuf_reg_remove(fab->dmabuf_reg_fd, mr->fd);
261 err = dmabuf_reg_query(fab->dmabuf_reg_fd, mr->base, mr->size, &fd);
262 if (err == -ENOENT) {
263 FI_INFO(fab->fabric_hook.hprov, FI_LOG_MR,
264 "Remove entry: base 0x%"PRIx64" size %"PRIu64" fd %d\n",
265 mr->base, mr->size, mr->fd);
266 put_dmabuf_fd(mr->fd);
267 }
268 ofi_mutex_unlock(&fab->mutex);
269 }
270
271 static int hook_dmabuf_peer_mem_mr_close(struct fid *fid)
272 {
273 struct dmabuf_peer_mem_mr *mr;
274
275 mr = container_of(fid, struct dmabuf_peer_mem_mr, mr_hook.mr.fid);
276
277 release_mr_fd(mr);
278
279 hook_close(fid);
280
281 return FI_SUCCESS;
282 }
283
284 static struct fi_ops dmabuf_peer_mem_mr_fid_ops = {
285 .size = sizeof(struct fi_ops),
286 .close = hook_dmabuf_peer_mem_mr_close,
287 .bind = hook_bind,
288 .control = hook_control,
289 .ops_open = hook_ops_open,
290 };
291
292 static int hook_dmabuf_peer_mem_mr_regattr(struct fid *fid,
293 const struct fi_mr_attr *attr,
294 uint64_t flags, struct fid_mr **mr)
295 {
296 struct hook_domain *dom;
297 struct dmabuf_peer_mem_mr *mymr;
298 int ret;
299
300 mymr = calloc(1, sizeof *mymr);
301 if (!mymr)
302 return -FI_ENOMEM;
303
304 dom = container_of(fid, struct hook_domain, domain.fid);
305
306 mymr->mr_hook.domain = dom;
307 mymr->mr_hook.mr.fid.fclass = FI_CLASS_MR;
308 mymr->mr_hook.mr.fid.context = attr->context;
309 mymr->mr_hook.mr.fid.ops = &dmabuf_peer_mem_mr_fid_ops;
310
311 get_mr_fd(mymr, attr->iov_count, attr->mr_iov);
312
313 if (mymr->fd != -1 && attr->iface == FI_HMEM_SYSTEM)
314 ((struct fi_mr_attr *)attr)->iface = FI_HMEM_ZE;
315
316 ret = fi_mr_regattr(dom->hdomain, attr, flags, &mymr->mr_hook.hmr);
317 if (ret) {
318 release_mr_fd(mymr);
319 free(mymr);
320 } else {
321 mymr->mr_hook.mr.mem_desc = mymr->mr_hook.hmr->mem_desc;
322 mymr->mr_hook.mr.key = mymr->mr_hook.hmr->key;
323 *mr = &mymr->mr_hook.mr;
324 }
325
326 return ret;
327 }
328
329 static int hook_dmabuf_peer_mem_mr_regv(struct fid *fid,
330 const struct iovec *iov,
331 size_t count, uint64_t access,
332 uint64_t offset, uint64_t requested_key,
333 uint64_t flags, struct fid_mr **mr,
334 void *context)
335 {
336 struct fi_mr_attr attr;
337
338 attr.mr_iov = iov;
339 attr.iov_count = count;
340 attr.access = access;
341 attr.offset = offset;
342 attr.requested_key = requested_key;
343 attr.context = context;
344 attr.auth_key_size = 0;
345 attr.auth_key = NULL;
346 attr.iface = FI_HMEM_SYSTEM;
347
348 return hook_dmabuf_peer_mem_mr_regattr(fid, &attr, flags, mr);
349 }
350
351 static int hook_dmabuf_peer_mem_mr_reg(struct fid *fid, const void *buf,
352 size_t len, uint64_t access,
353 uint64_t offset, uint64_t requested_key,
354 uint64_t flags, struct fid_mr **mr,
355 void *context)
356 {
357 struct iovec iov;
358
359 iov.iov_base = (void *) buf;
360 iov.iov_len = len;
361 return hook_dmabuf_peer_mem_mr_regv(fid, &iov, 1, access, offset,
362 requested_key, flags, mr, context);
363 }
364
365 static struct fi_ops_mr hook_dmabuf_peer_mem_mr_ops = {
366 .size = sizeof(struct fi_ops_mr),
367 .reg = hook_dmabuf_peer_mem_mr_reg,
368 .regv = hook_dmabuf_peer_mem_mr_regv,
369 .regattr = hook_dmabuf_peer_mem_mr_regattr,
370 };
371
372 static int hook_dmabuf_peer_mem_domain_init(struct fid *fid)
373 {
374 struct fid_domain *domain = container_of(fid, struct fid_domain, fid);
375 domain->mr = &hook_dmabuf_peer_mem_mr_ops;
376 return 0;
377 }
378
379 static int hook_dmabuf_peer_mem_fabric_close(struct fid *fid)
380 {
381 struct dmabuf_peer_mem_fabric *fab;
382
383 fab = container_of(fid, struct dmabuf_peer_mem_fabric, fabric_hook);
384 close(fab->dmabuf_reg_fd);
385 ofi_mutex_destroy(&fab->mutex);
386 hook_close(fid);
387
388 return FI_SUCCESS;
389 }
390
391 static struct fi_ops dmabuf_peer_mem_fabric_fid_ops = {
392 .size = sizeof(struct fi_ops),
393 .close = hook_dmabuf_peer_mem_fabric_close,
394 .bind = hook_bind,
395 .control = hook_control,
396 .ops_open = hook_ops_open,
397 };
398
399 static int hook_dmabuf_peer_mem_fabric(struct fi_fabric_attr *attr,
400 struct fid_fabric **fabric,
401 void *context)
402 {
403 struct fi_provider *hprov = context;
404 struct dmabuf_peer_mem_fabric *fab;
405 extern struct hook_prov_ctx hook_dmabuf_peer_mem_ctx;
406 int fd;
407
408 if (ofi_prov_ctx(hprov)->type != OFI_PROV_CORE) {
409 FI_TRACE(hprov, FI_LOG_FABRIC,
410 "Skip installing dmabuf_peer_mem hook\n");
411 return -FI_EINVAL;
412 }
413
414 FI_TRACE(hprov, FI_LOG_FABRIC, "Installing dmabuf_peer_mem hook\n");
415
416 fd = open(dmabuf_reg_dev_name, 0);
417 if (fd < 0) {
418 FI_WARN(hprov, FI_LOG_FABRIC,
419 "Failed to install dmabuf_peer_mem hook: couldn't open %s\n",
420 dmabuf_reg_dev_name);
421 return -errno;
422 }
423
424 fab = calloc(1, sizeof *fab);
425 if (!fab) {
426 close(fd);
427 return -FI_ENOMEM;
428 }
429
430 ofi_mutex_init(&fab->mutex);
431 fab->dmabuf_reg_fd = fd;
432 hook_fabric_init(&fab->fabric_hook, HOOK_DMABUF_PEER_MEM, attr->fabric,
433 hprov, &dmabuf_peer_mem_fabric_fid_ops,
434 &hook_dmabuf_peer_mem_ctx);
435 *fabric = &fab->fabric_hook.fabric;
436 return 0;
437 }
438
439 struct hook_prov_ctx hook_dmabuf_peer_mem_ctx = {
440 .prov = {
441 .version = OFI_VERSION_DEF_PROV,
442 /* We're a pass-through provider, so the fi_version is always the latest */
443 .fi_version = OFI_VERSION_LATEST,
444 .name = "ofi_hook_dmabuf_peer_mem",
445 .getinfo = NULL,
446 .fabric = hook_dmabuf_peer_mem_fabric,
447 .cleanup = NULL,
448 },
449 };
450
451 HOOK_DMABUF_PEER_MEM_INI
452 {
453 #ifdef HAVE_DMABUF_PEER_MEM_DL
454 ze_hmem_init();
455 #endif
456
457 hook_dmabuf_peer_mem_ctx.ini_fid[FI_CLASS_DOMAIN] =
458 hook_dmabuf_peer_mem_domain_init;
459 return &hook_dmabuf_peer_mem_ctx.prov;
460 }