Debian Codegraph @

tests/test_cuda_dmabuf.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2022 Nvidia Inc. All rights reserved. See COPYING file
3
4 import unittest
5 import errno
6
7 from pyverbs.pyverbs_error import PyverbsRDMAError
8 from tests.base import RCResources, RDMATestCase
9 from pyverbs.mr import DmaBufMR, MREx, DMAHandle
10 from pyverbs.qp import QPAttr
11 import tests.cuda_utils as cu
12 from pyverbs.libibverbs_enums import ibv_access_flags, ibv_wr_opcode, ibv_mr_init_attr_mask
13 import tests.utils as u
14
15 try:
16 from cuda import cuda, cudart, nvrtc
17 cu.CUDA_FOUND = True
18 except ImportError:
19 cu.CUDA_FOUND = False
20
21 GPU_PAGE_SIZE = 1 << 16
22
23
24 @cu.set_mem_io_cuda_methods
25 class DmabufCudaRes(RCResources):
26 def __init__(self, dev_name, ib_port, gid_index,
27 mr_access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE):
28 """
29 Initializes MR and DMA BUF resources on top of a CUDA memory.
30 Uses RC QPs for traffic.
31 :param dev_name: Device name to be used
32 :param ib_port: IB port of the device to use
33 :param gid_index: Which GID index to use
34 :param mr_access: The MR access
35 """
36 self.mr_access = mr_access
37 self.cuda_addr = None
38 super().__init__(dev_name=dev_name, ib_port=ib_port, gid_index=gid_index)
39
40 def dmabuf_cuda_init(self):
41 self.cuda_addr = cu.check_cuda_errors(cuda.cuMemAlloc(GPU_PAGE_SIZE))
42
43 attr_flag = 1
44 cu.check_cuda_errors(cuda.cuPointerSetAttribute(
45 attr_flag,
46 cuda.CUpointer_attribute.CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
47 int(self.cuda_addr)))
48
49 dmabuf_fd = cu.check_cuda_errors(
50 cuda.cuMemGetHandleForAddressRange(self.cuda_addr, GPU_PAGE_SIZE,
51 cuda.CUmemRangeHandleType.CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0))
52 return dmabuf_fd
53
54 def create_mr(self):
55 dmabuf_fd = self.dmabuf_cuda_init()
56 try:
57 self.mr = DmaBufMR(self.pd, self.msg_size, self.mr_access, dmabuf_fd)
58 except PyverbsRDMAError as ex:
59 if ex.error_code == errno.EOPNOTSUPP:
60 raise unittest.SkipTest('Registering DMABUF with DmaBufMR is not supported')
61 raise ex
62
63 def create_qp_attr(self):
64 qp_attr = QPAttr(port_num=self.ib_port)
65 qp_access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \
66 ibv_access_flags.IBV_ACCESS_REMOTE_READ
67 qp_attr.qp_access_flags = qp_access
68 return qp_attr
69
70
71 @cu.set_mem_io_cuda_methods
72 class MRExDmabufCudaRes(DmabufCudaRes):
73 """Resource class that registers an MREx with dma-buf FD."""
74
75 @u.skip_unsupported
76 def create_mr(self):
77 dmabuf_fd = self.dmabuf_cuda_init()
78 self.mr = MREx(self.pd, self.msg_size, self.mr_access,
79 comp_mask=ibv_mr_init_attr_mask.IBV_REG_MR_MASK_FD | \
80 ibv_mr_init_attr_mask.IBV_REG_MR_MASK_IOVA, fd=dmabuf_fd, iova=0)
81
82
83 @cu.set_mem_io_cuda_methods
84 class MRExDmabufDmaHCudaRes(DmabufCudaRes):
85 """Resource class that registers an MREx with dma-buf FD and DMAHandle."""
86
87 @u.skip_unsupported
88 def create_mr(self):
89 dmabuf_fd = self.dmabuf_cuda_init()
90 try:
91 attr = u.create_dmah_init_attr()
92 dmah = DMAHandle(self.ctx, attr)
93 except PyverbsRDMAError as ex:
94 if ex.error_code == errno.EOPNOTSUPP:
95 raise unittest.SkipTest('DMAHandle is not supported')
96 raise ex
97
98 self.mr = MREx(self.pd, self.msg_size, self.mr_access,
99 comp_mask=ibv_mr_init_attr_mask.IBV_REG_MR_MASK_FD | \
100 ibv_mr_init_attr_mask.IBV_REG_MR_MASK_IOVA, fd=dmabuf_fd, iova=0, dmah=dmah)
101
102 @cu.set_init_cuda_methods
103 class DmabufCudaTest(RDMATestCase):
104 """
105 Test RDMA traffic over CUDA memory
106 """
107
108 def test_cuda_dmabuf_rdma_write_traffic(self):
109 """
110 Runs RDMA Write traffic over CUDA allocated memory using DMA BUF and
111 RC QPs.
112 """
113 access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE
114 self.create_players(DmabufCudaRes, mr_access=access)
115 u.rdma_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
116
117 def test_mrex_cuda_dmabuf_rdma_write_traffic(self):
118 """
119 Runs RDMA Write traffic over CUDA allocated memory using DMA BUF and
120 RC QPs with MREx.
121 """
122 access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE
123 self.create_players(MRExDmabufCudaRes, mr_access=access)
124 u.rdma_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
125
126 def test_mrex_dmah_cuda_dmabuf_rdma_write_traffic(self):
127 """Runs RDMA Write traffic using MREx + DMAHandle over CUDA dma-buf."""
128 access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE
129 self.create_players(MRExDmabufDmaHCudaRes, mr_access=access)
130 u.rdma_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)