Debian Codegraph @

tests/test_mlx5_dmabuf.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2024 Nvidia Inc. All rights reserved. See COPYING file
3
4 from os import strerror
5 import unittest
6 import errno
7
8 from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr
9 from pyverbs.providers.mlx5.mlx5dv_dmabuf import Mlx5DmaBufMR
10 from pyverbs.pyverbs_error import PyverbsRDMAError
11 from pyverbs.providers.mlx5.mlx5_enums import MLX5DV_REG_DMABUF_ACCESS_DATA_DIRECT_
12 from tests.mlx5_base import Mlx5RDMATestCase
13 from tests.base import RCResources
14 from pyverbs.qp import QPAttr
15 import tests.cuda_utils as cu
16 from pyverbs.libibverbs_enums import ibv_access_flags, ibv_wr_opcode
17 import tests.utils as u
18
19 try:
20 from cuda import cuda
21 cu.CUDA_FOUND = True
22 except ImportError:
23 cu.CUDA_FOUND = False
24
25
26 GPU_PAGE_SIZE = 1 << 16
27
28
29 def requires_data_direct_support():
30 """
31 Check if the device support data-direct
32 """
33 def outer(func):
34 def inner(instance):
35 with Mlx5Context(Mlx5DVContextAttr(), name=instance.dev_name) as ctx:
36 try:
37 ctx.get_data_direct_sysfs_path()
38 except PyverbsRDMAError as ex:
39 if ex.error_code == errno.ENODEV:
40 raise unittest.SkipTest('There is no data direct device in the system')
41 raise ex
42 return func(instance)
43 return inner
44 return outer
45
46
47 @cu.set_mem_io_cuda_methods
48 class Mlx5DmabufCudaRes(RCResources):
49 def __init__(self, dev_name, ib_port, gid_index,
50 mr_access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE, mlx5_access=0):
51 """
52 Initializes data-direct MR and DMA BUF resources on top of a CUDA memory.
53 Uses RC QPs for traffic.
54 :param dev_name: Device name to be used
55 :param ib_port: IB port of the device to use
56 :param gid_index: Which GID index to use
57 :param mr_access: The MR access
58 :param mlx5_access: The data-direct access
59 """
60 self.mr_access = mr_access
61 self.mlx5_access = mlx5_access
62 self.cuda_addr = None
63 super().__init__(dev_name=dev_name, ib_port=ib_port, gid_index=gid_index)
64
65 def create_mr(self):
66 self.cuda_addr = cu.check_cuda_errors(cuda.cuMemAlloc(GPU_PAGE_SIZE))
67
68 attr_flag = 1
69 cu.check_cuda_errors(cuda.cuPointerSetAttribute(
70 attr_flag,
71 cuda.CUpointer_attribute.CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
72 int(self.cuda_addr)))
73
74 cuda_flag = cuda.CUmemRangeHandleType.CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD
75 cuda_dma_mapping_type = cuda.CUmemRangeFlags.CU_MEM_RANGE_FLAG_DMA_BUF_MAPPING_TYPE_PCIE \
76 if self.mlx5_access else 0
77 dmabuf_fd = cu.check_cuda_errors(
78 cuda.cuMemGetHandleForAddressRange(self.cuda_addr,
79 GPU_PAGE_SIZE,
80 cuda_flag,
81 cuda_dma_mapping_type))
82 try:
83 self.mr = Mlx5DmaBufMR(self.pd, offset=0, length=self.msg_size, access=self.mr_access,
84 fd=dmabuf_fd, mlx5_access=self.mlx5_access)
85 except PyverbsRDMAError as ex:
86 if ex.error_code == errno.EOPNOTSUPP:
87 raise unittest.SkipTest('Registering DV DMABUF MR is not supported')
88 raise ex
89
90 def create_qp_attr(self):
91 qp_attr = QPAttr(port_num=self.ib_port)
92 qp_access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \
93 ibv_access_flags.IBV_ACCESS_REMOTE_READ
94 qp_attr.qp_access_flags = qp_access
95 return qp_attr
96
97
98 @cu.set_init_cuda_methods
99 class Mlx5DmabufCudaTest(Mlx5RDMATestCase):
100 """
101 Test data-direct DV verbs
102 """
103 @requires_data_direct_support()
104 def test_data_direct_sysfs_path_bad_length(self):
105 """
106 Query data direct sysfs path with buffer of 5 bytes. This is bad flow since 5 bytes aren't
107 enough for any sysfs path, so ENOSPC should be raised.
108 """
109 ctx = Mlx5Context(Mlx5DVContextAttr(), name=self.dev_name)
110 try:
111 path = ctx.get_data_direct_sysfs_path(5)
112 except PyverbsRDMAError as ex:
113 self.assertEqual(ex.error_code, errno.ENOSPC,
114 f'Got {strerror(ex.error_code)} but Expected {strerror(errno.ENOSPC)}')
115 else:
116 raise PyverbsRDMAError('Successfully queried data direct sysfs path with 5 bytes: '
117 f'{path}')
118
119 def test_dv_dmabuf_mr(self):
120 """
121 Creates dmabuf MR with DV API. mlx5_access is 0, so the MR is regular dmabuf MR.
122 Run RDMA write traffic.
123 """
124 access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE
125 self.create_players(Mlx5DmabufCudaRes, mr_access=access)
126 u.rdma_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
127
128 @requires_data_direct_support()
129 def test_dv_dmabuf_mr_data_direct(self):
130 """
131 Runs RDMA Write traffic over CUDA allocated memory using Data Direct DMA BUF and
132 RC QPs.
133 """
134 mr_access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE
135 self.create_players(Mlx5DmabufCudaRes, mr_access=mr_access,
136 mlx5_access=MLX5DV_REG_DMABUF_ACCESS_DATA_DIRECT_)
137 u.rdma_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)