Debian Codegraph @

tests/test_mlx5_cuda_umem.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2022 Nvidia Inc. All rights reserved. See COPYING file
3
4 import resource
5
6 from pyverbs.providers.mlx5.mlx5dv import Mlx5DevxObj, WqeDataSeg, Mlx5UMEM
7 from tests.mlx5_base import Mlx5DevxRcResources, Mlx5DevxTrafficBase
8 from pyverbs.providers.mlx5.mlx5_enums import MLX5DV_UMEM_MASK_DMABUF
9 import tests.cuda_utils as cu
10 from pyverbs.libibverbs_enums import ibv_access_flags
11
12 try:
13 from cuda import cuda, cudart, nvrtc
14 cu.CUDA_FOUND = True
15 except ImportError:
16 cu.CUDA_FOUND = False
17
18 GPU_PAGE_SIZE = 1 << 16
19
20
21 @cu.set_mem_io_cuda_methods
22 class CudaDevxRes(Mlx5DevxRcResources):
23 def __init__(self, dev_name, ib_port, gid_index,
24 mr_access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE):
25 """
26 Initialize DevX resources with CUDA memory allocations.
27 :param dev_name: Device name to be used
28 :param ib_port: IB port of the device to use
29 :param gid_index: Which GID index to use
30 :param mr_access: The MR access
31 """
32 self.mr_access = mr_access
33 self.cuda_addr = None
34 self.dmabuf_fd = None
35 self.umem = None
36 self.mkey = None
37 self.lkey = None
38 self.lkey = None
39 super().__init__(dev_name=dev_name, ib_port=ib_port, gid_index=gid_index)
40
41 def init_resources(self):
42 self.alloc_cuda_mem()
43 super().init_resources()
44 self.create_dmabuf_umem()
45 self.create_mkey()
46
47 def get_wqe_data_segment(self):
48 return WqeDataSeg(self.msg_size, self.lkey, int(self.cuda_addr))
49
50 def alloc_cuda_mem(self):
51 """
52 Allocates CUDA memory and a DMABUF FD on that memory.
53 """
54 self.cuda_addr = cu.check_cuda_errors(cuda.cuMemAlloc(GPU_PAGE_SIZE))
55
56 # Sync between memory operations
57 attr_value = 1
58 cu.check_cuda_errors(cuda.cuPointerSetAttribute(
59 attr_value,
60 cuda.CUpointer_attribute.CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
61 int(self.cuda_addr)
62 ))
63
64 # Memory address and size must be aligned to page size to get a handle
65 assert (GPU_PAGE_SIZE % resource.getpagesize() == 0 and
66 int(self.cuda_addr) % resource.getpagesize() == 0)
67 self.dmabuf_fd = cu.check_cuda_errors(
68 cuda.cuMemGetHandleForAddressRange(self.cuda_addr,
69 GPU_PAGE_SIZE,
70 cuda.CUmemRangeHandleType.CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD,
71 0))
72
73 def create_mr(self):
74 pass
75
76 def create_dmabuf_umem(self):
77 umem_aligment = resource.getpagesize()
78 self.umem = Mlx5UMEM(self.ctx, GPU_PAGE_SIZE, 0,
79 umem_aligment, self.mr_access, umem_aligment,
80 MLX5DV_UMEM_MASK_DMABUF, self.dmabuf_fd)
81
82 def create_mkey(self):
83 from tests.mlx5_prm_structs import SwMkc, CreateMkeyIn, CreateMkeyOut
84 accesses = [ibv_access_flags.IBV_ACCESS_LOCAL_WRITE, ibv_access_flags.IBV_ACCESS_REMOTE_READ,
85 ibv_access_flags.IBV_ACCESS_REMOTE_WRITE]
86 lw, rr, rw = (list(map(lambda access: int(self.mr_access & access != 0), accesses)))
87 mkey_ctx = SwMkc(lr=1, lw=lw, rr=rr, rw=rw, access_mode_1_0=0x1,
88 start_addr=int(self.cuda_addr),
89 len=GPU_PAGE_SIZE, pd=self.dv_pd.pdn, qpn=0xffffff)
90 self.mkey = Mlx5DevxObj(self.ctx, CreateMkeyIn(sw_mkc=mkey_ctx,
91 mkey_umem_id=self.umem.umem_id,
92 mkey_umem_valid=1),
93 len(CreateMkeyOut()))
94 self.lkey = CreateMkeyOut(self.mkey.out_view).mkey_index << 8
95
96
97 @cu.set_init_cuda_methods
98 class Mlx5GpuDevxRcTrafficTest(Mlx5DevxTrafficBase):
99 """
100 Test DevX traffic over CUDA memory using DMA BUF and UMEM
101 """
102
103 @cu.requires_cuda
104 def test_mlx_devx_cuda_send_imm_traffic(self):
105 """
106 Creates two DevX RC QPs and runs SEND_IMM traffic over CUDA allocated
107 memory using UMEM and DMA BUF.
108 """
109 self.create_players(CudaDevxRes)
110 # Send traffic
111 self.send_imm_traffic()