| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright (c) 2025 NVIDIA Corporation . All rights reserved. See COPYING file |
| 3 |
|
| 4 |
from pyverbs.libibverbs_enums import ibv_access_flags, ibv_wr_opcode, ibv_odp_transport_cap_bits, \ |
| 5 |
ibv_tph_mem_type |
| 6 |
from tests.base import PyverbsAPITestCase, RCResources, RDMATestCase |
| 7 |
from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError |
| 8 |
import pyverbs.device as d |
| 9 |
from pyverbs.pd import PD |
| 10 |
from pyverbs.mr import MREx, DMAHandle |
| 11 |
from pyverbs.qp import QPAttr |
| 12 |
import tests.utils as u |
| 13 |
|
| 14 |
|
| 15 |
class DMAHandleTest(PyverbsAPITestCase): |
| 16 |
@u.skip_unsupported |
| 17 |
def test_dmah_with_mrex(self): |
| 18 |
"""Verify DMAHandle can be used during MREx registration.""" |
| 19 |
with d.Context(name=self.dev_name) as ctx: |
| 20 |
with PD(ctx) as pd: |
| 21 |
attr = u.create_dmah_init_attr() |
| 22 |
with DMAHandle(ctx, attr) as dmah: |
| 23 |
length = u.get_mr_length() |
| 24 |
access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE |
| 25 |
with MREx(pd, length, access, dmah=dmah) as mr: |
| 26 |
pass |
| 27 |
|
| 28 |
@u.skip_unsupported |
| 29 |
def test_dmah_invalid_ph(self): |
| 30 |
"""Verify DMAHandle with invalid PH value, max ph value is 3.""" |
| 31 |
with d.Context(name=self.dev_name) as ctx: |
| 32 |
with PD(ctx) as pd: |
| 33 |
attr = u.create_dmah_init_attr(ph=4) |
| 34 |
with self.assertRaises(PyverbsError): |
| 35 |
DMAHandle(ctx, attr) |
| 36 |
|
| 37 |
@u.skip_unsupported |
| 38 |
def test_dmah_persistent_memory(self): |
| 39 |
"""Attempt to create DMAHandle targeting persistent memory.""" |
| 40 |
with d.Context(name=self.dev_name) as ctx: |
| 41 |
attr = u.create_dmah_init_attr(tph_mem_type=ibv_tph_mem_type.IBV_TPH_MEM_TYPE_PM) |
| 42 |
with DMAHandle(ctx, attr): |
| 43 |
pass |
| 44 |
|
| 45 |
@u.skip_unsupported |
| 46 |
def test_dmah_invalid_mem_type(self): |
| 47 |
"""Pass an unsupported TPH memory-type and verify provider rejects it.""" |
| 48 |
with d.Context(name=self.dev_name) as ctx: |
| 49 |
attr = u.create_dmah_init_attr(tph_mem_type=0xFE) |
| 50 |
with self.assertRaises(PyverbsError): |
| 51 |
DMAHandle(ctx, attr) |
| 52 |
|
| 53 |
@u.skip_unsupported |
| 54 |
def test_dmah_inval_cpu_id(self): |
| 55 |
"""Attempt to create DMAHandle with invalid CPU ID (0xffff). Expect failure.""" |
| 56 |
with d.Context(name=self.dev_name) as ctx: |
| 57 |
attr = u.create_dmah_init_attr(cpu_id=0xffff, ph=3) |
| 58 |
with self.assertRaises(PyverbsError): |
| 59 |
DMAHandle(ctx, attr) |
| 60 |
|
| 61 |
@u.skip_unsupported |
| 62 |
def test_dmah_mrex_odp_bad_flow(self): |
| 63 |
"""Attempt to register ODP-capable MREx with DMAHandle. |
| 64 |
Expect failure since ODP isn't supported with DMAHandle.""" |
| 65 |
with d.Context(name=self.dev_name) as ctx: |
| 66 |
# Check ODP support; skip if not available |
| 67 |
odp_cap = (ibv_odp_transport_cap_bits.IBV_ODP_SUPPORT_SEND | |
| 68 |
ibv_odp_transport_cap_bits.IBV_ODP_SUPPORT_RECV) |
| 69 |
u.odp_supported(ctx, 'rc', odp_cap) |
| 70 |
with PD(ctx) as pd: |
| 71 |
attr = u.create_dmah_init_attr() |
| 72 |
dmah = DMAHandle(ctx, attr) |
| 73 |
with self.assertRaises(PyverbsError): |
| 74 |
length = u.get_mr_length() |
| 75 |
access = (ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | |
| 76 |
ibv_access_flags.IBV_ACCESS_ON_DEMAND) |
| 77 |
MREx(pd, length, access, dmah=dmah) |
| 78 |
|
| 79 |
|
| 80 |
class DmaHandleMRExRC(RCResources): |
| 81 |
"""RC resource class that registers an MREx with a DMAHandle.""" |
| 82 |
|
| 83 |
def __init__(self, dev_name, ib_port, gid_index, |
| 84 |
mr_access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE, msg_size=1024): |
| 85 |
self.dmah = None |
| 86 |
self.mr_access = mr_access |
| 87 |
super().__init__(dev_name=dev_name, ib_port=ib_port, gid_index=gid_index, |
| 88 |
msg_size=msg_size) |
| 89 |
|
| 90 |
def create_dmah(self): |
| 91 |
"""Allocate a DMAHandle using the existing device Context.""" |
| 92 |
attr = u.create_dmah_init_attr() |
| 93 |
self.dmah = DMAHandle(self.ctx, attr) |
| 94 |
|
| 95 |
@u.skip_unsupported |
| 96 |
def create_mr(self): |
| 97 |
self.create_dmah() |
| 98 |
self.mr = MREx(self.pd, self.msg_size, self.mr_access, dmah=self.dmah) |
| 99 |
|
| 100 |
def create_qp_attr(self): |
| 101 |
qp_attr = QPAttr(port_num=self.ib_port) |
| 102 |
qp_access = (ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | |
| 103 |
ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | |
| 104 |
ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC) |
| 105 |
qp_attr.qp_access_flags = qp_access |
| 106 |
return qp_attr |
| 107 |
|
| 108 |
|
| 109 |
class DmaHandleTrafficTest(RDMATestCase): |
| 110 |
"""Traffic tests for MREx + DMAHandle combinations.""" |
| 111 |
|
| 112 |
def setUp(self): |
| 113 |
super().setUp() |
| 114 |
self.iters = 10 |
| 115 |
self.server = None |
| 116 |
self.client = None |
| 117 |
self.traffic_args = None |
| 118 |
|
| 119 |
def test_dmah_mrex_rc_send(self): |
| 120 |
"""Checks basic RC send/recv traffic with DMAHandle-registered MREx.""" |
| 121 |
access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE |
| 122 |
self.create_players(DmaHandleMRExRC, mr_access=access, msg_size=1024) |
| 123 |
u.traffic(**self.traffic_args) |
| 124 |
|
| 125 |
def test_dmah_mrex_rc_rdma_write(self): |
| 126 |
"""Validates RC RDMA Write traffic with DMAHandle & MREx.""" |
| 127 |
access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE |
| 128 |
self.create_players(DmaHandleMRExRC, mr_access=access, msg_size=1024) |
| 129 |
u.rdma_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE) |
| 130 |
|
| 131 |
def test_dmah_mrex_rc_atomic(self): |
| 132 |
"""Tests RC atomic fetch&add using a DMAHandle-backed MREx.""" |
| 133 |
access = (ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | |
| 134 |
ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC | |
| 135 |
ibv_access_flags.IBV_ACCESS_REMOTE_WRITE) |
| 136 |
self.create_players(DmaHandleMRExRC, mr_access=access, msg_size=8) |
| 137 |
u.atomic_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD) |