| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright 2020-2025 Amazon.com, Inc. or its affiliates. All rights reserved. |
| 3 |
""" |
| 4 |
Test module for efa direct-verbs. |
| 5 |
""" |
| 6 |
|
| 7 |
import unittest |
| 8 |
import random |
| 9 |
import errno |
| 10 |
|
| 11 |
import pyverbs.providers.efa.efa_enums as efa_e |
| 12 |
from pyverbs.base import PyverbsRDMAError |
| 13 |
import pyverbs.providers.efa.efadv as efa |
| 14 |
from pyverbs.qp import QPInitAttrEx |
| 15 |
from pyverbs.addr import AHAttr |
| 16 |
from pyverbs.cq import CQ |
| 17 |
from pyverbs.libibverbs_enums import ibv_qp_type, ibv_qp_create_send_ops_flags, ibv_qp_init_attr_mask, \ |
| 18 |
ibv_wr_opcode, ibv_access_flags |
| 19 |
from pyverbs.pd import PD |
| 20 |
|
| 21 |
from tests.efa_base import EfaAPITestCase, EfaRDMATestCase, SRDResources |
| 22 |
import tests.utils as u |
| 23 |
|
| 24 |
|
| 25 |
|
| 26 |
class EfaQueryDeviceTest(EfaAPITestCase): |
| 27 |
""" |
| 28 |
Test various functionalities of the direct verbs class. |
| 29 |
""" |
| 30 |
def test_efadv_query(self): |
| 31 |
""" |
| 32 |
Verify that it's possible to read EFA direct-verbs. |
| 33 |
""" |
| 34 |
with efa.EfaContext(name=self.ctx.name) as efa_ctx: |
| 35 |
try: |
| 36 |
efa_attrs = efa_ctx.query_efa_device() |
| 37 |
if self.config['verbosity']: |
| 38 |
print(f'\n{efa_attrs}') |
| 39 |
except PyverbsRDMAError as ex: |
| 40 |
if ex.error_code == errno.EOPNOTSUPP: |
| 41 |
raise unittest.SkipTest('Not supported on non EFA devices') |
| 42 |
raise ex |
| 43 |
|
| 44 |
|
| 45 |
class EfaAHTest(EfaAPITestCase): |
| 46 |
""" |
| 47 |
Test functionality of the EfaAH class |
| 48 |
""" |
| 49 |
def test_efadv_query_ah(self): |
| 50 |
""" |
| 51 |
Test efadv_query_ah() |
| 52 |
""" |
| 53 |
pd = PD(self.ctx) |
| 54 |
try: |
| 55 |
gr = u.get_global_route(self.ctx, port_num=self.ib_port) |
| 56 |
ah_attr = AHAttr(gr=gr, is_global=1, port_num=self.ib_port) |
| 57 |
ah = efa.EfaAH(pd, attr=ah_attr) |
| 58 |
query_ah_attr = ah.query_efa_ah() |
| 59 |
if self.config['verbosity']: |
| 60 |
print(f'\n{query_ah_attr}') |
| 61 |
except PyverbsRDMAError as ex: |
| 62 |
if ex.error_code == errno.EOPNOTSUPP: |
| 63 |
raise unittest.SkipTest('Not supported on non EFA devices') |
| 64 |
raise ex |
| 65 |
|
| 66 |
|
| 67 |
class EfaQPTest(EfaAPITestCase): |
| 68 |
""" |
| 69 |
Test SRD QP class |
| 70 |
""" |
| 71 |
def test_efadv_create_driver_qp(self): |
| 72 |
""" |
| 73 |
Test efadv_create_driver_qp() |
| 74 |
""" |
| 75 |
with PD(self.ctx) as pd: |
| 76 |
with CQ(self.ctx, 100) as cq: |
| 77 |
qia = u.get_qp_init_attr(cq, self.attr) |
| 78 |
qia.qp_type = ibv_qp_type.IBV_QPT_DRIVER |
| 79 |
try: |
| 80 |
qp = efa.SRDQP(pd, qia) |
| 81 |
except PyverbsRDMAError as ex: |
| 82 |
if ex.error_code == errno.EOPNOTSUPP: |
| 83 |
raise unittest.SkipTest("Create SRD QP is not supported") |
| 84 |
raise ex |
| 85 |
|
| 86 |
|
| 87 |
class EfaQPExTest(EfaAPITestCase): |
| 88 |
""" |
| 89 |
Test SRD QPEx class |
| 90 |
""" |
| 91 |
def test_efadv_create_qp_ex(self): |
| 92 |
""" |
| 93 |
Test efadv_create_qp_ex() |
| 94 |
""" |
| 95 |
with PD(self.ctx) as pd: |
| 96 |
with CQ(self.ctx, 100) as cq: |
| 97 |
qiaEx = get_qp_init_attr_ex(cq, pd, self.attr) |
| 98 |
efaqia = efa.EfaQPInitAttr() |
| 99 |
efaqia.driver_qp_type = efa_e.EFADV_QP_DRIVER_TYPE_SRD |
| 100 |
try: |
| 101 |
qp = efa.SRDQPEx(self.ctx, qiaEx, efaqia) |
| 102 |
except PyverbsRDMAError as ex: |
| 103 |
if ex.error_code == errno.EOPNOTSUPP: |
| 104 |
raise unittest.SkipTest("Create SRD QPEx is not supported") |
| 105 |
raise ex |
| 106 |
|
| 107 |
|
| 108 |
def get_random_send_op_flags(): |
| 109 |
send_ops_flags = [ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, |
| 110 |
ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND_WITH_IMM, |
| 111 |
ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_READ, |
| 112 |
ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE, |
| 113 |
ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM] |
| 114 |
selected = u.sample(send_ops_flags) |
| 115 |
selected_ops_flags = 0 |
| 116 |
for s in selected: |
| 117 |
selected_ops_flags += s.value |
| 118 |
return selected_ops_flags |
| 119 |
|
| 120 |
def get_qp_init_attr_ex(cq, pd, attr): |
| 121 |
qp_cap = u.random_qp_cap(attr) |
| 122 |
sig = random.randint(0, 1) |
| 123 |
mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD | ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS |
| 124 |
send_ops_flags = get_random_send_op_flags() |
| 125 |
qia = QPInitAttrEx(qp_type=ibv_qp_type.IBV_QPT_DRIVER, cap=qp_cap, sq_sig_all=sig, comp_mask=mask, |
| 126 |
create_flags=0, max_tso_header=0, send_ops_flags=send_ops_flags) |
| 127 |
qia.send_cq = cq |
| 128 |
qia.recv_cq = cq |
| 129 |
qia.pd = pd |
| 130 |
return qia |
| 131 |
|
| 132 |
|
| 133 |
class EfaCqTest(EfaRDMATestCase): |
| 134 |
def setUp(self): |
| 135 |
super().setUp() |
| 136 |
self.iters = 100 |
| 137 |
self.server = None |
| 138 |
self.client = None |
| 139 |
|
| 140 |
def create_players(self, dev_cap, wc_flags, send_ops_flags, qp_count=8): |
| 141 |
super().create_players(SRDResources, send_ops_flags=send_ops_flags, qp_count=qp_count, |
| 142 |
required_dev_cap=dev_cap, wc_flags=wc_flags) |
| 143 |
self.server.remote_gid = self.client.ctx.query_gid(self.client.ib_port, self.client.gid_index) |
| 144 |
|
| 145 |
def test_dv_cq_ex_with_sgid(self): |
| 146 |
wc_flag = efa_e.EFADV_WC_EX_WITH_SGID |
| 147 |
dev_cap = efa_e.EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID |
| 148 |
self.create_players(dev_cap, wc_flag, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1) |
| 149 |
recv_wr = u.get_recv_wr(self.server) |
| 150 |
self.server.qps[0].post_recv(recv_wr) |
| 151 |
ah_client = u.get_global_ah(self.client, self.gid_index, self.ib_port) |
| 152 |
_ , sg = u.get_send_elements(self.client, False) |
| 153 |
u.send(self.client, sg, ibv_wr_opcode.IBV_WR_SEND, new_send=True, qp_idx=0, ah=ah_client) |
| 154 |
u.poll_cq_ex(self.client.cq) |
| 155 |
u.poll_cq_ex(self.server.cq, sgid=self.server.remote_gid) |
| 156 |
|
| 157 |
|
| 158 |
class EfaMRTest(EfaAPITestCase): |
| 159 |
""" |
| 160 |
Test various functionalities of the EfaMR class. |
| 161 |
""" |
| 162 |
def test_efadv_query_mr(self): |
| 163 |
with PD(self.ctx) as pd: |
| 164 |
try: |
| 165 |
mr = efa.EfaMR(pd, 16, ibv_access_flags.IBV_ACCESS_LOCAL_WRITE) |
| 166 |
mr_attrs = mr.query() |
| 167 |
if self.config['verbosity']: |
| 168 |
print(f'\n{mr_attrs}') |
| 169 |
|
| 170 |
assert(mr_attrs.ic_id_validity & ~(efa_e.EFADV_MR_ATTR_VALIDITY_RECV_IC_ID | |
| 171 |
efa_e.EFADV_MR_ATTR_VALIDITY_RDMA_READ_IC_ID | |
| 172 |
efa_e.EFADV_MR_ATTR_VALIDITY_RDMA_RECV_IC_ID) == 0) |
| 173 |
except PyverbsRDMAError as ex: |
| 174 |
if ex.error_code in [errno.EOPNOTSUPP, errno.ENOTTY, errno.EPROTONOSUPPORT]: |
| 175 |
raise unittest.SkipTest(f'Query MR not supported, errno={ex.error_code}') |
| 176 |
raise ex |