| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright (c) 2024 NVIDIA Corporation . All rights reserved. See COPYING file |
| 3 |
|
| 4 |
import unittest |
| 5 |
import random |
| 6 |
import errno |
| 7 |
|
| 8 |
from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \ |
| 9 |
Mlx5DVQPInitAttr, Mlx5QP, Mlx5DVCQInitAttr, Mlx5CQ |
| 10 |
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError, \ |
| 11 |
PyverbsError |
| 12 |
from pyverbs.cq import CQ, CQEX, PollCqAttr, CqInitAttrEx |
| 13 |
from pyverbs.qp import QPInitAttrEx, QPCap, QPAttr |
| 14 |
from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_context_comp_mask, mlx5dv_qp_init_attr_mask, \ |
| 15 |
mlx5dv_cq_init_attr_mask, mlx5dv_qp_create_flags |
| 16 |
from pyverbs.wr import SGE, SendWR, RecvWR |
| 17 |
from pyverbs.mr import MR |
| 18 |
from pyverbs.libibverbs_enums import ibv_qp_type, ibv_qp_create_send_ops_flags, ibv_qp_init_attr_mask, \ |
| 19 |
ibv_access_flags, ibv_qp_type, ibv_wr_opcode, IBV_WC_STANDARD_FLAGS |
| 20 |
|
| 21 |
from tests.base import RCResources, RDMATestCase |
| 22 |
from tests.mlx5_base import Mlx5RcResources |
| 23 |
import tests.utils as u |
| 24 |
|
| 25 |
|
| 26 |
def create_ooo_dv_qp(res, max_recv_wr=1000, qp_type=ibv_qp_type.IBV_QPT_RC): |
| 27 |
dv_ctx = res.ctx.query_mlx5_device() |
| 28 |
if not dv_ctx.comp_mask & mlx5dv_context_comp_mask.MLX5DV_CONTEXT_MASK_OOO_RECV_WRS: |
| 29 |
raise unittest.SkipTest('DV QP OOO feature is not supported') |
| 30 |
send_ops_flags = ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND | ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND_WITH_IMM | \ |
| 31 |
ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE | ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_READ | \ |
| 32 |
ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM |
| 33 |
qp_cap = QPCap(max_recv_wr=max_recv_wr, max_send_wr=max_recv_wr) |
| 34 |
comp_mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD | ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS |
| 35 |
qp_init_attr = QPInitAttrEx(cap=qp_cap, pd=res.pd, scq=res.cq, |
| 36 |
rcq=res.cq, qp_type=qp_type, |
| 37 |
send_ops_flags=send_ops_flags, |
| 38 |
comp_mask=comp_mask) |
| 39 |
dv_comp_mask = mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS |
| 40 |
attr = Mlx5DVQPInitAttr(comp_mask=dv_comp_mask, |
| 41 |
create_flags=res.dvqp_create_flags) |
| 42 |
try: |
| 43 |
qp = Mlx5QP(res.ctx, qp_init_attr, attr) |
| 44 |
res.qps.append(qp) |
| 45 |
res.qps_num.append(qp.qp_num) |
| 46 |
res.psns.append(random.getrandbits(24)) |
| 47 |
except PyverbsRDMAError as ex: |
| 48 |
raise ex |
| 49 |
|
| 50 |
|
| 51 |
class Mlx5OOORcRes(Mlx5RcResources): |
| 52 |
def __init__(self, dev_name, ib_port, gid_index, msg_size=1024, dvqp_create_flags=0, **kwargs): |
| 53 |
""" |
| 54 |
Initialize mlx5 DV QP resources based on RCResources. |
| 55 |
:param dev_name: Device name to be used |
| 56 |
:param ib_port: IB port of the device to use |
| 57 |
:param gid_index: Which GID index to use |
| 58 |
:param msg_size: The resource msg size |
| 59 |
:param dvqp_create_flags: DV QP create flags |
| 60 |
:param kwargs: General arguments |
| 61 |
""" |
| 62 |
self.qp_access_flags = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \ |
| 63 |
ibv_access_flags.IBV_ACCESS_REMOTE_READ |
| 64 |
self.dvqp_create_flags = dvqp_create_flags |
| 65 |
super().__init__(dev_name, ib_port, gid_index, msg_size=msg_size, **kwargs) |
| 66 |
|
| 67 |
|
| 68 |
def create_qp_attr(self): |
| 69 |
attr = super().create_qp_attr() |
| 70 |
attr.qp_access_flags = self.qp_access_flags |
| 71 |
return attr |
| 72 |
|
| 73 |
def create_qps(self): |
| 74 |
for _ in range(self.qp_count): |
| 75 |
create_ooo_dv_qp(self) |
| 76 |
|
| 77 |
def create_mr(self): |
| 78 |
self.mr = MR(self.pd, self.msg_size, self.qp_access_flags) |
| 79 |
|
| 80 |
def create_cq(self): |
| 81 |
wc_flags = IBV_WC_STANDARD_FLAGS |
| 82 |
cia = CqInitAttrEx(cqe=2000, wc_flags=wc_flags) |
| 83 |
dvcq_init_attr = Mlx5DVCQInitAttr() |
| 84 |
dvcq_init_attr.comp_mask |= mlx5dv_cq_init_attr_mask.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE |
| 85 |
dvcq_init_attr.cqe_size = 64 |
| 86 |
dvcq_init_attr.comp_mask |= mlx5dv_cq_init_attr_mask.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE |
| 87 |
try: |
| 88 |
self.cq = Mlx5CQ(self.ctx, cia, dvcq_init_attr) |
| 89 |
except PyverbsRDMAError as ex: |
| 90 |
if ex.error_code == errno.EOPNOTSUPP: |
| 91 |
raise unittest.SkipTest('Create Mlx5DV CQ is not supported') |
| 92 |
raise ex |
| 93 |
|
| 94 |
|
| 95 |
class DvOOOQPTest(RDMATestCase): |
| 96 |
|
| 97 |
def test_ooo_qp_bad_flow(self): |
| 98 |
""" |
| 99 |
DDP - OOO Recv WRs bad flow test |
| 100 |
1. Create QP with max recv WRs possible and validate it with querying the QP |
| 101 |
2. Try to create QP with more then max recv wr supported |
| 102 |
3. Try to create QP with unsupported QP type |
| 103 |
""" |
| 104 |
self.create_players(Mlx5OOORcRes, dvqp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_OOO_DP) |
| 105 |
dv_ctx = self.server.ctx.query_mlx5_device() |
| 106 |
max_rc_rwrs = dv_ctx.ooo_recv_wrs_caps['max_rc'] |
| 107 |
create_ooo_dv_qp(self.server, max_recv_wr=max_rc_rwrs) |
| 108 |
attr, init_attr = self.server.qps[-1].query(0x1ffffff) |
| 109 |
self.assertEqual(max_rc_rwrs, init_attr.cap.max_recv_wr) |
| 110 |
# Try to create QP with more then max recv wr supported |
| 111 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 112 |
create_ooo_dv_qp(self.server, max_rc_rwrs + 1) |
| 113 |
self.assertEqual(ex.exception.error_code, errno.EINVAL) |
| 114 |
# Try to create QP with unsupported QP type |
| 115 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 116 |
create_ooo_dv_qp(self.server, qp_type=ibv_qp_type.IBV_QPT_RAW_PACKET) |
| 117 |
self.assertEqual(ex.exception.error_code, errno.EOPNOTSUPP) |
| 118 |
|
| 119 |
def test_ooo_qp_send_traffic(self): |
| 120 |
""" |
| 121 |
DV QP OOO traffic opcode SEND |
| 122 |
""" |
| 123 |
self.create_players(Mlx5OOORcRes, dvqp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_OOO_DP) |
| 124 |
u.traffic_poll_at_once(self, msg_size=int(self.server.msg_size / self.iters), |
| 125 |
iterations=self.iters) |
| 126 |
|
| 127 |
def test_ooo_qp_rdma_write_imm_traffic(self): |
| 128 |
""" |
| 129 |
DV QP OOO traffic opcode RDMA_WRITE_WITH_IMM |
| 130 |
""" |
| 131 |
self.create_players(Mlx5OOORcRes, dvqp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_OOO_DP) |
| 132 |
u.traffic_poll_at_once(self, msg_size=int(self.server.msg_size / self.iters), |
| 133 |
iterations=self.iters, opcode=ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM) |