# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2024 NVIDIA Corporation . All rights reserved. See COPYING file import unittest import random import errno from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \ Mlx5DVQPInitAttr, Mlx5QP, Mlx5DVCQInitAttr, Mlx5CQ from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError, \ PyverbsError from pyverbs.cq import CQ, CQEX, PollCqAttr, CqInitAttrEx from pyverbs.qp import QPInitAttrEx, QPCap, QPAttr from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_context_comp_mask, mlx5dv_qp_init_attr_mask, \ mlx5dv_cq_init_attr_mask, mlx5dv_qp_create_flags from pyverbs.wr import SGE, SendWR, RecvWR from pyverbs.mr import MR from pyverbs.libibverbs_enums import ibv_qp_type, ibv_qp_create_send_ops_flags, ibv_qp_init_attr_mask, \ ibv_access_flags, ibv_qp_type, ibv_wr_opcode, IBV_WC_STANDARD_FLAGS from tests.base import RCResources, RDMATestCase from tests.mlx5_base import Mlx5RcResources import tests.utils as u def create_ooo_dv_qp(res, max_recv_wr=1000, qp_type=ibv_qp_type.IBV_QPT_RC): dv_ctx = res.ctx.query_mlx5_device() if not dv_ctx.comp_mask & mlx5dv_context_comp_mask.MLX5DV_CONTEXT_MASK_OOO_RECV_WRS: raise unittest.SkipTest('DV QP OOO feature is not supported') send_ops_flags = ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND | ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND_WITH_IMM | \ ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE | ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_READ | \ ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM qp_cap = QPCap(max_recv_wr=max_recv_wr, max_send_wr=max_recv_wr) comp_mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD | ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS qp_init_attr = QPInitAttrEx(cap=qp_cap, pd=res.pd, scq=res.cq, rcq=res.cq, qp_type=qp_type, send_ops_flags=send_ops_flags, comp_mask=comp_mask) dv_comp_mask = mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS attr = Mlx5DVQPInitAttr(comp_mask=dv_comp_mask, create_flags=res.dvqp_create_flags) try: qp = Mlx5QP(res.ctx, qp_init_attr, attr) res.qps.append(qp) res.qps_num.append(qp.qp_num) res.psns.append(random.getrandbits(24)) except PyverbsRDMAError as ex: raise ex class Mlx5OOORcRes(Mlx5RcResources): def __init__(self, dev_name, ib_port, gid_index, msg_size=1024, dvqp_create_flags=0, **kwargs): """ Initialize mlx5 DV QP resources based on RCResources. :param dev_name: Device name to be used :param ib_port: IB port of the device to use :param gid_index: Which GID index to use :param msg_size: The resource msg size :param dvqp_create_flags: DV QP create flags :param kwargs: General arguments """ self.qp_access_flags = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \ ibv_access_flags.IBV_ACCESS_REMOTE_READ self.dvqp_create_flags = dvqp_create_flags super().__init__(dev_name, ib_port, gid_index, msg_size=msg_size, **kwargs) def create_qp_attr(self): attr = super().create_qp_attr() attr.qp_access_flags = self.qp_access_flags return attr def create_qps(self): for _ in range(self.qp_count): create_ooo_dv_qp(self) def create_mr(self): self.mr = MR(self.pd, self.msg_size, self.qp_access_flags) def create_cq(self): wc_flags = IBV_WC_STANDARD_FLAGS cia = CqInitAttrEx(cqe=2000, wc_flags=wc_flags) dvcq_init_attr = Mlx5DVCQInitAttr() dvcq_init_attr.comp_mask |= mlx5dv_cq_init_attr_mask.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE dvcq_init_attr.cqe_size = 64 dvcq_init_attr.comp_mask |= mlx5dv_cq_init_attr_mask.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE try: self.cq = Mlx5CQ(self.ctx, cia, dvcq_init_attr) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create Mlx5DV CQ is not supported') raise ex class DvOOOQPTest(RDMATestCase): def test_ooo_qp_bad_flow(self): """ DDP - OOO Recv WRs bad flow test 1. Create QP with max recv WRs possible and validate it with querying the QP 2. Try to create QP with more then max recv wr supported 3. Try to create QP with unsupported QP type """ self.create_players(Mlx5OOORcRes, dvqp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_OOO_DP) dv_ctx = self.server.ctx.query_mlx5_device() max_rc_rwrs = dv_ctx.ooo_recv_wrs_caps['max_rc'] create_ooo_dv_qp(self.server, max_recv_wr=max_rc_rwrs) attr, init_attr = self.server.qps[-1].query(0x1ffffff) self.assertEqual(max_rc_rwrs, init_attr.cap.max_recv_wr) # Try to create QP with more then max recv wr supported with self.assertRaises(PyverbsRDMAError) as ex: create_ooo_dv_qp(self.server, max_rc_rwrs + 1) self.assertEqual(ex.exception.error_code, errno.EINVAL) # Try to create QP with unsupported QP type with self.assertRaises(PyverbsRDMAError) as ex: create_ooo_dv_qp(self.server, qp_type=ibv_qp_type.IBV_QPT_RAW_PACKET) self.assertEqual(ex.exception.error_code, errno.EOPNOTSUPP) def test_ooo_qp_send_traffic(self): """ DV QP OOO traffic opcode SEND """ self.create_players(Mlx5OOORcRes, dvqp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_OOO_DP) u.traffic_poll_at_once(self, msg_size=int(self.server.msg_size / self.iters), iterations=self.iters) def test_ooo_qp_rdma_write_imm_traffic(self): """ DV QP OOO traffic opcode RDMA_WRITE_WITH_IMM """ self.create_players(Mlx5OOORcRes, dvqp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_OOO_DP) u.traffic_poll_at_once(self, msg_size=int(self.server.msg_size / self.iters), iterations=self.iters, opcode=ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM)