Debian Codegraph @

tests/test_mlx5_ooo_qp.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2024 NVIDIA Corporation . All rights reserved. See COPYING file
3
4 import unittest
5 import random
6 import errno
7
8 from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \
9 Mlx5DVQPInitAttr, Mlx5QP, Mlx5DVCQInitAttr, Mlx5CQ
10 from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError, \
11 PyverbsError
12 from pyverbs.cq import CQ, CQEX, PollCqAttr, CqInitAttrEx
13 from pyverbs.qp import QPInitAttrEx, QPCap, QPAttr
14 from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_context_comp_mask, mlx5dv_qp_init_attr_mask, \
15 mlx5dv_cq_init_attr_mask, mlx5dv_qp_create_flags
16 from pyverbs.wr import SGE, SendWR, RecvWR
17 from pyverbs.mr import MR
18 from pyverbs.libibverbs_enums import ibv_qp_type, ibv_qp_create_send_ops_flags, ibv_qp_init_attr_mask, \
19 ibv_access_flags, ibv_qp_type, ibv_wr_opcode, IBV_WC_STANDARD_FLAGS
20
21 from tests.base import RCResources, RDMATestCase
22 from tests.mlx5_base import Mlx5RcResources
23 import tests.utils as u
24
25
26 def create_ooo_dv_qp(res, max_recv_wr=1000, qp_type=ibv_qp_type.IBV_QPT_RC):
27 dv_ctx = res.ctx.query_mlx5_device()
28 if not dv_ctx.comp_mask & mlx5dv_context_comp_mask.MLX5DV_CONTEXT_MASK_OOO_RECV_WRS:
29 raise unittest.SkipTest('DV QP OOO feature is not supported')
30 send_ops_flags = ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND | ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND_WITH_IMM | \
31 ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE | ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_READ | \
32 ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM
33 qp_cap = QPCap(max_recv_wr=max_recv_wr, max_send_wr=max_recv_wr)
34 comp_mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD | ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS
35 qp_init_attr = QPInitAttrEx(cap=qp_cap, pd=res.pd, scq=res.cq,
36 rcq=res.cq, qp_type=qp_type,
37 send_ops_flags=send_ops_flags,
38 comp_mask=comp_mask)
39 dv_comp_mask = mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS
40 attr = Mlx5DVQPInitAttr(comp_mask=dv_comp_mask,
41 create_flags=res.dvqp_create_flags)
42 try:
43 qp = Mlx5QP(res.ctx, qp_init_attr, attr)
44 res.qps.append(qp)
45 res.qps_num.append(qp.qp_num)
46 res.psns.append(random.getrandbits(24))
47 except PyverbsRDMAError as ex:
48 raise ex
49
50
51 class Mlx5OOORcRes(Mlx5RcResources):
52 def __init__(self, dev_name, ib_port, gid_index, msg_size=1024, dvqp_create_flags=0, **kwargs):
53 """
54 Initialize mlx5 DV QP resources based on RCResources.
55 :param dev_name: Device name to be used
56 :param ib_port: IB port of the device to use
57 :param gid_index: Which GID index to use
58 :param msg_size: The resource msg size
59 :param dvqp_create_flags: DV QP create flags
60 :param kwargs: General arguments
61 """
62 self.qp_access_flags = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \
63 ibv_access_flags.IBV_ACCESS_REMOTE_READ
64 self.dvqp_create_flags = dvqp_create_flags
65 super().__init__(dev_name, ib_port, gid_index, msg_size=msg_size, **kwargs)
66
67
68 def create_qp_attr(self):
69 attr = super().create_qp_attr()
70 attr.qp_access_flags = self.qp_access_flags
71 return attr
72
73 def create_qps(self):
74 for _ in range(self.qp_count):
75 create_ooo_dv_qp(self)
76
77 def create_mr(self):
78 self.mr = MR(self.pd, self.msg_size, self.qp_access_flags)
79
80 def create_cq(self):
81 wc_flags = IBV_WC_STANDARD_FLAGS
82 cia = CqInitAttrEx(cqe=2000, wc_flags=wc_flags)
83 dvcq_init_attr = Mlx5DVCQInitAttr()
84 dvcq_init_attr.comp_mask |= mlx5dv_cq_init_attr_mask.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE
85 dvcq_init_attr.cqe_size = 64
86 dvcq_init_attr.comp_mask |= mlx5dv_cq_init_attr_mask.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE
87 try:
88 self.cq = Mlx5CQ(self.ctx, cia, dvcq_init_attr)
89 except PyverbsRDMAError as ex:
90 if ex.error_code == errno.EOPNOTSUPP:
91 raise unittest.SkipTest('Create Mlx5DV CQ is not supported')
92 raise ex
93
94
95 class DvOOOQPTest(RDMATestCase):
96
97 def test_ooo_qp_bad_flow(self):
98 """
99 DDP - OOO Recv WRs bad flow test
100 1. Create QP with max recv WRs possible and validate it with querying the QP
101 2. Try to create QP with more then max recv wr supported
102 3. Try to create QP with unsupported QP type
103 """
104 self.create_players(Mlx5OOORcRes, dvqp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_OOO_DP)
105 dv_ctx = self.server.ctx.query_mlx5_device()
106 max_rc_rwrs = dv_ctx.ooo_recv_wrs_caps['max_rc']
107 create_ooo_dv_qp(self.server, max_recv_wr=max_rc_rwrs)
108 attr, init_attr = self.server.qps[-1].query(0x1ffffff)
109 self.assertEqual(max_rc_rwrs, init_attr.cap.max_recv_wr)
110 # Try to create QP with more then max recv wr supported
111 with self.assertRaises(PyverbsRDMAError) as ex:
112 create_ooo_dv_qp(self.server, max_rc_rwrs + 1)
113 self.assertEqual(ex.exception.error_code, errno.EINVAL)
114 # Try to create QP with unsupported QP type
115 with self.assertRaises(PyverbsRDMAError) as ex:
116 create_ooo_dv_qp(self.server, qp_type=ibv_qp_type.IBV_QPT_RAW_PACKET)
117 self.assertEqual(ex.exception.error_code, errno.EOPNOTSUPP)
118
119 def test_ooo_qp_send_traffic(self):
120 """
121 DV QP OOO traffic opcode SEND
122 """
123 self.create_players(Mlx5OOORcRes, dvqp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_OOO_DP)
124 u.traffic_poll_at_once(self, msg_size=int(self.server.msg_size / self.iters),
125 iterations=self.iters)
126
127 def test_ooo_qp_rdma_write_imm_traffic(self):
128 """
129 DV QP OOO traffic opcode RDMA_WRITE_WITH_IMM
130 """
131 self.create_players(Mlx5OOORcRes, dvqp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_OOO_DP)
132 u.traffic_poll_at_once(self, msg_size=int(self.server.msg_size / self.iters),
133 iterations=self.iters, opcode=ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM)