Debian Codegraph @

tests/test_mlx5_rdmacm.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2020 Nvidia Corporation. All rights reserved. See COPYING file
3
4 import unittest
5 import errno
6
7 from pyverbs.providers.mlx5.mlx5dv import Mlx5DVQPInitAttr, Mlx5QP, \
8 Mlx5DVDCInitAttr, Mlx5Context
9 from tests.test_rdmacm import CMAsyncConnection
10 from tests.mlx5_base import Mlx5PyverbsAPITestCase, Mlx5RDMACMBaseTest
11 from pyverbs.pyverbs_error import PyverbsRDMAError
12 from pyverbs.srq import SRQ, SrqInitAttr, SrqAttr
13 from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_qp_init_attr_mask, mlx5dv_dc_type
14 from tests.base_rdmacm import AsyncCMResources
15 from pyverbs.qp import QPCap, QPInitAttrEx
16 from pyverbs.cmid import ConnParam
17 from tests.base import DCT_KEY
18 from pyverbs.addr import AH
19 from pyverbs.libibverbs_enums import ibv_wr_opcode, ibv_qp_create_send_ops_flags, ibv_qp_init_attr_mask, \
20 ibv_qp_type, ibv_qp_state
21 from pyverbs.cq import CQ
22 import tests.utils as u
23
24
25 class DcCMConnection(CMAsyncConnection):
26 """
27 Implement RDMACM connection management for asynchronous CMIDs using DC as
28 an external QP.
29 """
30 def create_cm_res(self, ip_addr, passive, **kwargs):
31 self.cm_res = DcCMResources(addr=ip_addr, passive=passive, **kwargs)
32 if passive:
33 self.cm_res.create_cmid()
34 else:
35 for conn_idx in range(self.num_conns):
36 self.cm_res.create_cmid(conn_idx)
37 self.get_and_update_addrinfo(passive, ip_addr)
38
39 def _ext_qp_server_traffic(self):
40 recv_wr = u.get_recv_wr(self.cm_res)
41 for _ in range(self.cm_res.num_msgs):
42 u.post_recv(self.cm_res, recv_wr)
43 self.syncer.wait()
44 for _ in range(self.cm_res.num_msgs):
45 u.poll_cq(self.cm_res.cq)
46
47 def _ext_qp_client_traffic(self):
48 self.cm_res.remote_dct_num = self.cm_res.remote_qpn
49 _, send_wr = u.get_send_elements(self.cm_res, self.cm_res.passive)
50 ah = AH(self.cm_res.cmid.pd, attr=self.cm_res.remote_ah)
51 self.syncer.wait()
52 for send_idx in range(self.cm_res.num_msgs):
53 dci_idx = send_idx % len(self.cm_res.qps)
54 u.post_send_ex(self.cm_res, send_wr, ibv_wr_opcode.IBV_WR_SEND, ah=ah,
55 qp_idx=dci_idx)
56 u.poll_cq(self.cm_res.cq)
57
58 def disconnect(self):
59 if self.cm_res.reserved_qp_num and self.cm_res.passive:
60 Mlx5Context.reserved_qpn_dealloc(self.cm_res.child_id.context,
61 self.cm_res.reserved_qp_num)
62 self.cm_res.reserved_qp_num = 0
63 super().disconnect()
64
65
66 class DcCMResources(AsyncCMResources):
67 """
68 DcCMResources class contains resources for RDMA CM asynchronous
69 communication using DC as an external QP.
70 """
71 def __init__(self, addr=None, passive=None, **kwargs):
72 """
73 Init DcCMResources instance.
74 :param passive: Indicate if this CM is the passive CM.
75 """
76 super().__init__(passive=passive, **kwargs)
77 self.srq = None
78 self.remote_dct_num = None
79 self.reserved_qp_num = 0
80
81 def create_qp(self, conn_idx=0):
82 """
83 Create an RDMACM QP. If self.with_ext_qp is set, then an external CQ and
84 DC QP will be created. In case that CQ is already created, it is used
85 for the newly created QP.
86 """
87 try:
88 if not self.passive:
89 # Create the DCI QPs.
90 cmid = self.cmids[conn_idx]
91 self.create_cq(cmid)
92 qp_init_attr = self.create_qp_init_attr(cmid, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND)
93 attr = Mlx5DVQPInitAttr(comp_mask=mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_DC,
94 dc_init_attr=Mlx5DVDCInitAttr())
95 self.qps[conn_idx] = Mlx5QP(cmid.context, qp_init_attr, attr)
96
97 if self.passive and conn_idx == 0:
98 # Create the DCT QP only for the first connection.
99 cmid = self.child_id
100 self.create_cq(cmid)
101 self.create_srq(cmid)
102 qp_init_attr = self.create_qp_init_attr(cmid)
103 dc_attr = Mlx5DVDCInitAttr(dc_type=mlx5dv_dc_type.MLX5DV_DCTYPE_DCT,
104 dct_access_key=DCT_KEY)
105 attr = Mlx5DVQPInitAttr(comp_mask=mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_DC,
106 dc_init_attr=dc_attr)
107 self.qps[conn_idx] = Mlx5QP(cmid.context, qp_init_attr, attr)
108 except PyverbsRDMAError as ex:
109 if ex.error_code == errno.EOPNOTSUPP:
110 raise unittest.SkipTest('Create DC QP is not supported')
111 raise ex
112
113 def create_qp_cap(self):
114 return QPCap(self.num_msgs, 0, 1, 0)
115
116 def create_qp_init_attr(self, cmid, send_ops_flags=0):
117 comp_mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD
118 if send_ops_flags:
119 comp_mask |= ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS
120 return QPInitAttrEx(cap=self.create_qp_cap(), pd=cmid.pd, scq=self.cq,
121 rcq=self.cq, srq=self.srq, qp_type=ibv_qp_type.IBV_QPT_DRIVER,
122 send_ops_flags=send_ops_flags, comp_mask=comp_mask,
123 sq_sig_all=1)
124
125 def create_srq(self, cmid):
126 srq_init_attr = SrqInitAttr(SrqAttr(max_wr=self.num_msgs))
127 try:
128 self.srq = SRQ(cmid.pd, srq_init_attr)
129 except PyverbsRDMAError as ex:
130 if ex.error_code == errno.EOPNOTSUPP:
131 raise unittest.SkipTest('Create SRQ is not supported')
132 raise ex
133
134 def modify_ext_qp_to_rts(self, conn_idx=0):
135 cmids = self.child_ids if self.passive else self.cmids
136 if not self.passive or not conn_idx:
137 qp = self.qps[conn_idx]
138 attr, _ = cmids[conn_idx].init_qp_attr(ibv_qp_state.IBV_QPS_INIT)
139 qp.to_init(attr)
140 attr, _ = cmids[conn_idx].init_qp_attr(ibv_qp_state.IBV_QPS_RTR)
141 qp.to_rtr(attr)
142 if not self.passive:
143 # The passive QP is DCT which should stay in RTR state.
144 self.remote_ah = attr.ah_attr
145 attr, _ = cmids[conn_idx].init_qp_attr(ibv_qp_state.IBV_QPS_RTS)
146 qp.to_rts(attr)
147
148 def create_conn_param(self, qp_num=0, conn_idx=0):
149 if conn_idx and self.passive:
150 try:
151 ctx = self.child_id.context
152 self.reserved_qp_num = Mlx5Context.reserved_qpn_alloc(ctx)
153 except PyverbsRDMAError as ex:
154 if ex.error_code == errno.EOPNOTSUPP:
155 raise unittest.SkipTest('Alloc reserved QP number is not supported')
156 raise ex
157 qp_num = self.reserved_qp_num
158 else:
159 qp_num = self.qps[conn_idx].qp_num
160 return ConnParam(qp_num=qp_num)
161
162
163 class Mlx5CMTestCase(Mlx5RDMACMBaseTest):
164 """
165 Mlx5 RDMACM test class.
166 """
167 def test_rdmacm_async_traffic_dc_external_qp(self):
168 """
169 Connect multiple RDMACM connections using DC as an external QP for
170 traffic.
171 """
172 self.two_nodes_rdmacm_traffic(DcCMConnection, self.rdmacm_traffic,
173 with_ext_qp=True, num_conns=2)
174
175
176 class ReservedQPTest(Mlx5PyverbsAPITestCase):
177
178 def test_reservered_qpn(self):
179 """
180 Alloc reserved qpn multiple times and then dealloc the qpns. In addition,
181 the test includes bad flows where a fake qpn gets deallocated, and a
182 real qpn gets deallocated twice.
183 """
184 try:
185 # Alloc qp number multiple times.
186 qpns = []
187 for i in range(1000):
188 qpns.append(Mlx5Context.reserved_qpn_alloc(self.ctx))
189 for i in range(1000):
190 Mlx5Context.reserved_qpn_dealloc(self.ctx, qpns[i])
191
192 # Dealloc qp number that was not allocated.
193 qpn = Mlx5Context.reserved_qpn_alloc(self.ctx)
194 with self.assertRaises(PyverbsRDMAError) as ex:
195 fake_qpn = qpn - 1
196 Mlx5Context.reserved_qpn_dealloc(self.ctx, fake_qpn)
197 self.assertEqual(ex.exception.error_code, errno.EINVAL)
198
199 # Try to dealloc same qp number twice.
200 Mlx5Context.reserved_qpn_dealloc(self.ctx, qpn)
201 with self.assertRaises(PyverbsRDMAError) as ex:
202 Mlx5Context.reserved_qpn_dealloc(self.ctx, qpn)
203 self.assertEqual(ex.exception.error_code, errno.EINVAL)
204
205 except PyverbsRDMAError as ex:
206 if ex.error_code == errno.EOPNOTSUPP:
207 raise unittest.SkipTest('Alloc reserved QP number is not supported')
208 raise ex