Debian Codegraph @

tests/test_qpex.py

raw
1 import unittest
2 import random
3 import errno
4
5 from pyverbs.qp import QPCap, QPInitAttrEx, QPAttr, QPEx, QP
6 from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError
7 from pyverbs.mr import MW, MWBindInfo
8 from pyverbs.base import inc_rkey
9 from tests.utils import wc_status_to_str
10 from pyverbs.libibverbs_enums import ibv_qp_type, ibv_qp_create_send_ops_flags, ibv_qp_init_attr_mask, \
11 ibv_access_flags, ibv_placement_type, ibv_selectivity_level, ibv_wr_opcode, ibv_wc_status, \
12 ibv_send_flags, ibv_mw_type
13
14 from tests.base import UDResources, RCResources, RDMATestCase, XRCResources
15 import tests.utils as u
16
17 class QpExUDSend(UDResources):
18 def create_qps(self):
19 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_UD, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND)
20
21
22 class QpExRCSend(RCResources):
23 def create_qps(self):
24 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND)
25
26
27 class QpExXRCSend(XRCResources):
28 def create_qps(self):
29 qp_attr = QPAttr(port_num=self.ib_port)
30 qp_attr.pkey_index = 0
31 for _ in range(self.qp_count):
32 attr_ex = QPInitAttrEx(qp_type=ibv_qp_type.IBV_QPT_XRC_RECV,
33 comp_mask=ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_XRCD,
34 xrcd=self.xrcd)
35 qp_attr.qp_access_flags = ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \
36 ibv_access_flags.IBV_ACCESS_REMOTE_READ
37 recv_qp = QP(self.ctx, attr_ex, qp_attr)
38 self.rqp_lst.append(recv_qp)
39
40 send_qp = u.create_qp_ex(self, ibv_qp_type.IBV_QPT_XRC_SEND, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND)
41 self.sqp_lst.append(send_qp)
42 self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
43 self.psns.append(random.getrandbits(24))
44
45
46 class QpExUDSendImm(UDResources):
47 def create_qps(self):
48 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_UD, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND_WITH_IMM)
49
50
51 class QpExRCSendImm(RCResources):
52 def create_qps(self):
53 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND_WITH_IMM)
54
55
56 class QpExXRCSendImm(XRCResources):
57 def create_qps(self):
58 qp_attr = QPAttr(port_num=self.ib_port)
59 qp_attr.pkey_index = 0
60 for _ in range(self.qp_count):
61 attr_ex = QPInitAttrEx(qp_type=ibv_qp_type.IBV_QPT_XRC_RECV,
62 comp_mask=ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_XRCD,
63 xrcd=self.xrcd)
64 qp_attr.qp_access_flags = ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \
65 ibv_access_flags.IBV_ACCESS_REMOTE_READ
66 recv_qp = QP(self.ctx, attr_ex, qp_attr)
67 self.rqp_lst.append(recv_qp)
68
69 send_qp = u.create_qp_ex(self, ibv_qp_type.IBV_QPT_XRC_SEND,
70 ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND_WITH_IMM)
71 self.sqp_lst.append(send_qp)
72 self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
73 self.psns.append(random.getrandbits(24))
74
75
76 class QpExRCFlush(RCResources):
77 ptype = ibv_placement_type.IBV_FLUSH_GLOBAL
78 level = ibv_selectivity_level.IBV_FLUSH_RANGE
79 def create_qps(self):
80 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC,
81 ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_FLUSH | \
82 ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE)
83
84 def create_mr(self):
85 try:
86 self.mr = u.create_custom_mr(self, ibv_access_flags.IBV_ACCESS_FLUSH_GLOBAL | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE)
87 except PyverbsRDMAError as ex:
88 if ex.error_code == errno.EINVAL:
89 raise unittest.SkipTest('Create mr with IBV_ACCESS_FLUSH_GLOBAL access flag is not supported in kernel')
90 raise ex
91
92
93 class QpExRCAtomicWrite(RCResources):
94 def create_qps(self):
95 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_ATOMIC_WRITE)
96
97 def create_mr(self):
98 self.mr = u.create_custom_mr(self, ibv_access_flags.IBV_ACCESS_REMOTE_WRITE)
99
100
101 class QpExRCRDMAWrite(RCResources):
102 def create_qps(self):
103 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE)
104
105 def create_mr(self):
106 self.mr = u.create_custom_mr(self, ibv_access_flags.IBV_ACCESS_REMOTE_WRITE)
107
108
109 class QpExRCRDMAWriteImm(RCResources):
110 def create_qps(self):
111 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)
112
113 def create_mr(self):
114 self.mr = u.create_custom_mr(self, ibv_access_flags.IBV_ACCESS_REMOTE_WRITE)
115
116
117 class QpExRCRDMARead(RCResources):
118 def create_qps(self):
119 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_READ)
120
121 def create_mr(self):
122 self.mr = u.create_custom_mr(self, ibv_access_flags.IBV_ACCESS_REMOTE_READ)
123
124
125 class QpExRCAtomicCmpSwp(RCResources):
126 def create_qps(self):
127 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP)
128 self.mr = u.create_custom_mr(self, ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC)
129
130
131 class QpExRCAtomicFetchAdd(RCResources):
132 def create_qps(self):
133 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
134 self.mr = u.create_custom_mr(self, ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC)
135
136
137 class QpExRCBindMw(RCResources):
138 def create_qps(self):
139 u.create_qp_ex(self, ibv_qp_type.IBV_QPT_RC, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE |
140 ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_BIND_MW)
141
142 def create_mr(self):
143 self.mr = u.create_custom_mr(self, ibv_access_flags.IBV_ACCESS_REMOTE_WRITE |
144 ibv_access_flags.IBV_ACCESS_MW_BIND)
145
146
147 class QpExTestCase(RDMATestCase):
148 """ Run traffic using the new post send API. """
149 def setUp(self):
150 super().setUp()
151 self.iters = 100
152
153 def test_qp_ex_ud_send(self):
154 self.create_players(QpExUDSend)
155 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND)
156
157 def test_qp_ex_ud_zero_size(self):
158 self.create_players(QpExUDSend)
159 self.client.msg_size = 0
160 self.server.msg_size = 0
161 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND)
162
163 def test_qp_ex_rc_send(self):
164 self.create_players(QpExRCSend)
165 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND)
166
167 def test_qp_ex_xrc_send(self):
168 self.create_players(QpExXRCSend)
169 u.xrc_traffic(self.client, self.server, send_op=ibv_wr_opcode.IBV_WR_SEND)
170
171 def test_qp_ex_ud_send_imm(self):
172 self.create_players(QpExUDSendImm)
173 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND_WITH_IMM)
174
175 def test_qp_ex_rc_send_imm(self):
176 self.create_players(QpExRCSendImm)
177 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND_WITH_IMM)
178
179 def test_qp_ex_xrc_send_imm(self):
180 self.create_players(QpExXRCSendImm)
181 u.xrc_traffic(self.client, self.server, send_op=ibv_wr_opcode.IBV_WR_SEND_WITH_IMM)
182
183 def test_qp_ex_rc_flush(self):
184 self.create_players(QpExRCFlush)
185 wcs = u.flush_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_FLUSH)
186 if wcs[0].status != ibv_wc_status.IBV_WC_SUCCESS:
187 raise PyverbsError(f'Unexpected {wc_status_to_str(wcs[0].status)}')
188
189 self.client.level = ibv_selectivity_level.IBV_FLUSH_MR
190 wcs = u.flush_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_FLUSH)
191 if wcs[0].status != ibv_wc_status.IBV_WC_SUCCESS:
192 raise PyverbsError(f'Unexpected {wc_status_to_str(wcs[0].status)}')
193
194 def test_qp_ex_rc_flush_type_violate(self):
195 self.create_players(QpExRCFlush)
196 self.client.ptype = ibv_placement_type.IBV_FLUSH_PERSISTENT
197 wcs = u.flush_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_FLUSH)
198 if wcs[0].status != ibv_wc_status.IBV_WC_REM_ACCESS_ERR:
199 raise PyverbsError(f'Expected errors {wc_status_to_str(ibv_wc_status.IBV_WC_REM_ACCESS_ERR)} - got {wc_status_to_str(wcs[0].status)}')
200
201 def test_qp_ex_rc_atomic_write(self):
202 self.create_players(QpExRCAtomicWrite)
203 self.client.msg_size = 8
204 self.server.msg_size = 8
205 u.rdma_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_ATOMIC_WRITE)
206
207 def test_qp_ex_rc_rdma_write(self):
208 self.create_players(QpExRCRDMAWrite)
209 u.rdma_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
210
211 def test_qp_ex_rc_rdma_write_imm(self):
212 self.create_players(QpExRCRDMAWriteImm)
213 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM)
214
215 def test_qp_ex_rc_rdma_write_zero_size(self):
216 self.create_players(QpExRCRDMAWrite)
217 self.client.msg_size = 0
218 self.server.msg_size = 0
219 u.rdma_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
220
221 def test_qp_ex_rc_rdma_read(self):
222 self.create_players(QpExRCRDMARead)
223 self.server.mr.write('s' * self.server.msg_size, self.server.msg_size)
224 u.rdma_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_READ)
225
226 def test_qp_ex_rc_rdma_read_zero_size(self):
227 self.create_players(QpExRCRDMARead)
228 self.client.msg_size = 0
229 self.server.msg_size = 0
230 self.server.mr.write('s' * self.server.msg_size, self.server.msg_size)
231 u.rdma_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_READ)
232
233 def test_qp_ex_rc_atomic_cmp_swp(self):
234 self.create_players(QpExRCAtomicCmpSwp)
235 self.client.msg_size = 8 # Atomic work on 64b operators
236 self.server.msg_size = 8
237 self.server.mr.write('s' * 8, 8)
238 u.atomic_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_ATOMIC_CMP_AND_SWP)
239
240 def test_qp_ex_rc_atomic_fetch_add(self):
241 self.create_players(QpExRCAtomicFetchAdd)
242 self.client.msg_size = 8 # Atomic work on 64b operators
243 self.server.msg_size = 8
244 self.server.mr.write('s' * 8, 8)
245 u.atomic_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD)
246
247 def test_qp_ex_rc_bind_mw(self):
248 """
249 Verify bind memory window operation using the new post_send API.
250 Instead of checking through regular pingpong style traffic, we'll
251 do as follows:
252 - Register an MR with remote write access
253 - Bind a MW without remote write permission to the MR
254 - Verify that remote write fails
255 Since it's a unique flow, it's an integral part of that test rather
256 than a utility method.
257 """
258 self.create_players(QpExRCBindMw)
259 client_sge = u.get_send_elements(self.client, False)[1]
260 # Create a MW and bind it
261 self.server.qp.wr_start()
262 self.server.qp.wr_id = 0x123
263 self.server.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED
264 bind_info = MWBindInfo(self.server.mr, self.server.mr.buf, self.server.mr.length,
265 ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
266 try:
267 mw = MW(self.server.pd, mw_type=ibv_mw_type.IBV_MW_TYPE_2)
268 except PyverbsRDMAError as ex:
269 if ex.error_code == errno.EOPNOTSUPP:
270 raise unittest.SkipTest('Memory Window allocation is not supported')
271 raise ex
272 new_key = inc_rkey(mw.rkey)
273 self.server.qp.wr_bind_mw(mw, new_key, bind_info)
274 self.server.qp.wr_complete()
275 u.poll_cq(self.server.cq)
276 # Verify that remote write fails
277 self.client.qp.wr_start()
278 self.client.qp.wr_id = 0x124
279 self.client.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED
280 self.client.qp.wr_rdma_write(new_key, self.server.mr.buf)
281 self.client.qp.wr_set_sge(client_sge)
282 self.client.qp.wr_complete()
283 wcs = u._poll_cq(self.client.cq)
284 if wcs[0].status != ibv_wc_status.IBV_WC_REM_ACCESS_ERR:
285 raise PyverbsRDMAError(f'Completion status is {wc_status_to_str(wcs[0].status)}')
286
287 def test_post_receive_qp_state_bad_flow(self):
288 self.create_players(QpExUDSend)
289 u.post_rq_state_bad_flow(self)
290
291 def test_post_send_qp_state_bad_flow(self):
292 self.create_players(QpExUDSend)
293 u.post_sq_state_bad_flow(self)
294
295 def test_full_rq_bad_flow(self):
296 self.create_players(QpExUDSend)
297 u.full_rq_bad_flow(self)
298
299 def test_rq_with_larger_sgl_bad_flow(self):
300 self.create_players(QpExUDSend)
301 u.create_rq_with_larger_sgl_bad_flow(self)