Debian Codegraph @

tests/test_qp.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
3 # Copyright (c) 2020 Kamal Heib <[email protected]>, All rights reserved. See COPYING file
4 # Copyright 2020-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
5
6 """
7 Test module for pyverbs' qp module.
8 """
9 import unittest
10 import random
11 import errno
12 import os
13
14 from pyverbs.pyverbs_error import PyverbsRDMAError
15 from pyverbs.qp import QPAttr, QP
16 from tests.base import PyverbsAPITestCase, RDMATestCase, RCResources
17 import pyverbs.utils as pu
18 import pyverbs.device as d
19 from pyverbs.libibverbs_enums import ibv_qp_type, ibv_qp_state, ibv_qp_attr_mask, ibv_node_type, \
20 ibv_wr_opcode, ibv_query_qp_data_in_order_flags, ibv_query_qp_data_in_order_caps
21 from pyverbs.pd import PD
22 from pyverbs.cq import CQ
23 import tests.utils as u
24
25
26 class QPTest(PyverbsAPITestCase):
27 """
28 Test various functionalities of the QP class.
29 """
30
31 def create_qp(self, creator, qp_init_attr, is_ex, with_attr, port_num):
32 """
33 Auxiliary function to create QP object.
34 """
35 try:
36 qp_attr = (None, QPAttr(port_num=port_num))[with_attr]
37 return QP(creator, qp_init_attr, qp_attr)
38 except PyverbsRDMAError as ex:
39 if ex.error_code == errno.EOPNOTSUPP:
40 with_str = ('without', 'with')[with_attr] + ('', ' extended')[is_ex]
41 qp_type_str = pu.qp_type_to_str(qp_init_attr.qp_type)
42 raise unittest.SkipTest(f'Create {qp_type_str} QP {with_str} attrs is not supported')
43 raise ex
44
45 def create_qp_common_test(self, qp_type, qp_state, is_ex, with_attr, qp_attr_edit_callback=None):
46 """
47 Common function used by create QP tests.
48 """
49 with PD(self.ctx) as pd:
50 with CQ(self.ctx, 100, None, None, 0) as cq:
51 if qp_type == ibv_qp_type.IBV_QPT_RAW_PACKET:
52 if not (u.is_eth(self.ctx, self.ib_port) and u.has_cap_net_raw()):
53 raise unittest.SkipTest('To Create RAW QP must be done by user with '\
54 'CAP_NET_RAW on Ethernet link layer')
55
56 if is_ex:
57 qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, qp_type)
58 creator = self.ctx
59 else:
60 qia = u.get_qp_init_attr(cq, self.attr)
61 qia.qp_type = qp_type
62 creator = pd
63
64 if qp_attr_edit_callback:
65 qia = qp_attr_edit_callback(qia)
66
67 qp = self.create_qp(creator, qia, is_ex, with_attr, self.ib_port)
68
69 qp_type_str = pu.qp_type_to_str(qp_type)
70 qp_state_str = pu.qp_state_to_str(qp_state)
71 assert qp.qp_state == qp_state , f'{qp_type_str} QP should have been in {qp_state_str}'
72
73 def test_create_rc_qp_no_attr(self):
74 """
75 Test RC QP creation via ibv_create_qp without a QPAttr object provided.
76 """
77 self.create_qp_common_test(ibv_qp_type.IBV_QPT_RC, ibv_qp_state.IBV_QPS_RESET, False, False)
78
79 def test_create_uc_qp_no_attr(self):
80 """
81 Test UC QP creation via ibv_create_qp without a QPAttr object provided.
82 """
83 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UC, ibv_qp_state.IBV_QPS_RESET, False, False)
84
85 def test_create_ud_qp_no_attr(self):
86 """
87 Test UD QP creation via ibv_create_qp without a QPAttr object provided.
88 """
89 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UD, ibv_qp_state.IBV_QPS_RESET, False, False)
90
91 def test_create_raw_qp_no_attr(self):
92 """
93 Test RAW Packet QP creation via ibv_create_qp without a QPAttr object
94 provided.
95 Raw Packet is skipped for non-root users / Infiniband link layer.
96 """
97 self.create_qp_common_test(ibv_qp_type.IBV_QPT_RAW_PACKET, ibv_qp_state.IBV_QPS_RESET, False, False)
98
99 def test_create_rc_qp_with_attr(self):
100 """
101 Test RC QP creation via ibv_create_qp with a QPAttr object provided.
102 """
103 self.create_qp_common_test(ibv_qp_type.IBV_QPT_RC, ibv_qp_state.IBV_QPS_INIT, False, True)
104
105 def test_create_uc_qp_with_attr(self):
106 """
107 Test UC QP creation via ibv_create_qp with a QPAttr object provided.
108 """
109 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UC, ibv_qp_state.IBV_QPS_INIT, False, True)
110
111 def test_create_ud_qp_with_attr(self):
112 """
113 Test UD QP creation via ibv_create_qp with a QPAttr object provided.
114 """
115 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UD, ibv_qp_state.IBV_QPS_RTS, False, True)
116
117 def test_create_raw_qp_with_attr(self):
118 """
119 Test RAW Packet QP creation via ibv_create_qp with a QPAttr object
120 provided.
121 Raw Packet is skipped for non-root users / Infiniband link layer.
122 """
123 self.create_qp_common_test(ibv_qp_type.IBV_QPT_RAW_PACKET, ibv_qp_state.IBV_QPS_RTS, False, True)
124
125 def test_create_rc_qp_ex_no_attr(self):
126 """
127 Test RC QP creation via ibv_create_qp_ex without a QPAttr object
128 provided.
129 """
130 self.create_qp_common_test(ibv_qp_type.IBV_QPT_RC, ibv_qp_state.IBV_QPS_RESET, True, False)
131
132 def test_create_uc_qp_ex_no_attr(self):
133 """
134 Test UC QP creation via ibv_create_qp_ex without a QPAttr object
135 provided.
136 """
137 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UC, ibv_qp_state.IBV_QPS_RESET, True, False)
138
139 def test_create_ud_qp_ex_no_attr(self):
140 """
141 Test UD QP creation via ibv_create_qp_ex without a QPAttr object
142 provided.
143 """
144 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UD, ibv_qp_state.IBV_QPS_RESET, True, False)
145
146 def test_create_raw_qp_ex_no_attr(self):
147 """
148 Test Raw Packet QP creation via ibv_create_qp_ex without a QPAttr object
149 provided.
150 Raw Packet is skipped for non-root users / Infiniband link layer.
151 """
152 self.create_qp_common_test(ibv_qp_type.IBV_QPT_RAW_PACKET, ibv_qp_state.IBV_QPS_RESET, True, False)
153
154 def test_create_rc_qp_ex_with_attr(self):
155 """
156 Test RC QP creation via ibv_create_qp_ex with a QPAttr object provided.
157 """
158 self.create_qp_common_test(ibv_qp_type.IBV_QPT_RC, ibv_qp_state.IBV_QPS_INIT, True, True)
159
160 def test_create_uc_qp_ex_with_attr(self):
161 """
162 Test UC QP creation via ibv_create_qp_ex with a QPAttr object provided.
163 """
164 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UC, ibv_qp_state.IBV_QPS_INIT, True, True)
165
166 def test_create_ud_qp_ex_with_attr(self):
167 """
168 Test UD QP creation via ibv_create_qp_ex with a QPAttr object provided.
169 """
170 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UD, ibv_qp_state.IBV_QPS_RTS, True, True)
171
172 def test_create_raw_qp_ex_with_attr(self):
173 """
174 Test Raw Packet QP creation via ibv_create_qp_ex with a QPAttr object
175 provided.
176 Raw Packet is skipped for non-root users / Infiniband link layer.
177 """
178 self.create_qp_common_test(ibv_qp_type.IBV_QPT_RAW_PACKET, ibv_qp_state.IBV_QPS_RTS, True, True)
179
180 def qp_attr_edit_max_send_wr_callback(self, qp_init_attr):
181 qp_init_attr.max_send_wr = 0xffffffff # max_uint32
182 return qp_init_attr
183
184 def qp_attr_edit_max_send_sge_callback(self, qp_init_attr):
185 qp_init_attr.max_send_sge = 0xffff # max_uint16
186 return qp_init_attr
187
188 def qp_attr_edit_max_recv_sge_callback(self, qp_init_attr):
189 qp_init_attr.max_recv_sge = 0xffff # max_uint16
190 return qp_init_attr
191
192 def qp_attr_edit_max_recv_wr_callback(self, qp_init_attr):
193 qp_init_attr.max_recv_wr = 0xffffffff # max_uint32
194 return qp_init_attr
195
196 def test_create_raw_qp_ex_with_illegal_caps_max_send_wr(self):
197 """
198 Test Raw Packet QP creation via ibv_create_qp_ex with a QPAttr object with illegal max_send_wr.
199 """
200 dev_attr = self.ctx.query_device()
201 if dev_attr.max_qp_wr < 0xffffffff:
202 with self.assertRaises(PyverbsRDMAError) as ex:
203 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UD, ibv_qp_state.IBV_QPS_RTS, False, True,
204 qp_attr_edit_callback=self.qp_attr_edit_max_send_wr_callback)
205 self.assertNotEqual(ex.exception.error_code, 0)
206
207 def test_create_raw_qp_ex_with_illegal_caps_max_send_sge(self):
208 """
209 Test Raw Packet QP creation via ibv_create_qp_ex with a QPAttr object with illegal max_send_sge.
210 """
211 dev_attr = self.ctx.query_device()
212 if dev_attr.max_sge < 0xffff:
213 with self.assertRaises(PyverbsRDMAError) as ex:
214 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UD, ibv_qp_state.IBV_QPS_RTS, False, True,
215 qp_attr_edit_callback=self.qp_attr_edit_max_send_sge_callback)
216 self.assertNotEqual(ex.exception.error_code, 0)
217
218 def test_create_raw_qp_ex_with_illegal_caps_max_recv_sge(self):
219 """
220 Test Raw Packet QP creation via ibv_create_qp_ex with a QPAttr object with illegal max_recv_sge.
221 """
222 dev_attr = self.ctx.query_device()
223 if dev_attr.max_sge < 0xffff:
224 with self.assertRaises(PyverbsRDMAError) as ex:
225 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UD, ibv_qp_state.IBV_QPS_RTS, False, True,
226 qp_attr_edit_callback=self.qp_attr_edit_max_recv_sge_callback)
227 self.assertNotEqual(ex.exception.error_code, 0)
228
229 def test_create_raw_qp_ex_with_illegal_caps_max_recv_wr(self):
230 """
231 Test Raw Packet QP creation via ibv_create_qp_ex with a QPAttr object with illegal max_recv_wr.
232 """
233 dev_attr = self.ctx.query_device()
234 if dev_attr.max_qp_wr < 0xffffffff:
235 with self.assertRaises(PyverbsRDMAError) as ex:
236 self.create_qp_common_test(ibv_qp_type.IBV_QPT_UD, ibv_qp_state.IBV_QPS_RTS, False, True,
237 qp_attr_edit_callback=self.qp_attr_edit_max_recv_wr_callback)
238 self.assertNotEqual(ex.exception.error_code, 0)
239
240 def verify_qp_attrs(self, orig_cap, state, init_attr, attr):
241 self.assertEqual(state, attr.qp_state)
242 self.assertLessEqual(orig_cap.max_send_wr, init_attr.cap.max_send_wr)
243 self.assertLessEqual(orig_cap.max_recv_wr, init_attr.cap.max_recv_wr)
244 self.assertLessEqual(orig_cap.max_send_sge, init_attr.cap.max_send_sge)
245 self.assertLessEqual(orig_cap.max_recv_sge, init_attr.cap.max_recv_sge)
246 self.assertLessEqual(orig_cap.max_inline_data, init_attr.cap.max_inline_data)
247
248 def get_node_type(self):
249 for dev in d.get_device_list():
250 if dev.name.decode() == self.ctx.name:
251 return dev.node_type
252
253 def query_qp_common_test(self, qp_type):
254 with PD(self.ctx) as pd:
255 with CQ(self.ctx, 100, None, None, 0) as cq:
256 if qp_type == ibv_qp_type.IBV_QPT_RAW_PACKET:
257 if not (u.is_eth(self.ctx, self.ib_port) and u.has_cap_net_raw()):
258 raise unittest.SkipTest('To Create RAW QP must be done by user with '\
259 'CAP_NET_RAW on Ethernet link layer')
260
261 # Legacy QP
262 qia = u.get_qp_init_attr(cq, self.attr)
263 qia.qp_type = qp_type
264 caps = qia.cap
265 qp = self.create_qp(pd, qia, False, False, self.ib_port)
266 qp_attr, qp_init_attr = qp.query(ibv_qp_attr_mask.IBV_QP_STATE | ibv_qp_attr_mask.IBV_QP_CAP)
267 if self.get_node_type() == ibv_node_type.IBV_NODE_RNIC:
268 self.verify_qp_attrs(caps, ibv_qp_state.IBV_QPS_INIT, qp_init_attr, qp_attr)
269 else:
270 self.verify_qp_attrs(caps, ibv_qp_state.IBV_QPS_RESET, qp_init_attr, qp_attr)
271
272 # Extended QP
273 qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, qp_type)
274 caps = qia.cap # Save them to verify values later
275 qp = self.create_qp(self.ctx, qia, True, False, self.ib_port)
276 qp_attr, qp_init_attr = qp.query(ibv_qp_attr_mask.IBV_QP_STATE | ibv_qp_attr_mask.IBV_QP_CAP)
277 if self.get_node_type() == ibv_node_type.IBV_NODE_RNIC:
278 self.verify_qp_attrs(caps, ibv_qp_state.IBV_QPS_INIT, qp_init_attr, qp_attr)
279 else:
280 self.verify_qp_attrs(caps, ibv_qp_state.IBV_QPS_RESET, qp_init_attr, qp_attr)
281
282 def test_query_rc_qp(self):
283 """
284 Queries an RC QP after creation. Verifies that its properties are as
285 expected.
286 """
287 self.query_qp_common_test(ibv_qp_type.IBV_QPT_RC)
288
289 def test_query_uc_qp(self):
290 """
291 Queries an UC QP after creation. Verifies that its properties are as
292 expected.
293 """
294 self.query_qp_common_test(ibv_qp_type.IBV_QPT_UC)
295
296 def test_query_ud_qp(self):
297 """
298 Queries an UD QP after creation. Verifies that its properties are as
299 expected.
300 """
301 self.query_qp_common_test(ibv_qp_type.IBV_QPT_UD)
302
303 def test_query_raw_qp(self):
304 """
305 Queries an RAW Packet QP after creation. Verifies that its properties
306 are as expected.
307 Raw Packet is skipped for non-root users / Infiniband link layer.
308 """
309 self.query_qp_common_test(ibv_qp_type.IBV_QPT_RAW_PACKET)
310
311 def test_query_data_in_order(self):
312 """
313 Queries an UD QP data in order after moving it to RTS state.
314 Verifies that the result from the query is valid.
315 """
316 with PD(self.ctx) as pd:
317 with CQ(self.ctx, 100, None, None, 0) as cq:
318 qia = u.get_qp_init_attr(cq, self.attr)
319 qia.qp_type = ibv_qp_type.IBV_QPT_UD
320 qp = self.create_qp(pd, qia, False, True, self.ib_port)
321 is_data_in_order = qp.query_data_in_order(ibv_wr_opcode.IBV_WR_SEND)
322 self.assertIn(is_data_in_order, [0, 1], 'Data in order result with flags=0 is not valid')
323 is_data_in_order = qp.query_data_in_order(ibv_wr_opcode.IBV_WR_SEND,
324 ibv_query_qp_data_in_order_flags.IBV_QUERY_QP_DATA_IN_ORDER_RETURN_CAPS)
325 valid_results = [0,
326 ibv_query_qp_data_in_order_caps.IBV_QUERY_QP_DATA_IN_ORDER_ALIGNED_128_BYTES,
327 ibv_query_qp_data_in_order_caps.IBV_QUERY_QP_DATA_IN_ORDER_WHOLE_MSG | \
328 ibv_query_qp_data_in_order_caps.IBV_QUERY_QP_DATA_IN_ORDER_ALIGNED_128_BYTES]
329 self.assertIn(is_data_in_order, valid_results, 'Data in order result with flags=1 is not valid')
330
331 @u.skip_unsupported
332 def test_modify_ud_qp(self):
333 """
334 Queries a UD QP after calling modify(). Verifies that its properties are
335 as expected.
336 """
337 with PD(self.ctx) as pd:
338 with CQ(self.ctx, 100, None, None, 0) as cq:
339 # Legacy QP
340 qia = u.get_qp_init_attr(cq, self.attr)
341 qia.qp_type = ibv_qp_type.IBV_QPT_UD
342 qp = self.create_qp(pd, qia, False, False, self.ib_port)
343 qa = QPAttr()
344 qa.qkey = 0x123
345 qp.to_init(qa)
346 qp_attr, _ = qp.query(ibv_qp_attr_mask.IBV_QP_QKEY)
347 assert qp_attr.qkey == qa.qkey, 'Legacy QP, QKey is not as expected'
348 qp.to_rtr(qa)
349 qa.sq_psn = 0x45
350 qp.to_rts(qa)
351 qp_attr, _ = qp.query(ibv_qp_attr_mask.IBV_QP_SQ_PSN)
352 assert qp_attr.sq_psn == qa.sq_psn, 'Legacy QP, SQ PSN is not as expected'
353 qa.qp_state = ibv_qp_state.IBV_QPS_RESET
354 qp.modify(qa, ibv_qp_attr_mask.IBV_QP_STATE)
355 assert qp.qp_state == ibv_qp_state.IBV_QPS_RESET, 'Legacy QP, QP state is not as expected'
356 # Extended QP
357 qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, ibv_qp_type.IBV_QPT_UD)
358 qp = self.create_qp(self.ctx, qia, True, False, self.ib_port)
359 qa = QPAttr()
360 qa.qkey = 0x123
361 qp.to_init(qa)
362 qp_attr, _ = qp.query(ibv_qp_attr_mask.IBV_QP_QKEY)
363 assert qp_attr.qkey == qa.qkey, 'Extended QP, QKey is not as expected'
364 qp.to_rtr(qa)
365 qa.sq_psn = 0x45
366 qp.to_rts(qa)
367 qp_attr, _ = qp.query(ibv_qp_attr_mask.IBV_QP_SQ_PSN)
368 assert qp_attr.sq_psn == qa.sq_psn, 'Extended QP, SQ PSN is not as expected'
369 qa.qp_state = ibv_qp_state.IBV_QPS_RESET
370 qp.modify(qa, ibv_qp_attr_mask.IBV_QP_STATE)
371 assert qp.qp_state == ibv_qp_state.IBV_QPS_RESET, 'Extended QP, QP state is not as expected'
372
373 @u.skip_unsupported
374 @u.requires_cap_net_raw()
375 def test_create_ud_qp_with_privileged_qkey(self):
376 """
377 Test UD QP creation and modification with privileged QKEY 0x80000000.
378 Verifies that the privileged QKEY (IB_QP_PRIVILEGED_Q_KEY) can be set
379 and queried correctly for both legacy and extended QPs.
380 """
381 privileged_qkey = 0x80000000 # IB_QP_PRIVILEGED_Q_KEY
382
383 with PD(self.ctx) as pd:
384 with CQ(self.ctx, 100, None, None, 0) as cq:
385 # Test Legacy QP with privileged QKEY
386 qia = u.get_qp_init_attr(cq, self.attr)
387 qia.qp_type = ibv_qp_type.IBV_QPT_UD
388 qp = self.create_qp(pd, qia, False, False, self.ib_port)
389 qa = QPAttr()
390 qa.qkey = privileged_qkey
391 qp.to_init(qa)
392 qp_attr, _ = qp.query(ibv_qp_attr_mask.IBV_QP_QKEY)
393 self.assertEqual(qp_attr.qkey, qa.qkey,
394 f'Legacy QP, Privileged QKey is not as '\
395 f'expected. Got: 0x{qp_attr.qkey:08x}, Expected: 0x{qa.qkey:08x}')
396
397 # Test Extended QP with privileged QKEY
398 qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, ibv_qp_type.IBV_QPT_UD)
399 qp = self.create_qp(self.ctx, qia, True, False, self.ib_port)
400 qa = QPAttr()
401 qa.qkey = privileged_qkey
402 qp.to_init(qa)
403 qp_attr, _ = qp.query(ibv_qp_attr_mask.IBV_QP_QKEY)
404 self.assertEqual(qp_attr.qkey, qa.qkey,
405 f'Extended QP, Privileged QKey is not as '\
406 f'expected. Got: 0x{qp_attr.qkey:08x}, Expected: 0x{qa.qkey:08x}')
407
408 class RCQPTest(RDMATestCase):
409 """
410 Test various functionalities of the RC QP class.
411 """
412 def test_modify_rc_qp_rd_atomic(self):
413 """
414 This test verifies that the values of rd_atomic fields are
415 at least the requested value.
416 """
417 self.max_rd_atomic = 12
418 self.max_dest_rd_atomic = 12
419
420 self.create_players(RCResources, max_rd_atomic=self.max_rd_atomic,
421 max_dest_rd_atomic=self.max_dest_rd_atomic)
422
423 qp_attr, _ = self.server.qp.query(ibv_qp_attr_mask.IBV_QP_MAX_QP_RD_ATOMIC | \
424 ibv_qp_attr_mask.IBV_QP_MAX_DEST_RD_ATOMIC)
425
426 self.assertGreaterEqual(qp_attr.max_rd_atomic, self.max_rd_atomic,
427 'Max RD Atomic value is less than requested.')
428 self.assertGreaterEqual(qp_attr.max_dest_rd_atomic, self.max_dest_rd_atomic,
429 'Max Dest RD Atomic is less than requested.')
430
431
432 def get_qp_init_attr_ex(cq, pd, attr, attr_ex, qpt):
433 """
434 Creates a QPInitAttrEx object with a QP type of the provided <qpts> array
435 and other random values.
436 :param cq: CQ to be used as send and receive CQ
437 :param pd: A PD object to use
438 :param attr: Device attributes for capability checks
439 :param attr_ex: Extended device attributes for capability checks
440 :param qpt: QP type
441 :return: An initialized QPInitAttrEx object
442 """
443 qia = u.random_qp_init_attr_ex(attr_ex, attr, qpt)
444 qia.send_cq = cq
445 qia.recv_cq = cq
446 qia.pd = pd # Only XRCD can be created without a PD
447 return qia