Debian Codegraph @

tests/mlx5_base.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2020 NVIDIA Corporation . All rights reserved. See COPYING file
3
4 import unittest
5 import resource
6 import random
7 import struct
8 import errno
9 import math
10 import time
11 import sys
12
13 from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \
14 Mlx5DVQPInitAttr, Mlx5QP, Mlx5DVDCInitAttr, Mlx5DCIStreamInitAttr, \
15 Mlx5DevxObj, Mlx5UMEM, Mlx5UAR, WqeDataSeg, WqeCtrlSeg, Wqe, Mlx5Cqe64, \
16 Mlx5DVCQInitAttr, Mlx5CQ
17 from tests.base import RoCETrafficResources, set_rnr_attributes, DCT_KEY, \
18 RDMATestCase, PyverbsAPITestCase, RDMACMBaseTest, BaseResources, PATH_MTU, \
19 RNR_RETRY, RETRY_CNT, MIN_RNR_TIMER, TIMEOUT, MAX_RDMA_ATOMIC, RCResources, \
20 is_gid_available
21 from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError, \
22 PyverbsError
23 from pyverbs.providers.mlx5.mlx5dv_objects import Mlx5DvObj
24 from pyverbs.qp import QPCap, QPInitAttrEx, QPAttr
25 from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_qp_init_attr_mask, mlx5dv_dc_type, mlx5dv_qp_init_attr_mask, \
26 mlx5dv_context_attr_flags, mlx5dv_obj_type, mlx5dv_context_attr_comp_mask, MLX5_SEND_WQE_BB, MLX5_SEND_WQE_SHIFT, \
27 MLX5_WQE_CTRL_CQ_UPDATE, MLX5_OPCODE_SEND_IMM, MLX5_SND_DBR, MLX5_RCV_DBR, MLX5_CQE_INVALID, MLX5_CQE_RESP_ERR, \
28 MLX5_CQE_REQ_ERR, MLX5_CQE_REQ, MLX5_CQE_RESP_SEND_IMM, _MLX5DV_UAR_ALLOC_TYPE_NC
29 from pyverbs.addr import AHAttr, GlobalRoute
30 from pyverbs.cq import CqInitAttrEx
31 from pyverbs.device import FdArr
32 import pyverbs.mem_alloc as mem
33 import pyverbs.dma_util as dma
34 import pyverbs.device as d
35 from pyverbs.pd import PD
36 from pyverbs.libibverbs_enums import ibv_qp_init_attr_mask, ibv_qp_type, ibv_qp_state, \
37 ibv_access_flags, ibv_qp_attr_mask, ibv_wc_status, ibv_rereg_mr_flags, ibv_wr_opcode, \
38 ibv_gid_type_sysfs, ibv_qp_create_send_ops_flags
39 from pyverbs.mr import MR
40 import tests.utils
41
42 MLX5_CQ_SET_CI = 0
43 POLL_CQ_TIMEOUT = 5 # In seconds
44 PORT_STATE_TIMEOUT = 20 # In seconds
45
46 MELLANOX_VENDOR_ID = 0x02c9
47 MLX5_DEVS = {
48 0x1011, # MT4113 Connect-IB
49 0x1012, # Connect-IB Virtual Function
50 0x1013, # ConnectX-4
51 0x1014, # ConnectX-4 Virtual Function
52 0x1015, # ConnectX-4LX
53 0x1016, # ConnectX-4LX Virtual Function
54 0x1017, # ConnectX-5, PCIe 3.0
55 0x1018, # ConnectX-5 Virtual Function
56 0x1019, # ConnectX-5 Ex
57 0x101a, # ConnectX-5 Ex VF
58 0x101b, # ConnectX-6
59 0x101c, # ConnectX-6 VF
60 0x101d, # ConnectX-6 DX
61 0x101e, # ConnectX family mlx5Gen Virtual Function
62 0x101f, # ConnectX-6 LX
63 0x1021, # ConnectX-7
64 0x1023, # ConnectX-8
65 0x1025, # ConnectX-9
66 0x1027, # ConnectX-10
67 0xa2d2, # BlueField integrated ConnectX-5 network controller
68 0xa2d3, # BlueField integrated ConnectX-5 network controller VF
69 0xa2d6, # BlueField-2 integrated ConnectX-6 Dx network controller
70 0xa2dc, # BlueField-3 integrated ConnectX-7 network controller
71 0xa2df, # BlueField-4 integrated ConnectX-8 network controller
72 }
73
74 IB_SMP_ATTR_PORT_INFO = 0x0015
75 IB_MGMT_CLASS_SUBN_LID_ROUTED = 0x01
76 IB_MGMT_METHOD_GET = 0x01
77 DB_BF_DBR_LESS_BUF_OFFSET = 0x600
78
79
80 class PortStatus:
81 MLX5_PORT_UP = 1
82 MLX5_PORT_DOWN = 2
83
84
85 class PortState:
86 NO_STATE_CHANGE = 0
87 DOWN = 1
88 INIT = 2
89 ARMED = 3
90 ACTIVE = 4
91
92
93 def is_mlx5_dev(ctx):
94 dev_attrs = ctx.query_device()
95 return dev_attrs.vendor_id == MELLANOX_VENDOR_ID and \
96 dev_attrs.vendor_part_id in MLX5_DEVS
97
98
99 def skip_if_not_mlx5_dev(ctx):
100 if not is_mlx5_dev(ctx):
101 raise unittest.SkipTest('Can not run the test over non MLX5 device')
102
103
104 class Mlx5PyverbsAPITestCase(PyverbsAPITestCase):
105 def setUp(self):
106 super().setUp()
107 skip_if_not_mlx5_dev(self.ctx)
108
109
110 class Mlx5RDMATestCase(RDMATestCase):
111 def setUp(self):
112 super().setUp()
113 skip_if_not_mlx5_dev(d.Context(name=self.dev_name))
114
115
116 class Mlx5RDMACMBaseTest(RDMACMBaseTest):
117 def setUp(self):
118 super().setUp()
119 skip_if_not_mlx5_dev(d.Context(name=self.dev_name))
120
121
122 class Mlx5DcResources(RoCETrafficResources):
123 def __init__(self, dev_name, ib_port, gid_index, send_ops_flags,
124 qp_count=1, create_flags=0, mr_access=None):
125 self.send_ops_flags = send_ops_flags
126 self.create_flags = create_flags
127 if mr_access is None:
128 self.mr_access = (ibv_access_flags.IBV_ACCESS_REMOTE_WRITE |
129 ibv_access_flags.IBV_ACCESS_LOCAL_WRITE |
130 ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC |
131 ibv_access_flags.IBV_ACCESS_REMOTE_READ)
132 else:
133 self.mr_access = mr_access
134 super().__init__(dev_name, ib_port, gid_index, with_srq=True,
135 qp_count=qp_count)
136
137 def to_rts(self):
138 attr = self.create_qp_attr()
139 for i in range(self.qp_count):
140 self.qps[i].to_rts(attr)
141 self.dct_qp.to_rtr(attr)
142
143 def create_context(self):
144 mlx5dv_attr = Mlx5DVContextAttr()
145 try:
146 self.ctx = Mlx5Context(mlx5dv_attr, name=self.dev_name)
147 except PyverbsUserError as ex:
148 raise unittest.SkipTest(f'Could not open mlx5 context ({ex})')
149 except PyverbsRDMAError:
150 raise unittest.SkipTest('Opening mlx5 context is not supported')
151
152 def create_mr(self):
153 self.mr = MR(self.pd, self.msg_size, self.mr_access)
154
155 def create_qp_cap(self):
156 return QPCap(100, 0, 1, 0)
157
158 def create_qp_attr(self):
159 qp_attr = QPAttr(port_num=self.ib_port)
160 set_rnr_attributes(qp_attr)
161 qp_access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \
162 ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC | ibv_access_flags.IBV_ACCESS_REMOTE_READ
163 qp_attr.qp_access_flags = qp_access
164 gr = GlobalRoute(dgid=self.ctx.query_gid(self.ib_port, self.gid_index),
165 sgid_index=self.gid_index)
166 ah_attr = AHAttr(port_num=self.ib_port, is_global=1, gr=gr,
167 dlid=self.port_attr.lid)
168 qp_attr.ah_attr = ah_attr
169 return qp_attr
170
171 def create_qp_init_attr(self, send_ops_flags=0):
172 comp_mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD
173 if send_ops_flags:
174 comp_mask |= ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS
175 return QPInitAttrEx(cap=self.create_qp_cap(), pd=self.pd, scq=self.cq,
176 rcq=self.cq, srq=self.srq, qp_type=ibv_qp_type.IBV_QPT_DRIVER,
177 send_ops_flags=send_ops_flags, comp_mask=comp_mask,
178 sq_sig_all=1)
179
180 def create_qps(self):
181 # Create the DCI QPs.
182 qp_init_attr = self.create_qp_init_attr(self.send_ops_flags)
183 try:
184 for _ in range(self.qp_count):
185 comp_mask = mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_DC
186 if self.create_flags:
187 comp_mask |= mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS
188 attr = Mlx5DVQPInitAttr(comp_mask=comp_mask,
189 create_flags=self.create_flags,
190 dc_init_attr=Mlx5DVDCInitAttr())
191 qp = Mlx5QP(self.ctx, qp_init_attr, attr)
192 self.qps.append(qp)
193 self.qps_num.append(qp.qp_num)
194 self.psns.append(random.getrandbits(24))
195
196 # Create the DCT QP.
197 qp_init_attr = self.create_qp_init_attr()
198 dc_attr = Mlx5DVDCInitAttr(dc_type=mlx5dv_dc_type.MLX5DV_DCTYPE_DCT,
199 dct_access_key=DCT_KEY)
200 attr = Mlx5DVQPInitAttr(comp_mask=mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_DC,
201 dc_init_attr=dc_attr)
202 self.dct_qp = Mlx5QP(self.ctx, qp_init_attr, attr)
203 except PyverbsRDMAError as ex:
204 if ex.error_code == errno.EOPNOTSUPP:
205 raise unittest.SkipTest(f'Create DC QP is not supported')
206 raise ex
207
208
209 class Mlx5DcStreamsRes(Mlx5DcResources):
210 def __init__(self, dev_name, ib_port, gid_index, send_ops_flags,
211 qp_count=1, create_flags=0, mr_access=None):
212 super().__init__(dev_name, ib_port, gid_index, send_ops_flags,
213 qp_count, create_flags, mr_access)
214
215 def reset_qp(self, qp_idx):
216 qp_attr = QPAttr(qp_state=ibv_qp_state.IBV_QPS_RESET)
217 self.qps[qp_idx].modify(qp_attr, ibv_qp_attr_mask.IBV_QP_STATE)
218 self.qps[qp_idx].to_rts(qp_attr)
219
220 def get_stream_id(self, qp_idx):
221 return self.current_qp_stream_id[qp_idx]
222
223 def generate_stream_id(self, qp_idx):
224 self.current_qp_stream_id[qp_idx] += 1
225 # Reset stream id to check double-usage
226 if self.current_qp_stream_id[qp_idx] > self.dcis[qp_idx]['stream']+2:
227 self.current_qp_stream_id[qp_idx] = 1
228 return self.current_qp_stream_id[qp_idx]
229
230 def dci_reset_stream_id(self, qp_idx):
231 stream_id = self.get_stream_id(qp_idx)
232 Mlx5QP.modify_dci_stream_channel_id(self.qps[qp_idx], stream_id)
233
234 def generate_dci_attr(self, qpn):
235 # This array contains current number of log_dci_streams
236 # and log_dci_errored values per qp. For 1-st qp number
237 # of streams greater than number of errored and vice-versa
238 # for the 2nd qp.
239 qp_arr = {0: [3, 2], 1: [2, 3]}
240 try:
241 dci_caps = self.ctx.query_mlx5_device().dci_streams_caps
242 except PyverbsRDMAError as ex:
243 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
244 raise unittest.SkipTest('Get DCI caps is not supported')
245 raise ex
246 if not dci_caps or dci_caps['max_log_num_concurent'] == 0:
247 raise unittest.SkipTest('DCI caps is not supported by HW')
248 self.log_dci_streams = min(qp_arr.get(qpn, [1,1])[0],
249 dci_caps['max_log_num_concurent'])
250 self.log_dci_errored = min(qp_arr.get(qpn, [1,1])[1],
251 dci_caps['max_log_num_errored'])
252
253 def create_qps(self):
254 # Create the DCI QPs.
255 qp_init_attr = self.create_qp_init_attr(self.send_ops_flags)
256 self.dcis = {}
257 # This array contains current stream id
258 self.current_qp_stream_id = {}
259 comp_mask = mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_DC | \
260 mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_DCI_STREAMS
261 try:
262 for qpn in range(self.qp_count):
263 if self.create_flags:
264 comp_mask |= mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS
265 self.generate_dci_attr(qpn)
266 stream_ctx = Mlx5DCIStreamInitAttr(self.log_dci_streams,
267 self.log_dci_errored)
268 self.dcis[qpn] = {'stream': 1 << self.log_dci_streams,
269 'errored': 1 << self.log_dci_errored}
270 attr = Mlx5DVQPInitAttr(comp_mask=comp_mask,
271 create_flags=self.create_flags,
272 dc_init_attr=Mlx5DVDCInitAttr(dci_streams=stream_ctx))
273 qp = Mlx5QP(self.ctx, qp_init_attr, attr)
274 self.qps.append(qp)
275 # Different values for start point of stream id per qp
276 self.current_qp_stream_id[qpn] = qpn
277 self.qps_num.append(qp.qp_num)
278 self.psns.append(random.getrandbits(24))
279 # Create the DCT QP.
280 qp_init_attr = self.create_qp_init_attr()
281 dc_attr = Mlx5DVDCInitAttr(dc_type=mlx5dv_dc_type.MLX5DV_DCTYPE_DCT,
282 dct_access_key=DCT_KEY)
283 attr = Mlx5DVQPInitAttr(comp_mask=mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_DC,
284 dc_init_attr=dc_attr)
285 self.dct_qp = Mlx5QP(self.ctx, qp_init_attr, attr)
286 except PyverbsRDMAError as ex:
287 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
288 raise unittest.SkipTest('Create DC QP is not supported')
289 raise ex
290
291
292 class WqAttrs:
293 def __init__(self):
294 super().__init__()
295 self.wqe_num = 0
296 self.wqe_size = 0
297 self.wq_size = 0
298 self.head = 0
299 self.post_idx = 0
300 self.wqe_shift = 0
301 self.offset = 0
302
303 def __str__(self):
304 return str(vars(self))
305
306 def __format__(self, format_spec):
307 return str(self).__format__(format_spec)
308
309
310 class CqAttrs:
311 def __init__(self):
312 super().__init__()
313 self.cons_idx = 0
314 self.cqe_size = 64
315 self.ncqes = 256
316
317 def __str__(self):
318 return str(vars(self))
319
320 def __format__(self, format_spec):
321 return str(self).__format__(format_spec)
322
323
324 class QueueAttrs:
325 def __init__(self):
326 self.rq = WqAttrs()
327 self.sq = WqAttrs()
328 self.cq = CqAttrs()
329
330 def __str__(self):
331 print_format = '{}:\n\t{}\n'
332 return print_format.format('RQ Attributes', self.rq) + \
333 print_format.format('SQ Attributes', self.sq) + \
334 print_format.format('CQ Attributes', self.cq)
335
336
337 class Mlx5DevxRcResources(BaseResources):
338 """
339 Creates all the DevX resources needed for a traffic-ready RC DevX QP,
340 including methods to transit the WQs into RTS state.
341 It also includes traffic methods for post send/receive and poll.
342 The class currently supports post send with immediate, but can be
343 easily extended to support other opcodes in the future.
344 """
345 def __init__(self, dev_name, ib_port, gid_index, msg_size=1024, activate_port_state=False,
346 send_dbr_mode=0):
347 from tests.mlx5_prm_structs import SendDbrMode
348 super().__init__(dev_name, ib_port, gid_index)
349 self.umems = {}
350 self.send_dbr_mode = send_dbr_mode
351 self.msg_size = msg_size
352 self.num_msgs = 1000
353 self.imm = 0x03020100
354 self.uar = {}
355 self.max_recv_sge = 1
356 self.eqn = None
357 self.pd = None
358 self.dv_pd = None
359 self.mr = None
360 self.msi_vector = None
361 self.eq = None
362 self.cq = None
363 self.qp = None
364 self.qpn = None
365 self.psn = None
366 self.lid = None
367 self.gid = [0, 0, 0, 0]
368 # Remote attrs
369 self.rqpn = None
370 self.rpsn = None
371 self.rlid = None
372 self.rgid = [0, 0, 0, 0]
373 self.rmac = None
374 self.devx_objs = []
375 self.qattr = QueueAttrs()
376 self.with_odp = False
377 self.user_addr = None
378 if activate_port_state:
379 start_state_t = time.perf_counter()
380 self.change_port_state_with_registers(PortStatus.MLX5_PORT_UP)
381 admin_status, oper_status = self.query_port_state_with_registers()
382 while admin_status != PortStatus.MLX5_PORT_UP or oper_status != PortStatus.MLX5_PORT_UP:
383 if time.perf_counter() - start_state_t >= PORT_STATE_TIMEOUT:
384 raise PyverbsRDMAError('Could not change the port state to UP')
385 self.change_port_state_with_registers(PortStatus.MLX5_PORT_UP)
386 admin_status, oper_status = self.query_port_state_with_registers()
387 time.sleep(1)
388
389 mad_port_state = self.query_port_state_with_mads(ib_port)
390 while mad_port_state < PortState.ACTIVE:
391 if time.perf_counter() - start_state_t >= PORT_STATE_TIMEOUT:
392 raise PyverbsRDMAError('Could not change the port state to UP')
393 time.sleep(1)
394 mad_port_state = self.query_port_state_with_mads(ib_port)
395 if self.send_dbr_mode != SendDbrMode.DBR_VALID:
396 self.check_cap_send_dbr_mode()
397 self.init_resources()
398
399 def get_wqe_data_segment(self):
400 return WqeDataSeg(self.mr.length, self.mr.lkey, self.mr.buf)
401
402 def change_port_state_with_registers(self, state):
403 from tests.mlx5_prm_structs import PaosReg
404 paos_in = PaosReg(local_port=self.ib_port, admin_status=state, ase=1)
405 self.access_paos_register(paos_in)
406
407 def query_port_state_with_registers(self):
408 from tests.mlx5_prm_structs import PaosReg
409 paos_in = PaosReg(local_port=self.ib_port)
410 paos_out = self.access_paos_register(paos_in)
411 return paos_out.admin_status, paos_out.oper_status
412
413 def access_paos_register(self, paos_in, op_mod=0): # op_mod: 0 - write / 1 - read
414 from tests.mlx5_prm_structs import AccessPaosRegisterIn, \
415 AccessPaosRegisterOut, DevxOps
416 paos_reg_in = AccessPaosRegisterIn(op_mod=op_mod,
417 register_id=DevxOps.MLX5_CMD_OP_ACCESS_REGISTER_PAOS,
418 data=paos_in)
419 cmd_out = self.ctx.devx_general_cmd(paos_reg_in, len(AccessPaosRegisterOut()))
420 paos_reg_out = AccessPaosRegisterOut(cmd_out)
421 if paos_reg_out.status:
422 raise PyverbsRDMAError(f'Failed to access PAOS register ({paos_reg_out.syndrome})')
423 return paos_reg_out.data
424
425 def query_port_state_with_mads(self, ib_port):
426 from tests.mlx5_prm_structs import IbSmp
427 in_mad = IbSmp(base_version=1, mgmt_class=IB_MGMT_CLASS_SUBN_LID_ROUTED,
428 class_version=1, method=IB_MGMT_METHOD_GET,
429 attr_id=IB_SMP_ATTR_PORT_INFO, attr_mod=ib_port)
430 ib_smp_out = IbSmp(self._send_mad_cmd(ib_port, in_mad, 0x3))
431 return ib_smp_out.data[32] & 0xf
432
433 def _send_mad_cmd(self, ib_port, in_mad, op_mod):
434 from tests.mlx5_prm_structs import MadIfcIn, MadIfcOut
435 mad_ifc_in = MadIfcIn(op_mod=op_mod, port=ib_port, mad=in_mad)
436 cmd_out = self.ctx.devx_general_cmd(mad_ifc_in, len(MadIfcOut()))
437 mad_ifc_out = MadIfcOut(cmd_out)
438 if mad_ifc_out.status:
439 raise PyverbsRDMAError(f'Failed to send MAD with syndrome ({mad_ifc_out.syndrome})')
440 return mad_ifc_out.mad
441
442 def check_cap_send_dbr_mode(self):
443 """
444 Check the capability of the dbr less.
445 If the HCA cap have HCA cap 2, check if in HCA cap2 0x20(HCA CAP 2) + 0x1(current)
446 have the send_dbr_mode_no_dbr_ext.
447 """
448 from tests.mlx5_prm_structs import QueryCmdHcaCap2Out, \
449 QueryHcaCapIn, QueryCmdHcaCapOut, QueryHcaCapOp, QueryHcaCapMod, SendDbrMode
450 self.create_context()
451 query_cap_in = QueryHcaCapIn(op_mod=0x1)
452 query_cap_out = QueryCmdHcaCapOut(self.ctx.devx_general_cmd(
453 query_cap_in, len(QueryCmdHcaCapOut())))
454 if query_cap_out.status:
455 raise PyverbsRDMAError('Failed to query general HCA CAPs with syndrome '
456 f'({query_cap_out.syndrome}')
457
458 if not query_cap_out.capability.hca_cap_2:
459 raise unittest.SkipTest("The device doesn't support general HCA CAPs 2")
460 query_cap2_in = QueryHcaCapIn(op_mod=(QueryHcaCapOp.HCA_CAP_2 << 0x1) | \
461 QueryHcaCapMod.CURRENT)
462 query_cap2_out = QueryCmdHcaCap2Out(self.ctx.devx_general_cmd(
463 query_cap2_in, len(QueryCmdHcaCap2Out())))
464 if self.send_dbr_mode == SendDbrMode.NO_DBR_EXT and \
465 not query_cap2_out.capability.send_dbr_mode_no_dbr_ext:
466 raise unittest.SkipTest("The device doesn't support send_dbr_mode_no_dbr_ext cap")
467
468 if self.send_dbr_mode == SendDbrMode.NO_DBR_INT and \
469 not query_cap2_out.capability.send_dbr_mode_no_dbr_int:
470 raise unittest.SkipTest("The device doesn't support send_dbr_mode_no_dbr_int cap")
471
472 def init_resources(self):
473 if not self.is_eth():
474 self.query_lid()
475 else:
476 is_gid_available(self.gid_index)
477 self.query_gid()
478 self.create_pd()
479 self.create_mr()
480 self.query_eqn()
481 self.create_uar()
482 self.create_queue_attrs()
483 self.create_eq()
484 self.create_cq()
485 self.create_qp()
486 # Objects closure order is important, and must be done manually in DevX
487 self.devx_objs = [self.qp, self.cq] + list(self.uar.values()) + list(self.umems.values()) + [self.msi_vector, self.eq]
488
489
490 def query_lid(self):
491 from tests.mlx5_prm_structs import QueryHcaVportContextIn, \
492 QueryHcaVportContextOut, QueryHcaCapIn, QueryCmdHcaCapOut
493
494 query_cap_in = QueryHcaCapIn(op_mod=0x1)
495 query_cap_out = QueryCmdHcaCapOut(self.ctx.devx_general_cmd(
496 query_cap_in, len(QueryCmdHcaCapOut())))
497 if query_cap_out.status:
498 raise PyverbsRDMAError('Failed to query general HCA CAPs with syndrome '
499 f'({query_cap_out.syndrome}')
500 port_num = self.ib_port if query_cap_out.capability.num_ports >= 2 else 0
501 query_port_in = QueryHcaVportContextIn(port_num=port_num)
502 query_port_out = QueryHcaVportContextOut(self.ctx.devx_general_cmd(
503 query_port_in, len(QueryHcaVportContextOut())))
504 if query_port_out.status:
505 raise PyverbsRDMAError('Failed to query vport with syndrome '
506 f'({query_port_out.syndrome})')
507 self.lid = query_port_out.hca_vport_context.lid
508
509 def query_gid(self):
510 gid = self.ctx.query_gid(self.ib_port, self.gid_index).gid.split(':')
511 for i in range(0, len(gid), 2):
512 self.gid[int(i/2)] = int(gid[i] + gid[i+1], 16)
513
514 def is_eth(self):
515 from tests.mlx5_prm_structs import QueryHcaCapIn, \
516 QueryCmdHcaCapOut
517
518 query_cap_in = QueryHcaCapIn(op_mod=0x1)
519 query_cap_out = QueryCmdHcaCapOut(self.ctx.devx_general_cmd(
520 query_cap_in, len(QueryCmdHcaCapOut())))
521 if query_cap_out.status:
522 raise PyverbsRDMAError('Failed to query general HCA CAPs with syndrome '
523 f'({query_cap_out.syndrome})')
524 return query_cap_out.capability.port_type # 0:IB, 1:ETH
525
526 @staticmethod
527 def roundup_pow_of_two(val):
528 return pow(2, math.ceil(math.log2(val)))
529
530 def create_queue_attrs(self):
531 # RQ calculations
532 wqe_size = WqeDataSeg.sizeof() * self.max_recv_sge
533 self.qattr.rq.wqe_size = self.roundup_pow_of_two(wqe_size)
534 max_recv_wr = self.roundup_pow_of_two(self.num_msgs)
535 self.qattr.rq.wq_size = max(self.qattr.rq.wqe_size * max_recv_wr,
536 MLX5_SEND_WQE_BB)
537 self.qattr.rq.wqe_num = math.ceil(self.qattr.rq.wq_size / self.qattr.rq.wqe_size)
538 self.qattr.rq.wqe_shift = int(math.log2(self.qattr.rq.wqe_size - 1)) + 1
539
540 # SQ calculations
541 self.qattr.sq.offset = self.qattr.rq.wq_size
542 # 192 = max overhead size of all structs needed for all operations in RC
543 wqe_size = 192 + WqeDataSeg.sizeof()
544 # Align wqe size to MLX5_SEND_WQE_BB
545 self.qattr.sq.wqe_size = (wqe_size + MLX5_SEND_WQE_BB - 1) & ~(MLX5_SEND_WQE_BB - 1)
546 self.qattr.sq.wq_size = self.roundup_pow_of_two(self.qattr.sq.wqe_size * self.num_msgs)
547 self.qattr.sq.wqe_num = math.ceil(self.qattr.sq.wq_size / MLX5_SEND_WQE_BB)
548 self.qattr.sq.wqe_shift = int(math.log2(MLX5_SEND_WQE_BB))
549
550 def create_context(self):
551 try:
552 attr = Mlx5DVContextAttr(mlx5dv_context_attr_flags.MLX5DV_CONTEXT_FLAGS_DEVX)
553 self.ctx = Mlx5Context(attr, self.dev_name)
554 except PyverbsUserError as ex:
555 raise unittest.SkipTest(f'Could not open mlx5 context ({ex})')
556 except PyverbsRDMAError:
557 raise unittest.SkipTest('Opening mlx5 DevX context is not supported')
558
559 def create_pd(self):
560 self.pd = PD(self.ctx)
561 self.dv_pd = Mlx5DvObj(mlx5dv_obj_type.MLX5DV_OBJ_PD, pd=self.pd).dvpd
562
563 def create_mr(self):
564 access = ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | \
565 ibv_access_flags.IBV_ACCESS_REMOTE_READ
566 self.mr = MR(self.pd, self.msg_size, access)
567
568 def create_umem(self, size,
569 access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE,
570 alignment=resource.getpagesize()):
571 return Mlx5UMEM(self.ctx, size=size, alignment=alignment, access=access)
572
573 def create_uar(self):
574 self.uar['qp'] = Mlx5UAR(self.ctx, _MLX5DV_UAR_ALLOC_TYPE_NC)
575 self.uar['cq'] = Mlx5UAR(self.ctx, _MLX5DV_UAR_ALLOC_TYPE_NC)
576 if not self.uar['cq'].page_id or not self.uar['qp'].page_id:
577 raise PyverbsRDMAError('Failed to allocate UAR')
578
579 def query_eqn(self):
580 self.eqn = self.ctx.devx_query_eqn(0)
581
582 def create_cq(self):
583 from tests.mlx5_prm_structs import CreateCqIn, SwCqc, CreateCqOut
584
585 cq_size = self.roundup_pow_of_two(self.qattr.cq.cqe_size * self.qattr.cq.ncqes)
586 # Align to page size
587 pg_size = resource.getpagesize()
588 cq_size = (cq_size + pg_size - 1) & ~(pg_size - 1)
589 self.umems['cq'] = self.create_umem(size=cq_size)
590 self.umems['cq_dbr'] = self.create_umem(size=8, alignment=8)
591 log_cq_size = math.ceil(math.log2(self.qattr.cq.ncqes))
592 cmd_in = CreateCqIn(cq_umem_valid=1, cq_umem_id=self.umems['cq'].umem_id,
593 sw_cqc=SwCqc(c_eqn=self.eqn, uar_page=self.uar['cq'].page_id,
594 log_cq_size=log_cq_size, dbr_umem_valid=1,
595 dbr_umem_id=self.umems['cq_dbr'].umem_id))
596 self.cq = Mlx5DevxObj(self.ctx, cmd_in, len(CreateCqOut()))
597
598 def create_qp(self):
599 self.psn = random.getrandbits(24)
600 from tests.mlx5_prm_structs import SwQpc, CreateQpIn, DevxOps,\
601 CreateQpOut, CreateCqOut
602
603 self.psn = random.getrandbits(24)
604 qp_size = self.roundup_pow_of_two(self.qattr.rq.wq_size + self.qattr.sq.wq_size)
605 # Align to page size
606 pg_size = resource.getpagesize()
607 qp_size = (qp_size + pg_size - 1) & ~(pg_size - 1)
608 self.umems['qp'] = self.create_umem(size=qp_size)
609 self.umems['qp_dbr'] = self.create_umem(size=8, alignment=8)
610 self.log_rq_size = int(math.log2(self.qattr.rq.wqe_num - 1)) + 1
611 # Size of a receive WQE is 16*pow(2, log_rq_stride)
612 log_rq_stride = self.qattr.rq.wqe_shift - 4
613 log_sq_size = int(math.log2(self.qattr.sq.wqe_num - 1)) + 1
614 cqn = CreateCqOut(self.cq.out_view).cqn
615 qpc = SwQpc(st=DevxOps.MLX5_QPC_ST_RC, pd=self.dv_pd.pdn,
616 pm_state=DevxOps.MLX5_QPC_PM_STATE_MIGRATED,
617 log_rq_size=self.log_rq_size, log_sq_size=log_sq_size, ts_format=0x1,
618 log_rq_stride=log_rq_stride, uar_page=self.uar['qp'].page_id,
619 cqn_snd=cqn, cqn_rcv=cqn, dbr_umem_id=self.umems['qp_dbr'].umem_id,
620 dbr_umem_valid=1, send_dbr_mode=self.send_dbr_mode)
621 cmd_in = CreateQpIn(sw_qpc=qpc, wq_umem_id=self.umems['qp'].umem_id,
622 wq_umem_valid=1)
623 self.qp = Mlx5DevxObj(self.ctx, cmd_in, len(CreateQpOut()))
624 self.qpn = CreateQpOut(self.qp.out_view).qpn
625
626 def create_eq(self):
627 pass
628
629 def to_rts(self):
630 """
631 Moves the created QP to RTS state by modifying it using DevX through all
632 the needed states with all the required attributes.
633 rlid, rpsn, rqpn and rgid (when valid) must be already updated before
634 calling this method.
635 """
636 from tests.mlx5_prm_structs import DevxOps, ModifyQpIn, ModifyQpOut,\
637 CreateQpOut, SwQpc
638 cmd_out_len = len(ModifyQpOut())
639
640 # RST2INIT
641 qpn = CreateQpOut(self.qp.out_view).qpn
642 swqpc = SwQpc(rre=1, rwe=1)
643 swqpc.primary_address_path.vhca_port_num = self.ib_port
644 cmd_in = ModifyQpIn(opcode=DevxOps.MLX5_CMD_OP_RST2INIT_QP, qpn=qpn,
645 sw_qpc=swqpc)
646 self.qp.modify(cmd_in, cmd_out_len)
647
648 # INIT2RTR
649 swqpc = SwQpc(mtu=PATH_MTU, log_msg_max=20, remote_qpn=self.rqpn,
650 min_rnr_nak=MIN_RNR_TIMER, next_rcv_psn=self.rpsn)
651 swqpc.primary_address_path.vhca_port_num = self.ib_port
652 swqpc.primary_address_path.rlid = self.rlid
653 if self.is_eth():
654 # GID field is a must for Eth (or if GRH is set in IB)
655 swqpc.primary_address_path.rgid_rip = self.rgid
656 swqpc.primary_address_path.rmac = self.rmac
657 swqpc.primary_address_path.src_addr_index = self.gid_index
658 swqpc.primary_address_path.hop_limit = tests.utils.PacketConsts.TTL_HOP_LIMIT
659 # UDP sport must be reserved for roce v1 and v1.5
660 if self.ctx.query_gid_type(self.ib_port, self.gid_index) == ibv_gid_type_sysfs.IBV_GID_TYPE_SYSFS_ROCE_V2:
661 swqpc.primary_address_path.udp_sport = 0xdcba
662 else:
663 swqpc.primary_address_path.rlid = self.rlid
664 cmd_in = ModifyQpIn(opcode=DevxOps.MLX5_CMD_OP_INIT2RTR_QP, qpn=qpn,
665 sw_qpc=swqpc)
666 self.qp.modify(cmd_in, cmd_out_len)
667
668 # RTR2RTS
669 swqpc = SwQpc(retry_count=RETRY_CNT, rnr_retry=RNR_RETRY,
670 next_send_psn=self.psn, log_sra_max=MAX_RDMA_ATOMIC)
671 swqpc.primary_address_path.vhca_port_num = self.ib_port
672 swqpc.primary_address_path.ack_timeout = TIMEOUT
673 cmd_in = ModifyQpIn(opcode=DevxOps.MLX5_CMD_OP_RTR2RTS_QP, qpn=qpn,
674 sw_qpc=swqpc)
675 self.qp.modify(cmd_in, cmd_out_len)
676
677 def pre_run(self, rpsn, rqpn, rgid=0, rlid=0, rmac=0):
678 """
679 Configure Resources before running traffic
680 :param rpsns: Remote PSN (packet serial number)
681 :param rqpn: Remote QP number
682 :param rgid: Remote GID
683 :param rlid: Remote LID
684 :param rmac: Remote MAC (valid for RoCE)
685 :return: None
686 """
687 self.rpsn = rpsn
688 self.rqpn = rqpn
689 self.rgid = rgid
690 self.rlid = rlid
691 self.rmac = rmac
692 self.to_rts()
693
694 def post_send(self):
695 """
696 Posts one send WQE to the SQ by doing all the required work such as
697 building the control/data segments, updating and ringing the dbr,
698 updating the producer indexes, etc.
699 """
700 from tests.mlx5_prm_structs import SendDbrMode
701 buffer_address = self.uar['qp'].reg_addr
702 if self.send_dbr_mode == SendDbrMode.NO_DBR_EXT:
703 # Address of DB blueflame register
704 buffer_address = self.uar['qp'].base_addr + DB_BF_DBR_LESS_BUF_OFFSET
705
706 idx = self.qattr.sq.post_idx if self.qattr.sq.post_idx < self.qattr.sq.wqe_num else 0
707 buf_offset = self.qattr.sq.offset + (idx << MLX5_SEND_WQE_SHIFT)
708 # Prepare WQE
709 imm_be32 = struct.unpack("<I", struct.pack(">I", self.imm + self.qattr.sq.post_idx))[0]
710 ctrl_seg = WqeCtrlSeg(imm=imm_be32, fm_ce_se=MLX5_WQE_CTRL_CQ_UPDATE)
711 data_seg = self.get_wqe_data_segment()
712 ctrl_seg.opmod_idx_opcode = (self.qattr.sq.post_idx & 0xffff) << 8 | MLX5_OPCODE_SEND_IMM
713 size_in_octowords = int((ctrl_seg.sizeof() + data_seg.sizeof()) / 16)
714 ctrl_seg.qpn_ds = self.qpn << 8 | size_in_octowords
715 Wqe([ctrl_seg, data_seg], self.umems['qp'].umem_addr + buf_offset)
716 self.qattr.sq.post_idx += int((size_in_octowords * 16 +
717 MLX5_SEND_WQE_BB - 1) / MLX5_SEND_WQE_BB)
718 # Make sure descriptors are written
719 dma.udma_to_dev_barrier()
720 if not self.send_dbr_mode:
721 # Update the doorbell record
722 mem.writebe32(self.umems['qp_dbr'].umem_addr,
723 self.qattr.sq.post_idx & 0xffff, MLX5_SND_DBR)
724 dma.udma_to_dev_barrier()
725 # Ring the doorbell and post the WQE
726 dma.mmio_write64_as_be(buffer_address, mem.read64(ctrl_seg.addr))
727
728 def post_recv(self):
729 """
730 Posts one receive WQE to the RQ by doing all the required work such as
731 building the control/data segments, updating the dbr and the producer
732 indexes.
733 """
734 buf_offset = self.qattr.rq.offset + self.qattr.rq.wqe_size * self.qattr.rq.head
735 # Prepare WQE
736 data_seg = self.get_wqe_data_segment()
737 Wqe([data_seg], self.umems['qp'].umem_addr + buf_offset)
738 # Update indexes
739 self.qattr.rq.post_idx += 1
740 self.qattr.rq.head = self.qattr.rq.head + 1 if self.qattr.rq.head + 1 < self.qattr.rq.wqe_num else 0
741 # Update the doorbell record
742 dma.udma_to_dev_barrier()
743 mem.writebe32(self.umems['qp_dbr'].umem_addr,
744 self.qattr.rq.post_idx & 0xffff, MLX5_RCV_DBR)
745
746 def poll_cq(self):
747 """
748 Polls the CQ once and updates the consumer index upon success.
749 The CQE opcode and owner bit are checked and verified.
750 This method does busy-waiting as long as it gets an empty CQE, until a
751 timeout of POLL_CQ_TIMEOUT seconds.
752 """
753 idx = self.qattr.cq.cons_idx % self.qattr.cq.ncqes
754 cq_owner_flip = not(not(self.qattr.cq.cons_idx & self.qattr.cq.ncqes))
755 cqe_start_addr = self.umems['cq'].umem_addr + (idx * self.qattr.cq.cqe_size)
756 cqe = None
757 start_poll_t = time.perf_counter()
758 while cqe is None:
759 cqe = Mlx5Cqe64(cqe_start_addr)
760 if (cqe.opcode == MLX5_CQE_INVALID) or \
761 (cqe.owner ^ cq_owner_flip) or cqe.is_empty():
762 if time.perf_counter() - start_poll_t >= POLL_CQ_TIMEOUT:
763 raise PyverbsRDMAError(f'CQE #{self.qattr.cq.cons_idx} '
764 f'is empty or invalid:\n{cqe.dump()}')
765 cqe = None
766
767 # After CQE ownership check, must do memory barrier and re-read the CQE.
768 dma.udma_from_dev_barrier()
769 cqe = Mlx5Cqe64(cqe_start_addr)
770
771 if cqe.opcode == MLX5_CQE_RESP_ERR:
772 raise PyverbsRDMAError(f'Got a CQE #{self.qattr.cq.cons_idx} '
773 f'with responder error:\n{cqe.dump()}')
774 elif cqe.opcode == MLX5_CQE_REQ_ERR:
775 raise PyverbsRDMAError(f'Got a CQE #{self.qattr.cq.cons_idx} '
776 f'with requester error:\n{cqe.dump()}')
777
778 self.qattr.cq.cons_idx += 1
779 mem.writebe32(self.umems['cq_dbr'].umem_addr,
780 self.qattr.cq.cons_idx & 0xffffff, MLX5_CQ_SET_CI)
781 return cqe
782
783 def close_resources(self):
784 for obj in self.devx_objs:
785 if obj:
786 obj.close()
787
788
789 class Mlx5DevxTrafficBase(Mlx5RDMATestCase):
790 """
791 A base class for mlx5 DevX traffic tests.
792 This class does not include any tests, but provides quick players (client,
793 server) creation and provides a traffic method.
794 """
795 def tearDown(self):
796 if self.server:
797 self.server.close_resources()
798 if self.client:
799 self.client.close_resources()
800 super().tearDown()
801
802 def create_players(self, resources, **resource_arg):
803 """
804 Initialize tests resources.
805 :param resources: The RDMA resources to use.
806 :param resource_arg: Dictionary of args that specify the resources
807 specific attributes.
808 :return: None
809 """
810 self.server = resources(**self.dev_info, **resource_arg)
811 self.client = resources(**self.dev_info, **resource_arg)
812 self.pre_run()
813
814 def pre_run(self):
815 self.server.pre_run(self.client.psn, self.client.qpn, self.client.gid,
816 self.client.lid, self.mac_addr)
817 self.client.pre_run(self.server.psn, self.server.qpn, self.server.gid,
818 self.server.lid, self.mac_addr)
819
820 def invalidate_mr_pages(self):
821 if self.client.with_odp:
822 mem.madvise(self.client.mr.buf, self.client.msg_size)
823 self.client.mem_write('c' * self.client.msg_size, self.client.msg_size)
824 if self.server.with_odp:
825 mem.madvise(self.server.mr.buf, self.server.msg_size)
826
827 def send_imm_traffic(self):
828 self.client.mem_write('c' * self.client.msg_size, self.client.msg_size)
829 for _ in range(self.client.num_msgs):
830 cons_idx = self.client.qattr.cq.cons_idx
831 self.invalidate_mr_pages()
832 self.server.post_recv()
833 self.client.post_send()
834 # Poll client and verify received cqe opcode
835 send_cqe = self.client.poll_cq()
836 self.assertEqual(send_cqe.opcode, MLX5_CQE_REQ,
837 'Unexpected CQE opcode')
838 # Poll server and verify received cqe opcode
839 recv_cqe = self.server.poll_cq()
840 self.assertEqual(recv_cqe.opcode, MLX5_CQE_RESP_SEND_IMM,
841 'Unexpected CQE opcode')
842 msg_received = self.server.mem_read()
843 # Validate data (of received message and immediate value)
844 tests.utils.validate(msg_received, True, self.server.msg_size)
845 imm_inval_pkey = recv_cqe.imm_inval_pkey
846 if sys.byteorder == 'big':
847 imm_inval_pkey = int.from_bytes(
848 imm_inval_pkey.to_bytes(4, byteorder='big'), 'little')
849 self.assertEqual(imm_inval_pkey, self.client.imm + cons_idx)
850 self.server.mem_write('s' * self.server.msg_size, self.server.msg_size)
851
852
853 class Mlx5RcResources(RCResources):
854 def __init__(self, dev_name, ib_port, gid_index, **kwargs):
855 self.dv_send_ops_flags = 0
856 self.send_ops_flags = 0
857 self.create_send_ops_flags()
858 super().__init__(dev_name, ib_port, gid_index, **kwargs)
859
860 def create_send_ops_flags(self):
861 self.dv_send_ops_flags = 0
862 self.send_ops_flags = ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND
863
864 def create_context(self):
865 mlx5dv_attr = Mlx5DVContextAttr()
866 try:
867 self.ctx = Mlx5Context(mlx5dv_attr, name=self.dev_name)
868 except PyverbsUserError as ex:
869 raise unittest.SkipTest(f'Could not open mlx5 context ({ex})')
870 except PyverbsRDMAError:
871 raise unittest.SkipTest('Opening mlx5 context is not supported')
872
873 def create_qp_init_attr(self):
874 comp_mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD | ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS
875 return QPInitAttrEx(cap=self.create_qp_cap(), pd=self.pd, scq=self.cq,
876 rcq=self.cq, qp_type=ibv_qp_type.IBV_QPT_RC,
877 send_ops_flags=self.send_ops_flags,
878 comp_mask=comp_mask)
879
880 def create_qps(self):
881 try:
882 qp_init_attr = self.create_qp_init_attr()
883 comp_mask = mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS
884 if self.dv_send_ops_flags:
885 comp_mask |= mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS
886 attr = Mlx5DVQPInitAttr(comp_mask=comp_mask,
887 send_ops_flags=self.dv_send_ops_flags)
888 qp = Mlx5QP(self.ctx, qp_init_attr, attr)
889 self.qps.append(qp)
890 self.qps_num.append(qp.qp_num)
891 self.psns.append(random.getrandbits(24))
892 except PyverbsRDMAError as ex:
893 if ex.error_code == errno.EOPNOTSUPP:
894 raise unittest.SkipTest('Create Mlx5DV QP is not supported')
895 raise ex
896
897 def create_cq(self):
898 """
899 Initializes self.cq with a dv_cq
900 :return: None
901 """
902 dvcq_init_attr = Mlx5DVCQInitAttr()
903 try:
904 self.cq = Mlx5CQ(self.ctx, CqInitAttrEx(), dvcq_init_attr)
905 except PyverbsRDMAError as ex:
906 if ex.error_code == errno.EOPNOTSUPP:
907 raise unittest.SkipTest('Create Mlx5DV CQ is not supported')
908 raise ex
909
910
911 def create_privileged_context(agr_obj):
912 """
913 Creates a mlx5 privileged context.
914 :param agr_obj: Aggregation object which contains all resources necessary.
915 """
916 fds_paths = ['/dev/infiniband/mlx5_perm_ctrl_local',
917 '/dev/infiniband/mlx5_perm_ctrl_other_vhca']
918 open_fds = []
919 try:
920 for file_path in fds_paths:
921 fd = open(file_path, 'rb')
922 open_fds.append(fd)
923 agr_obj.fds = FdArr(arr=[fd.fileno() for fd in open_fds], count=len(open_fds))
924 attr = Mlx5DVContextAttr(flags=mlx5dv_context_attr_flags.MLX5DV_CONTEXT_FLAGS_DEVX,
925 comp_mask=mlx5dv_context_attr_comp_mask.MLX5DV_CONTEXT_ATTR_MASK_FD_ARRAY,
926 fds=agr_obj.fds)
927 agr_obj.ctx = Mlx5Context(attr, agr_obj.dev_name)
928 for fd in open_fds: fd.close()
929 except FileNotFoundError:
930 for fd in open_fds: fd.close()
931 raise unittest.SkipTest(f'FDs file not found')
932 except PyverbsUserError as ex:
933 for fd in open_fds: fd.close()
934 raise unittest.SkipTest(f'Could not open mlx5 context ({ex})')
935 except PyverbsRDMAError as ex:
936 for fd in open_fds: fd.close()
937 raise unittest.SkipTest(f'Opening mlx5 DevX context is not supported ({ex})')
938
939
940 class Mlx5PrivilegedRC(Mlx5RcResources):
941 def __init__(self, dev_name, ib_port, gid_index, msg_size=1024, **kwargs):
942 """
943 Initialize mlx5 DV privileged context resources based on Mlx5RcResources.
944 :param dev_name: Device name to be used
945 :param ib_port: IB port of the device to use
946 :param gid_index: Which GID index to use
947 :param msg_size: The resource msg size
948 :param kwargs: General arguments
949 """
950 self.open_fds = []
951 super().__init__(dev_name, ib_port, gid_index, msg_size=msg_size, **kwargs)
952
953 def create_context(self):
954 return create_privileged_context(self)
955
956 def close_fds(self):
957 for open_fd in self.open_fds:
958 open_fd.close()