Debian Codegraph @

tests/base.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2019 Mellanox Technologies, Inc . All rights reserved. See COPYING file
3
4 import multiprocessing as mp
5 import subprocess
6 import functools
7 import unittest
8 import tempfile
9 import random
10 import errno
11 import stat
12 import json
13 import sys
14 import os
15
16 from pyverbs.qp import QPCap, QPInitAttrEx, QPInitAttr, QPAttr, QP
17 from pyverbs.srq import SRQ, SrqInitAttrEx, SrqInitAttr, SrqAttr
18 from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
19 from pyverbs.addr import AHAttr, GlobalRoute
20 from pyverbs.xrcd import XRCD, XRCDInitAttr
21 from pyverbs.device import Context
22 from args_parser import parser
23 from pyverbs.librdmacm_enums import rdma_port_space
24 import pyverbs.device as d
25 from pyverbs.libibverbs_enums import ibv_mtu, ibv_port_state, ibv_gid_type_sysfs, ibv_access_flags,\
26 ibv_qp_type, ibv_qp_init_attr_mask, ibv_xrcd_init_attr_mask,\
27 ibv_srq_type, ibv_srq_init_attr_mask, IBV_LINK_LAYER_ETHERNET
28 from pyverbs.pd import PD
29 from pyverbs.cq import CQ
30 from pyverbs.mr import MR
31
32
33 PATH_MTU = ibv_mtu.IBV_MTU_1024
34 MAX_DEST_RD_ATOMIC = 1
35 NUM_OF_PROCESSES = 2
36 MC_IP_PREFIX = '230'
37 MAX_RDMA_ATOMIC = 20
38 MAX_RD_ATOMIC = 1
39 MIN_RNR_TIMER =12
40 RETRY_CNT = 7
41 RNR_RETRY = 7
42 TIMEOUT = 14
43 # Devices that don't support RoCEv2 should be added here
44 MLNX_VENDOR_ID = 0x02c9
45 CX3_MLNX_PART_ID = 4099
46 CX3Pro_MLNX_PART_ID = 4103
47 DCT_KEY = 0xbadc0de
48 # Dictionary: vendor_id -> array of part_ids of devices that lack RoCEv2 support
49 ROCEV2_UNSUPPORTED_DEVS = {MLNX_VENDOR_ID: [CX3Pro_MLNX_PART_ID,
50 CX3_MLNX_PART_ID]}
51
52
53 def has_roce_hw_bug(vendor_id, vendor_part_id):
54 return vendor_part_id in ROCEV2_UNSUPPORTED_DEVS.get(vendor_id, [])
55
56
57 def set_rnr_attributes(qp_attr):
58 """
59 Set default QP RNR attributes.
60 :param qp_attr: The QPAttr to set its attributes
61 :return: None
62 """
63 qp_attr.min_rnr_timer = MIN_RNR_TIMER
64 qp_attr.timeout = TIMEOUT
65 qp_attr.retry_cnt = RETRY_CNT
66 qp_attr.rnr_retry = RNR_RETRY
67
68
69 def is_gid_available(gid_index):
70 if gid_index is None:
71 raise unittest.SkipTest(f'No relevant GID found')
72
73
74 class PyverbsAPITestCase(unittest.TestCase):
75 def __init__(self, methodName='runTest'):
76 super().__init__(methodName)
77 # Hold the command line arguments
78 self.config = parser.get_config()
79 self.dev_name = None
80 self.ctx = None
81 self.attr = None
82 self.attr_ex = None
83 self.gid_index = 0
84 self.pre_environment = {}
85
86 def setUp(self):
87 """
88 Opens the device and queries it.
89 The results of the query and query_ex are stored in attr and attr_ex
90 instance attributes respectively.
91 If the user didn't pass a device name, the first device is chosen by
92 default.
93 """
94 self.ib_port = self.config['port']
95 self.dev_name = self.config['dev']
96 if not self.dev_name:
97 dev_list = d.get_device_list()
98 if not dev_list:
99 raise unittest.SkipTest('No IB devices found')
100 self.dev_name = dev_list[0].name.decode()
101
102 if self.config['gid']:
103 self.gid_index = self.config['gid']
104
105 self.create_context()
106 self.attr = self.ctx.query_device()
107 self.attr_ex = self.ctx.query_device_ex()
108
109 def create_context(self):
110 self.ctx = d.Context(name=self.dev_name)
111
112 def set_env_variable(self, var, value):
113 """
114 Set environment variable. The current value for each variable is stored
115 and is set back at the end of the test.
116 :param var: The name of the environment variable
117 :param value: The requested new value of this environment variable
118 """
119 if var not in self.pre_environment.keys():
120 self.pre_environment[var] = os.environ.get(var)
121 os.environ[var] = value
122
123 def tearDown(self):
124 for k, v in self.pre_environment.items():
125 if v is None:
126 os.environ.pop(k)
127 else:
128 os.environ[k] = v
129 self.ctx.close()
130
131
132 class RDMATestCase(unittest.TestCase):
133 ZERO_GID = '0000:0000:0000:0000'
134
135 def __init__(self, methodName='runTest', dev_name=None, ib_port=None,
136 gid_index=None, pkey_index=None, gid_type=None):
137 """
138 Initialize a RDMA test unit based on unittest.TestCase.
139 If no device was provided, it iterates over the existing devices, for
140 each port of each device, it checks which GID indexes are valid (in RoCE,
141 only IPv4 and IPv6 based GIDs are used). Each <dev, port, gid> is added
142 to an array and one entry is selected.
143 If a device was provided, the same process is done for all ports of this
144 device (in case they're not provided), and so on.
145 If gid_type is provided by the user, only GIDs of that type would be
146 be chosen (valid only if gid_index was not provided).
147 :param methodName: The base method to be used by the unittest
148 :param dev_name: Device name to use
149 :param ib_port: IB port of the device to use
150 :param gid_index: GID index to use
151 :param pkey_index: PKEY index to use
152 :param gid_type: If provided, only GIDs of gid_type will be chosen
153 (ignored if gid_index is provided by the user)
154 """
155 super(RDMATestCase, self).__init__(methodName)
156 # Hold the command line arguments
157 self.config = parser.get_config()
158 dev = self.config['dev']
159 self.dev_name = dev_name if dev_name else dev
160 self.ib_port = ib_port if ib_port else self.config['port']
161 self.gid_index = gid_index if gid_index else self.config['gid']
162 self.pkey_index = pkey_index
163 self.gid_type = gid_type if gid_index is None else None
164 self.ip_addr = None
165 self.mac_addr = None
166 self.pre_environment = {}
167 self.server = None
168 self.client = None
169 self.iters = 10
170
171
172 def is_eth_and_has_roce_hw_bug(self):
173 """
174 Check if the link layer is Ethernet and the device lacks RoCEv2 support
175 with a known HW bug.
176 return: True if the link layer is Ethernet and device is not supported
177 """
178 ctx = d.Context(name=self.dev_name)
179 port_attrs = ctx.query_port(self.ib_port)
180 dev_attrs = ctx.query_device()
181 vendor_id = dev_attrs.vendor_id
182 vendor_pid = dev_attrs.vendor_part_id
183 return port_attrs.link_layer == IBV_LINK_LAYER_ETHERNET and \
184 has_roce_hw_bug(vendor_id, vendor_pid)
185
186 @staticmethod
187 def get_net_name(dev, port=None):
188 if port is not None:
189 out = subprocess.check_output(['rdma', 'link', 'show', '-j'])
190 loaded_json = json.loads(out.decode())
191 for row in loaded_json:
192 try:
193 if row['ifname'] == dev and row['port'] == port:
194 return row['netdev']
195 except KeyError:
196 pass
197
198 if not os.path.exists(f'/sys/class/infiniband/{dev}/device/net/'):
199 return None
200
201 out = subprocess.check_output(['ls', f'/sys/class/infiniband/{dev}/device/net/'])
202 return out.decode().split('\n')[0]
203
204 @staticmethod
205 def get_ip_mac_address(ifname):
206 out = subprocess.check_output(['ip', '-j', 'addr', 'show', ifname])
207 loaded_json = json.loads(out.decode())
208 interface = loaded_json[0]['addr_info'][0]['local']
209 mac = loaded_json[0]['address']
210 if 'fe80::' in interface:
211 interface = interface + '%' + ifname
212 return interface, mac
213
214 def setUp(self):
215 """
216 Verify that the test case has dev_name, ib_port, gid_index and pkey index.
217 If not provided by the user, the first valid combination will be used.
218 """
219 if self.pkey_index is None:
220 # To avoid iterating the entire pkeys table, if a pkey index wasn't
221 # provided, use index 0 which is always valid
222 self.pkey_index = 0
223
224 self.args = []
225 if self.dev_name is not None:
226 ctx = d.Context(name=self.dev_name)
227 if self.ib_port is not None:
228 if self.gid_index is not None:
229 self._get_ip_mac(self.dev_name, self.ib_port, self.gid_index)
230 else:
231 # Add avaiable GIDs of the given dev_name + port
232 self._add_gids_per_port(ctx, self.dev_name, self.ib_port)
233 else:
234 # Add available GIDs for each port of the given dev_name
235 self._add_gids_per_device(ctx, self.dev_name)
236 else:
237 # Iterate available devices, add available GIDs for each of
238 # their ports
239 lst = d.get_device_list()
240 for dev in lst:
241 dev_name = dev.name.decode()
242 ctx = d.Context(name=dev_name)
243 self._add_gids_per_device(ctx, dev_name)
244
245 if not self.args:
246 raise unittest.SkipTest('No supported port is up, can\'t run traffic')
247 # Choose one combination and use it
248 self._select_config()
249 self.dev_info = {'dev_name': self.dev_name, 'ib_port': self.ib_port,
250 'gid_index': self.gid_index}
251
252 def _add_gids_per_port(self, ctx, dev, port):
253 # Don't add ports which are not active
254 port_attrs = ctx.query_port(port)
255 if port_attrs.state != ibv_port_state.IBV_PORT_ACTIVE:
256 return
257 if not port_attrs.gid_tbl_len:
258 self._get_ip_mac(dev, port, None)
259 return
260 dev_attrs = ctx.query_device()
261 vendor_id = dev_attrs.vendor_id
262 vendor_pid = dev_attrs.vendor_part_id
263 for idx in range(port_attrs.gid_tbl_len):
264 gid = ctx.query_gid(port, idx)
265 # Avoid adding ZERO GIDs
266 if gid.gid[-19:] == self.ZERO_GID:
267 continue
268 # Avoid RoCEv2 GIDs on unsupported devices
269 if port_attrs.link_layer == IBV_LINK_LAYER_ETHERNET and \
270 ctx.query_gid_type(port, idx) == \
271 ibv_gid_type_sysfs.IBV_GID_TYPE_SYSFS_ROCE_V2 and \
272 has_roce_hw_bug(vendor_id, vendor_pid):
273 continue
274 if self.gid_type is not None and ctx.query_gid_type(port, idx) != \
275 self.gid_type:
276 continue
277 self._get_ip_mac(dev, port, idx)
278
279 def _add_gids_per_device(self, ctx, dev):
280 self._add_gids_per_port(ctx, dev, self.ib_port)
281
282 def _get_ip_mac(self, dev, port, idx):
283 net_name = self.get_net_name(dev, port)
284 if net_name is None:
285 self.args.append([dev, port, idx, None, None])
286 return
287 try:
288 ip_addr, mac_addr = self.get_ip_mac_address(net_name)
289 except (KeyError, IndexError):
290 self.args.append([dev, port, idx, None, None])
291 else:
292 self.args.append([dev, port, idx, ip_addr, mac_addr])
293
294 def _select_config(self):
295 args_with_inet_ip = []
296 for arg in self.args:
297 if arg[3]:
298 args_with_inet_ip.append(arg)
299 if args_with_inet_ip:
300 args = args_with_inet_ip[0]
301 else:
302 args = self.args[0]
303 self.dev_name = args[0]
304 self.ib_port = args[1]
305 self.gid_index = args[2]
306 self.ip_addr = args[3]
307 self.mac_addr = args[4]
308
309 def set_env_variable(self, var, value):
310 """
311 Set environment variable. The current value for each variable is stored
312 and is set back at the end of the test.
313 :param var: The name of the environment variable
314 :param value: The requested new value of this environment variable
315 """
316 if var not in self.pre_environment.keys():
317 self.pre_environment[var] = os.environ.get(var)
318 os.environ[var] = value
319
320 def sync_remote_attr(self):
321 """
322 Sync the MR remote attributes between the server and the client.
323 """
324 self.server.rkey = self.client.mr.rkey
325 self.server.raddr = self.client.mr.buf
326 self.client.rkey = self.server.mr.rkey
327 self.client.raddr = self.server.mr.buf
328
329 def pre_run(self):
330 """
331 Configure Resources before running traffic.
332 pre_run() must be implemented by the client and server.
333 """
334 self.client.pre_run(self.server.psns, self.server.qps_num)
335 self.server.pre_run(self.client.psns, self.client.qps_num)
336
337 def create_players(self, resource, sync_attrs=True, **resource_arg):
338 """
339 Init test resources.
340 :param resource: The RDMA resources to use.
341 :param sync_attrs: If True, sync remote attrs such as rkey and raddr
342 :param resource_arg: Dict of args that specify the resource specific
343 attributes.
344 """
345 try:
346 self.client = resource(**self.dev_info, **resource_arg)
347 self.server = resource(**self.dev_info, **resource_arg)
348 except PyverbsRDMAError as ex:
349 if ex.error_code == errno.EOPNOTSUPP:
350 raise unittest.SkipTest(f'Create player of {resource.__name__} is not supported')
351 raise ex
352 self.pre_run()
353 if sync_attrs:
354 self.sync_remote_attr()
355 self.traffic_args = {'client': self.client, 'server': self.server,
356 'iters': self.iters, 'gid_idx': self.gid_index,
357 'port': self.ib_port}
358
359 def tearDown(self):
360 """
361 Restore the previous environment variables values before ending the test.
362 """
363 for k, v in self.pre_environment.items():
364 if v is None:
365 os.environ.pop(k)
366 else:
367 os.environ[k] = v
368 if self.server:
369 self.server.ctx.close()
370 if self.client:
371 self.client.ctx.close()
372 super().tearDown()
373
374
375 class RDMACMBaseTest(RDMATestCase):
376 """
377 Base RDMACM test class.
378 This class does not include any test, but rather implements generic
379 connection and traffic methods that are needed by RDMACM tests in general.
380 Each RDMACM test should have a class that inherits this class and extends
381 its functionalities if needed.
382 """
383 def setUp(self):
384 super().setUp()
385 if not self.ip_addr:
386 raise unittest.SkipTest('Device {} doesn\'t have net interface'
387 .format(self.dev_name))
388 is_gid_available(self.gid_index)
389
390 def two_nodes_rdmacm_traffic(self, connection_resources, test_flow, bad_flow=False,
391 **resource_kwargs):
392 """
393 Init and manage the rdmacm test processes. The exit code of the
394 test processes indicates if exception was thrown.
395 {0: pass, 2: exception was thrown, 5: skip test}
396 If needed, terminate those processes and raise an exception.
397 :param connection_resources: The CMConnection resources to use.
398 :param test_flow: The target RDMACM flow method to run.
399 :param bad_flow: If true, traffic is expected to fail.
400 :param resource_kwargs: Dict of args that specify the CMResources
401 specific attributes. Each test case can pass
402 here as key words the specific CMResources
403 attributes that are requested.
404 :return: None
405 """
406 if resource_kwargs.get('port_space', None) == rdma_port_space.RDMA_PS_UDP and \
407 self.is_eth_and_has_roce_hw_bug():
408 raise unittest.SkipTest('Device {} doesn\'t support UDP with RoCEv2'
409 .format(self.dev_name))
410 ctx = mp.get_context('fork')
411 self.syncer = ctx.Barrier(NUM_OF_PROCESSES, timeout=15)
412 self.notifier = ctx.Queue()
413 passive = ctx.Process(target=test_flow,
414 kwargs={'connection_resources': connection_resources,
415 'passive':True, **resource_kwargs})
416 active = ctx.Process(target=test_flow,
417 kwargs={'connection_resources': connection_resources,
418 'passive':False, **resource_kwargs})
419 passive.start()
420 active.start()
421 repeat_times=150 if not bad_flow else 3
422 proc_res = {}
423 for _ in range(repeat_times):
424 for proc in [passive, active]:
425 proc.join(0.1)
426 # Write the exit code of the proc.
427 if not proc.is_alive():
428 side = 'passive' if proc == passive else 'active'
429 if side not in proc_res.keys():
430 proc_res[side] = proc.exitcode
431 # If the processes is still alive kill them and fail the test.
432 proc_killed = False
433 for proc in [passive, active]:
434 if proc.is_alive():
435 proc.terminate()
436 proc_killed = True
437 # Check if need to skip this test
438 for side in proc_res.keys():
439 if proc_res[side] == 5:
440 raise unittest.SkipTest(f'SkipTest occurred on {side} side')
441 # Raise exeption if the test proceses was terminate.
442 if bad_flow and not proc_killed:
443 raise Exception('Bad flow: traffic passed which is not expected')
444 if not bad_flow and proc_killed:
445 raise Exception('RDMA CM test procces is stuck, kill the test')
446
447 # Check if the test processes raise exceptions.
448 for side in proc_res:
449 if proc_res[side] != 0:
450 raise Exception(f'Exception in {side} side occurred')
451
452 def rdmacm_traffic(self, connection_resources=None, passive=None, **kwargs):
453 """
454 Run RDMACM traffic between two CMIDs.
455 :param connection_resources: The connection resources to use.
456 :param passive: Indicate if this CMID is this the passive side.
457 :param kwargs: Arguments to be passed to the connection_resources.
458 :return: None
459 """
460 try:
461 player = connection_resources(ip_addr=self.ip_addr,
462 syncer=self.syncer,
463 notifier=self.notifier,
464 passive=passive, **kwargs)
465 player.establish_connection()
466 if kwargs.get('reject_conn'):
467 return
468 player.rdmacm_traffic()
469 player.disconnect()
470 except Exception as ex:
471 self._rdmacm_exception_handler(passive, ex)
472
473 def rdmacm_multicast_traffic(self, connection_resources=None, passive=None,
474 extended=False, leave_test=False, **kwargs):
475 """
476 Run RDMACM multicast traffic between two CMIDs.
477 :param connection_resources: The connection resources to use.
478 :param passive: Indicate if this CMID is the passive side.
479 :param extended: Use exteneded multicast join request. This request
480 allows CMID to join with specific join flags.
481 :param leave_test: Perform traffic after leaving the multicast group to
482 ensure leave works.
483 :param kwargs: Arguments to be passed to the connection_resources.
484 :return: None
485 """
486 try:
487 player = connection_resources(ip_addr=self.ip_addr, syncer=self.syncer,
488 notifier=self.notifier, passive=False,
489 **kwargs)
490 mc_addr = MC_IP_PREFIX + self.ip_addr[self.ip_addr.find('.'):]
491 player.join_to_multicast(src_addr=self.ip_addr, mc_addr=mc_addr,
492 extended=extended)
493 player.rdmacm_traffic(server=passive, multicast=True)
494 player.leave_multicast(mc_addr=mc_addr)
495 if leave_test:
496 player.rdmacm_traffic(server=passive, multicast=True)
497 except Exception as ex:
498 self._rdmacm_exception_handler(passive, ex)
499
500 def rdmacm_remote_traffic(self, connection_resources=None, passive=None,
501 remote_op='write', **kwargs):
502 """
503 Run RDMACM remote traffic between two CMIDs.
504 :param connection_resources: The connection resources to use.
505 :param passive: Indicate if this CMID is the passive side.
506 :param remote_op: The remote operation in the traffic.
507 :param kwargs: Arguments to be passed to the connection_resources.
508 :return: None
509 """
510 try:
511 player = connection_resources(ip_addr=self.ip_addr,
512 syncer=self.syncer,
513 notifier=self.notifier,
514 passive=passive,
515 remote_op=remote_op, **kwargs)
516 player.establish_connection()
517 player.remote_traffic(passive=passive, remote_op=remote_op)
518 player.disconnect()
519 except Exception as ex:
520 self._rdmacm_exception_handler(passive, ex)
521
522 @staticmethod
523 def _rdmacm_exception_handler(passive, exception):
524 if isinstance(exception, PyverbsRDMAError):
525 if exception.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
526 sys.exit(5)
527 if isinstance(exception, unittest.case.SkipTest):
528 sys.exit(5)
529 side = 'passive' if passive else 'active'
530 print(f'Player {side} got: {exception}')
531 sys.exit(2)
532
533
534 def catch_skiptest(func):
535 """
536 Decorator to catch unittest.SkipTest in __init__ resource functions.
537 It gracefully closes the context and all of its underlying resources.
538 """
539 @functools.wraps(func)
540 def wrapper(self, *args, **kwargs):
541 try:
542 func(self, *args, **kwargs)
543 except unittest.SkipTest as e:
544 if hasattr(self, 'ctx') and self.ctx:
545 self.ctx.close()
546 raise e
547 return wrapper
548
549
550 class SkipTestMeta(type):
551 """
552 Metaclass to automatically wrap __init__ in catch_skiptest.
553 It should only be used in resource classes, such as those inheriting from
554 BaseResources.
555 """
556 def __new__(cls, name, bases, dct):
557 if "__init__" in dct:
558 dct["__init__"] = catch_skiptest(dct["__init__"])
559 return super().__new__(cls, name, bases, dct)
560
561
562 class BaseResources(object, metaclass=SkipTestMeta):
563 """
564 BaseResources class is a base aggregator object which contains basic
565 resources like Context and PD. It opens a context over the given device
566 and port and allocates a PD.
567 """
568 def __init__(self, dev_name, ib_port, gid_index):
569 """
570 Initializes a BaseResources object.
571 :param dev_name: Device name to be used (default: 'ibp0s8f0')
572 :param ib_port: IB port of the device to use (default: 1)
573 :param gid_index: Which GID index to use (default: 0)
574 """
575 self.dev_name = dev_name
576 self.gid_index = gid_index
577 self.ib_port = ib_port
578 self.ctx = None
579 self.create_context()
580 self.create_pd()
581
582 def create_context(self):
583 self.ctx = Context(name=self.dev_name)
584
585 def create_pd(self):
586 self.pd = PD(self.ctx)
587
588 def mem_write(self, data, size, offset=0):
589 self.mr.write(data, size, offset)
590
591 def mem_read(self, size=None, offset=0):
592 size_ = self.msg_size if size is None else size
593 return self.mr.read(size_, offset)
594
595
596 class TrafficResources(BaseResources):
597 """
598 Basic traffic class. It provides the basic RDMA resources and operations
599 needed for traffic.
600 """
601 def __init__(self, dev_name, ib_port, gid_index, with_srq=False,
602 qp_count=1, msg_size=1024):
603 """
604 Initializes a TrafficResources object with the given values and creates
605 basic RDMA resources.
606 :param dev_name: Device name to be used
607 :param ib_port: IB port of the device to use
608 :param gid_index: Which GID index to use
609 :param with_srq: If True, create SRQ and attach to QPs
610 :param qp_count: Number of QPs to create
611 :param msg_size: Size of resource msg. If None, use 1024 as default.
612 """
613 super(TrafficResources, self).__init__(dev_name=dev_name,
614 ib_port=ib_port,
615 gid_index=gid_index)
616 self.msg_size = msg_size
617 self.num_msgs = 1000
618 self.port_attr = None
619 self.mr = None
620 self.use_mr_prefetch = None
621 self.srq = None
622 self.cq = None
623 self.qps = []
624 self.qps_num = []
625 self.psns = []
626 self.rqps_num = None
627 self.rpsns = None
628 self.with_srq = with_srq
629 self.qp_count = qp_count
630 self.init_resources()
631
632 @property
633 def qp(self):
634 return self.qps[0]
635
636 @property
637 def mr_lkey(self):
638 if self.mr:
639 return self.mr.lkey
640
641 def init_resources(self):
642 """
643 Initializes a CQ, MR and an RC QP.
644 :return: None
645 """
646 self.port_attr = self.ctx.query_port(self.ib_port)
647 self.create_cq()
648 if self.with_srq:
649 self.create_srq()
650 self.create_mr()
651 self.create_qps()
652
653 def create_cq(self):
654 """
655 Initializes self.cq with a CQ of depth <num_msgs> - defined by each
656 test.
657 :return: None
658 """
659 self.cq = CQ(self.ctx, self.num_msgs, None, None, 0)
660
661 def create_mr(self):
662 """
663 Initializes self.mr with an MR of length <msg_size> - defined by each
664 test.
665 :return: None
666 """
667 self.mr = MR(self.pd, self.msg_size, ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
668
669 def create_qp_cap(self):
670 return QPCap(max_recv_wr=self.num_msgs)
671
672 def create_qp_init_attr(self):
673 return QPInitAttr(qp_type=ibv_qp_type.IBV_QPT_RC, scq=self.cq, rcq=self.cq,
674 srq=self.srq, cap=self.create_qp_cap())
675
676 def create_qp_attr(self):
677 return QPAttr(port_num=self.ib_port)
678
679 def create_qps(self):
680 """
681 Initializes self.qps with RC QPs.
682 :return: None
683 """
684 qp_init_attr = self.create_qp_init_attr()
685 qp_attr = self.create_qp_attr()
686 for _ in range(self.qp_count):
687 try:
688 qp = QP(self.pd, qp_init_attr, qp_attr)
689 self.qps.append(qp)
690 self.qps_num.append(qp.qp_num)
691 self.psns.append(random.getrandbits(24))
692 except PyverbsRDMAError as ex:
693 if ex.error_code == errno.EOPNOTSUPP:
694 raise unittest.SkipTest(f'Create QP type {qp_init_attr.qp_type} is not supported')
695 raise ex
696
697 def create_srq_attr(self):
698 return SrqAttr(max_wr=self.num_msgs*self.qp_count)
699
700 def create_srq_init_attr(self):
701 return SrqInitAttr(self.create_srq_attr())
702
703 def create_srq(self):
704 srq_init_attr = self.create_srq_init_attr()
705 try:
706 self.srq = SRQ(self.pd, srq_init_attr)
707 except PyverbsRDMAError as ex:
708 if ex.error_code == errno.EOPNOTSUPP:
709 raise unittest.SkipTest('Create SRQ is not supported')
710 raise ex
711
712 def pre_run(self, rpsns, rqps_num):
713 """
714 Configure resources before running traffic and modifies the QP to RTS
715 if required.
716 :param rpsns: Remote PSNs (packet serial numbers)
717 :param rqps_num: Remote QPs Number
718 """
719 self.rpsns = rpsns
720 self.rqps_num = rqps_num
721 self.to_rts()
722
723 def to_rts(self):
724 """
725 Modify the QP's states to RTS and initialize it to be ready for traffic.
726 If not required, can be "passed" but must be implemented.
727 """
728 raise NotImplementedError()
729
730
731 class RoCETrafficResources(TrafficResources):
732 def __init__(self, dev_name, ib_port, gid_index, **kwargs):
733 is_gid_available(gid_index)
734 super(RoCETrafficResources, self).__init__(dev_name, ib_port, gid_index, **kwargs)
735
736
737 class RCResources(RoCETrafficResources):
738 def __init__(self, dev_name, ib_port, gid_index,
739 max_dest_rd_atomic=MAX_DEST_RD_ATOMIC, max_rd_atomic=MAX_RD_ATOMIC, **kwargs):
740 """
741 Initializes a TrafficResources object with the given values and creates
742 basic RDMA resources.
743 :param dev_name: Device name to be used
744 :param ib_port: IB port of the device to use
745 :param gid_index: Which GID index to use
746 :param max_dest_rd_atomic: Number of responder incoming rd_atomic operations that can
747 be handled
748 :param max_rd_atomic: Number of outstanding destination rd_atomic operations
749 """
750 self.max_dest_rd_atomic = max_dest_rd_atomic
751 self.max_rd_atomic = max_rd_atomic
752 super().__init__(dev_name, ib_port, gid_index, **kwargs)
753
754 def to_rts(self):
755 """
756 Set the QP attributes' values to arbitrary values (same values used in
757 ibv_rc_pingpong).
758 :return: None
759 """
760 attr = self.create_qp_attr()
761 attr.path_mtu = PATH_MTU
762 attr.max_dest_rd_atomic = self.max_dest_rd_atomic
763 set_rnr_attributes(attr)
764 attr.max_rd_atomic = self.max_rd_atomic
765 gr = GlobalRoute(dgid=self.ctx.query_gid(self.ib_port, self.gid_index),
766 sgid_index=self.gid_index)
767 ah_attr = AHAttr(port_num=self.ib_port, is_global=1, gr=gr,
768 dlid=self.port_attr.lid)
769 attr.ah_attr = ah_attr
770 for i in range(self.qp_count):
771 attr.dest_qp_num = self.rqps_num[i]
772 attr.rq_psn = self.psns[i]
773 attr.sq_psn = self.rpsns[i]
774 self.qps[i].to_rts(attr)
775
776
777 class UDResources(RoCETrafficResources):
778 UD_QKEY = 0x11111111
779 UD_PKEY_INDEX = 0
780 GRH_SIZE = 40
781
782 def create_mr(self):
783 self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE,
784 ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
785
786 def create_qp_init_attr(self):
787 return QPInitAttr(qp_type=ibv_qp_type.IBV_QPT_UD, scq=self.cq,
788 rcq=self.cq, srq=self.srq, cap=self.create_qp_cap())
789
790 def create_qps(self):
791 qp_init_attr = self.create_qp_init_attr()
792 qp_attr = self.create_qp_attr()
793 qp_attr.qkey = self.UD_QKEY
794 qp_attr.pkey_index = self.UD_PKEY_INDEX
795 for _ in range(self.qp_count):
796 try:
797 qp = QP(self.pd, qp_init_attr, qp_attr)
798 self.qps.append(qp)
799 self.qps_num.append(qp.qp_num)
800 self.psns.append(random.getrandbits(24))
801 except PyverbsRDMAError as ex:
802 if ex.error_code == errno.EOPNOTSUPP:
803 raise unittest.SkipTest(f'Create QP type {qp_init_attr.qp_type} is not supported')
804 raise ex
805
806 def to_rts(self):
807 pass
808
809
810 class RawResources(TrafficResources):
811 def create_qp_init_attr(self):
812 return QPInitAttr(qp_type=ibv_qp_type.IBV_QPT_RAW_PACKET, scq=self.cq,
813 rcq=self.cq, srq=self.srq, cap=self.create_qp_cap())
814
815 def pre_run(self, rpsns=None, rqps_num=None):
816 pass
817
818
819 class XRCResources(RoCETrafficResources):
820 def __init__(self, dev_name, ib_port, gid_index, qp_count=2):
821 self.temp_file = None
822 self.xrcd_fd = -1
823 self.xrcd = None
824 self.sqp_lst = []
825 self.rqp_lst = []
826 super(XRCResources, self).__init__(dev_name, ib_port, gid_index,
827 qp_count=qp_count)
828
829 def close(self):
830 os.close(self.xrcd_fd)
831 self.temp_file.close()
832
833 @property
834 def qp(self):
835 return self.sqp_lst[0]
836
837 def create_qps(self):
838 """
839 Initializes self.qps with an XRC SEND/RECV QPs.
840 :return: None
841 """
842 qp_attr = QPAttr(port_num=self.ib_port)
843 qp_attr.pkey_index = 0
844
845 for _ in range(self.qp_count):
846 attr_ex = QPInitAttrEx(qp_type=ibv_qp_type.IBV_QPT_XRC_RECV,
847 comp_mask=ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_XRCD,
848 xrcd=self.xrcd)
849 qp_attr.qp_access_flags = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | \
850 ibv_access_flags.IBV_ACCESS_REMOTE_READ | \
851 ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \
852 ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC
853 recv_qp = QP(self.ctx, attr_ex, qp_attr)
854 self.rqp_lst.append(recv_qp)
855
856 qp_caps = QPCap(max_send_wr=self.num_msgs, max_recv_sge=0,
857 max_recv_wr=0)
858 attr_ex = QPInitAttrEx(qp_type=ibv_qp_type.IBV_QPT_XRC_SEND, sq_sig_all=1,
859 comp_mask=ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD,
860 pd=self.pd, scq=self.cq, cap=qp_caps)
861 qp_attr.qp_access_flags = 0
862 send_qp =QP(self.ctx, attr_ex, qp_attr)
863 self.sqp_lst.append(send_qp)
864 self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
865 self.psns.append(random.getrandbits(24))
866
867 def create_xrcd(self):
868 """
869 Initializes self.xrcd with an XRC Domain object.
870 :return: None
871 """
872 self.temp_file = tempfile.NamedTemporaryFile()
873 self.xrcd_fd = os.open(self.temp_file.name, os.O_RDONLY | os.O_CREAT,
874 stat.S_IRUSR | stat.S_IRGRP)
875 init = XRCDInitAttr(
876 ibv_xrcd_init_attr_mask.IBV_XRCD_INIT_ATTR_FD | ibv_xrcd_init_attr_mask.IBV_XRCD_INIT_ATTR_OFLAGS,
877 os.O_CREAT, self.xrcd_fd)
878 try:
879 self.xrcd = XRCD(self.ctx, init)
880 except PyverbsRDMAError as ex:
881 if ex.error_code == errno.EOPNOTSUPP:
882 raise unittest.SkipTest('Create XRCD is not supported')
883 raise ex
884
885 def create_srq(self):
886 """
887 Initializes self.srq with a Shared Receive QP object.
888 :return: None
889 """
890 srq_attr = SrqInitAttrEx(max_wr=self.qp_count*self.num_msgs)
891 srq_attr.srq_type = ibv_srq_type.IBV_SRQT_XRC
892 srq_attr.pd = self.pd
893 srq_attr.xrcd = self.xrcd
894 srq_attr.cq = self.cq
895 srq_attr.comp_mask = ibv_srq_init_attr_mask.IBV_SRQ_INIT_ATTR_TYPE | ibv_srq_init_attr_mask.IBV_SRQ_INIT_ATTR_PD | \
896 ibv_srq_init_attr_mask.IBV_SRQ_INIT_ATTR_CQ | ibv_srq_init_attr_mask.IBV_SRQ_INIT_ATTR_XRCD
897 self.srq = SRQ(self.ctx, srq_attr)
898
899 def to_rts(self):
900 gid = self.ctx.query_gid(self.ib_port, self.gid_index)
901 gr = GlobalRoute(dgid=gid, sgid_index=self.gid_index)
902 ah_attr = AHAttr(port_num=self.ib_port, is_global=True,
903 gr=gr, dlid=self.port_attr.lid)
904 qp_attr = QPAttr()
905 qp_attr.max_rd_atomic = MAX_RD_ATOMIC
906 qp_attr.max_dest_rd_atomic = MAX_DEST_RD_ATOMIC
907 qp_attr.path_mtu = PATH_MTU
908 set_rnr_attributes(qp_attr)
909 qp_attr.ah_attr = ah_attr
910 for i in range(self.qp_count):
911 qp_attr.dest_qp_num = self.rqps_num[i][1]
912 qp_attr.rq_psn = self.psns[i]
913 qp_attr.sq_psn = self.rpsns[i]
914 self.rqp_lst[i].to_rts(qp_attr)
915 qp_attr.dest_qp_num = self.rqps_num[i][0]
916 self.sqp_lst[i].to_rts(qp_attr)
917
918 def init_resources(self):
919 self.create_xrcd()
920 super(XRCResources, self).init_resources()
921 self.create_srq()