| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright (c) 2019 Mellanox Technologies, Inc . All rights reserved. See COPYING file |
| 3 |
|
| 4 |
import multiprocessing as mp |
| 5 |
import subprocess |
| 6 |
import functools |
| 7 |
import unittest |
| 8 |
import tempfile |
| 9 |
import random |
| 10 |
import errno |
| 11 |
import stat |
| 12 |
import json |
| 13 |
import sys |
| 14 |
import os |
| 15 |
|
| 16 |
from pyverbs.qp import QPCap, QPInitAttrEx, QPInitAttr, QPAttr, QP |
| 17 |
from pyverbs.srq import SRQ, SrqInitAttrEx, SrqInitAttr, SrqAttr |
| 18 |
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError |
| 19 |
from pyverbs.addr import AHAttr, GlobalRoute |
| 20 |
from pyverbs.xrcd import XRCD, XRCDInitAttr |
| 21 |
from pyverbs.device import Context |
| 22 |
from args_parser import parser |
| 23 |
from pyverbs.librdmacm_enums import rdma_port_space |
| 24 |
import pyverbs.device as d |
| 25 |
from pyverbs.libibverbs_enums import ibv_mtu, ibv_port_state, ibv_gid_type_sysfs, ibv_access_flags,\ |
| 26 |
ibv_qp_type, ibv_qp_init_attr_mask, ibv_xrcd_init_attr_mask,\ |
| 27 |
ibv_srq_type, ibv_srq_init_attr_mask, IBV_LINK_LAYER_ETHERNET |
| 28 |
from pyverbs.pd import PD |
| 29 |
from pyverbs.cq import CQ |
| 30 |
from pyverbs.mr import MR |
| 31 |
|
| 32 |
|
| 33 |
PATH_MTU = ibv_mtu.IBV_MTU_1024 |
| 34 |
MAX_DEST_RD_ATOMIC = 1 |
| 35 |
NUM_OF_PROCESSES = 2 |
| 36 |
MC_IP_PREFIX = '230' |
| 37 |
MAX_RDMA_ATOMIC = 20 |
| 38 |
MAX_RD_ATOMIC = 1 |
| 39 |
MIN_RNR_TIMER =12 |
| 40 |
RETRY_CNT = 7 |
| 41 |
RNR_RETRY = 7 |
| 42 |
TIMEOUT = 14 |
| 43 |
# Devices that don't support RoCEv2 should be added here |
| 44 |
MLNX_VENDOR_ID = 0x02c9 |
| 45 |
CX3_MLNX_PART_ID = 4099 |
| 46 |
CX3Pro_MLNX_PART_ID = 4103 |
| 47 |
DCT_KEY = 0xbadc0de |
| 48 |
# Dictionary: vendor_id -> array of part_ids of devices that lack RoCEv2 support |
| 49 |
ROCEV2_UNSUPPORTED_DEVS = {MLNX_VENDOR_ID: [CX3Pro_MLNX_PART_ID, |
| 50 |
CX3_MLNX_PART_ID]} |
| 51 |
|
| 52 |
|
| 53 |
def has_roce_hw_bug(vendor_id, vendor_part_id): |
| 54 |
return vendor_part_id in ROCEV2_UNSUPPORTED_DEVS.get(vendor_id, []) |
| 55 |
|
| 56 |
|
| 57 |
def set_rnr_attributes(qp_attr): |
| 58 |
""" |
| 59 |
Set default QP RNR attributes. |
| 60 |
:param qp_attr: The QPAttr to set its attributes |
| 61 |
:return: None |
| 62 |
""" |
| 63 |
qp_attr.min_rnr_timer = MIN_RNR_TIMER |
| 64 |
qp_attr.timeout = TIMEOUT |
| 65 |
qp_attr.retry_cnt = RETRY_CNT |
| 66 |
qp_attr.rnr_retry = RNR_RETRY |
| 67 |
|
| 68 |
|
| 69 |
def is_gid_available(gid_index): |
| 70 |
if gid_index is None: |
| 71 |
raise unittest.SkipTest(f'No relevant GID found') |
| 72 |
|
| 73 |
|
| 74 |
class PyverbsAPITestCase(unittest.TestCase): |
| 75 |
def __init__(self, methodName='runTest'): |
| 76 |
super().__init__(methodName) |
| 77 |
# Hold the command line arguments |
| 78 |
self.config = parser.get_config() |
| 79 |
self.dev_name = None |
| 80 |
self.ctx = None |
| 81 |
self.attr = None |
| 82 |
self.attr_ex = None |
| 83 |
self.gid_index = 0 |
| 84 |
self.pre_environment = {} |
| 85 |
|
| 86 |
def setUp(self): |
| 87 |
""" |
| 88 |
Opens the device and queries it. |
| 89 |
The results of the query and query_ex are stored in attr and attr_ex |
| 90 |
instance attributes respectively. |
| 91 |
If the user didn't pass a device name, the first device is chosen by |
| 92 |
default. |
| 93 |
""" |
| 94 |
self.ib_port = self.config['port'] |
| 95 |
self.dev_name = self.config['dev'] |
| 96 |
if not self.dev_name: |
| 97 |
dev_list = d.get_device_list() |
| 98 |
if not dev_list: |
| 99 |
raise unittest.SkipTest('No IB devices found') |
| 100 |
self.dev_name = dev_list[0].name.decode() |
| 101 |
|
| 102 |
if self.config['gid']: |
| 103 |
self.gid_index = self.config['gid'] |
| 104 |
|
| 105 |
self.create_context() |
| 106 |
self.attr = self.ctx.query_device() |
| 107 |
self.attr_ex = self.ctx.query_device_ex() |
| 108 |
|
| 109 |
def create_context(self): |
| 110 |
self.ctx = d.Context(name=self.dev_name) |
| 111 |
|
| 112 |
def set_env_variable(self, var, value): |
| 113 |
""" |
| 114 |
Set environment variable. The current value for each variable is stored |
| 115 |
and is set back at the end of the test. |
| 116 |
:param var: The name of the environment variable |
| 117 |
:param value: The requested new value of this environment variable |
| 118 |
""" |
| 119 |
if var not in self.pre_environment.keys(): |
| 120 |
self.pre_environment[var] = os.environ.get(var) |
| 121 |
os.environ[var] = value |
| 122 |
|
| 123 |
def tearDown(self): |
| 124 |
for k, v in self.pre_environment.items(): |
| 125 |
if v is None: |
| 126 |
os.environ.pop(k) |
| 127 |
else: |
| 128 |
os.environ[k] = v |
| 129 |
self.ctx.close() |
| 130 |
|
| 131 |
|
| 132 |
class RDMATestCase(unittest.TestCase): |
| 133 |
ZERO_GID = '0000:0000:0000:0000' |
| 134 |
|
| 135 |
def __init__(self, methodName='runTest', dev_name=None, ib_port=None, |
| 136 |
gid_index=None, pkey_index=None, gid_type=None): |
| 137 |
""" |
| 138 |
Initialize a RDMA test unit based on unittest.TestCase. |
| 139 |
If no device was provided, it iterates over the existing devices, for |
| 140 |
each port of each device, it checks which GID indexes are valid (in RoCE, |
| 141 |
only IPv4 and IPv6 based GIDs are used). Each <dev, port, gid> is added |
| 142 |
to an array and one entry is selected. |
| 143 |
If a device was provided, the same process is done for all ports of this |
| 144 |
device (in case they're not provided), and so on. |
| 145 |
If gid_type is provided by the user, only GIDs of that type would be |
| 146 |
be chosen (valid only if gid_index was not provided). |
| 147 |
:param methodName: The base method to be used by the unittest |
| 148 |
:param dev_name: Device name to use |
| 149 |
:param ib_port: IB port of the device to use |
| 150 |
:param gid_index: GID index to use |
| 151 |
:param pkey_index: PKEY index to use |
| 152 |
:param gid_type: If provided, only GIDs of gid_type will be chosen |
| 153 |
(ignored if gid_index is provided by the user) |
| 154 |
""" |
| 155 |
super(RDMATestCase, self).__init__(methodName) |
| 156 |
# Hold the command line arguments |
| 157 |
self.config = parser.get_config() |
| 158 |
dev = self.config['dev'] |
| 159 |
self.dev_name = dev_name if dev_name else dev |
| 160 |
self.ib_port = ib_port if ib_port else self.config['port'] |
| 161 |
self.gid_index = gid_index if gid_index else self.config['gid'] |
| 162 |
self.pkey_index = pkey_index |
| 163 |
self.gid_type = gid_type if gid_index is None else None |
| 164 |
self.ip_addr = None |
| 165 |
self.mac_addr = None |
| 166 |
self.pre_environment = {} |
| 167 |
self.server = None |
| 168 |
self.client = None |
| 169 |
self.iters = 10 |
| 170 |
|
| 171 |
|
| 172 |
def is_eth_and_has_roce_hw_bug(self): |
| 173 |
""" |
| 174 |
Check if the link layer is Ethernet and the device lacks RoCEv2 support |
| 175 |
with a known HW bug. |
| 176 |
return: True if the link layer is Ethernet and device is not supported |
| 177 |
""" |
| 178 |
ctx = d.Context(name=self.dev_name) |
| 179 |
port_attrs = ctx.query_port(self.ib_port) |
| 180 |
dev_attrs = ctx.query_device() |
| 181 |
vendor_id = dev_attrs.vendor_id |
| 182 |
vendor_pid = dev_attrs.vendor_part_id |
| 183 |
return port_attrs.link_layer == IBV_LINK_LAYER_ETHERNET and \ |
| 184 |
has_roce_hw_bug(vendor_id, vendor_pid) |
| 185 |
|
| 186 |
@staticmethod |
| 187 |
def get_net_name(dev, port=None): |
| 188 |
if port is not None: |
| 189 |
out = subprocess.check_output(['rdma', 'link', 'show', '-j']) |
| 190 |
loaded_json = json.loads(out.decode()) |
| 191 |
for row in loaded_json: |
| 192 |
try: |
| 193 |
if row['ifname'] == dev and row['port'] == port: |
| 194 |
return row['netdev'] |
| 195 |
except KeyError: |
| 196 |
pass |
| 197 |
|
| 198 |
if not os.path.exists(f'/sys/class/infiniband/{dev}/device/net/'): |
| 199 |
return None |
| 200 |
|
| 201 |
out = subprocess.check_output(['ls', f'/sys/class/infiniband/{dev}/device/net/']) |
| 202 |
return out.decode().split('\n')[0] |
| 203 |
|
| 204 |
@staticmethod |
| 205 |
def get_ip_mac_address(ifname): |
| 206 |
out = subprocess.check_output(['ip', '-j', 'addr', 'show', ifname]) |
| 207 |
loaded_json = json.loads(out.decode()) |
| 208 |
interface = loaded_json[0]['addr_info'][0]['local'] |
| 209 |
mac = loaded_json[0]['address'] |
| 210 |
if 'fe80::' in interface: |
| 211 |
interface = interface + '%' + ifname |
| 212 |
return interface, mac |
| 213 |
|
| 214 |
def setUp(self): |
| 215 |
""" |
| 216 |
Verify that the test case has dev_name, ib_port, gid_index and pkey index. |
| 217 |
If not provided by the user, the first valid combination will be used. |
| 218 |
""" |
| 219 |
if self.pkey_index is None: |
| 220 |
# To avoid iterating the entire pkeys table, if a pkey index wasn't |
| 221 |
# provided, use index 0 which is always valid |
| 222 |
self.pkey_index = 0 |
| 223 |
|
| 224 |
self.args = [] |
| 225 |
if self.dev_name is not None: |
| 226 |
ctx = d.Context(name=self.dev_name) |
| 227 |
if self.ib_port is not None: |
| 228 |
if self.gid_index is not None: |
| 229 |
self._get_ip_mac(self.dev_name, self.ib_port, self.gid_index) |
| 230 |
else: |
| 231 |
# Add avaiable GIDs of the given dev_name + port |
| 232 |
self._add_gids_per_port(ctx, self.dev_name, self.ib_port) |
| 233 |
else: |
| 234 |
# Add available GIDs for each port of the given dev_name |
| 235 |
self._add_gids_per_device(ctx, self.dev_name) |
| 236 |
else: |
| 237 |
# Iterate available devices, add available GIDs for each of |
| 238 |
# their ports |
| 239 |
lst = d.get_device_list() |
| 240 |
for dev in lst: |
| 241 |
dev_name = dev.name.decode() |
| 242 |
ctx = d.Context(name=dev_name) |
| 243 |
self._add_gids_per_device(ctx, dev_name) |
| 244 |
|
| 245 |
if not self.args: |
| 246 |
raise unittest.SkipTest('No supported port is up, can\'t run traffic') |
| 247 |
# Choose one combination and use it |
| 248 |
self._select_config() |
| 249 |
self.dev_info = {'dev_name': self.dev_name, 'ib_port': self.ib_port, |
| 250 |
'gid_index': self.gid_index} |
| 251 |
|
| 252 |
def _add_gids_per_port(self, ctx, dev, port): |
| 253 |
# Don't add ports which are not active |
| 254 |
port_attrs = ctx.query_port(port) |
| 255 |
if port_attrs.state != ibv_port_state.IBV_PORT_ACTIVE: |
| 256 |
return |
| 257 |
if not port_attrs.gid_tbl_len: |
| 258 |
self._get_ip_mac(dev, port, None) |
| 259 |
return |
| 260 |
dev_attrs = ctx.query_device() |
| 261 |
vendor_id = dev_attrs.vendor_id |
| 262 |
vendor_pid = dev_attrs.vendor_part_id |
| 263 |
for idx in range(port_attrs.gid_tbl_len): |
| 264 |
gid = ctx.query_gid(port, idx) |
| 265 |
# Avoid adding ZERO GIDs |
| 266 |
if gid.gid[-19:] == self.ZERO_GID: |
| 267 |
continue |
| 268 |
# Avoid RoCEv2 GIDs on unsupported devices |
| 269 |
if port_attrs.link_layer == IBV_LINK_LAYER_ETHERNET and \ |
| 270 |
ctx.query_gid_type(port, idx) == \ |
| 271 |
ibv_gid_type_sysfs.IBV_GID_TYPE_SYSFS_ROCE_V2 and \ |
| 272 |
has_roce_hw_bug(vendor_id, vendor_pid): |
| 273 |
continue |
| 274 |
if self.gid_type is not None and ctx.query_gid_type(port, idx) != \ |
| 275 |
self.gid_type: |
| 276 |
continue |
| 277 |
self._get_ip_mac(dev, port, idx) |
| 278 |
|
| 279 |
def _add_gids_per_device(self, ctx, dev): |
| 280 |
self._add_gids_per_port(ctx, dev, self.ib_port) |
| 281 |
|
| 282 |
def _get_ip_mac(self, dev, port, idx): |
| 283 |
net_name = self.get_net_name(dev, port) |
| 284 |
if net_name is None: |
| 285 |
self.args.append([dev, port, idx, None, None]) |
| 286 |
return |
| 287 |
try: |
| 288 |
ip_addr, mac_addr = self.get_ip_mac_address(net_name) |
| 289 |
except (KeyError, IndexError): |
| 290 |
self.args.append([dev, port, idx, None, None]) |
| 291 |
else: |
| 292 |
self.args.append([dev, port, idx, ip_addr, mac_addr]) |
| 293 |
|
| 294 |
def _select_config(self): |
| 295 |
args_with_inet_ip = [] |
| 296 |
for arg in self.args: |
| 297 |
if arg[3]: |
| 298 |
args_with_inet_ip.append(arg) |
| 299 |
if args_with_inet_ip: |
| 300 |
args = args_with_inet_ip[0] |
| 301 |
else: |
| 302 |
args = self.args[0] |
| 303 |
self.dev_name = args[0] |
| 304 |
self.ib_port = args[1] |
| 305 |
self.gid_index = args[2] |
| 306 |
self.ip_addr = args[3] |
| 307 |
self.mac_addr = args[4] |
| 308 |
|
| 309 |
def set_env_variable(self, var, value): |
| 310 |
""" |
| 311 |
Set environment variable. The current value for each variable is stored |
| 312 |
and is set back at the end of the test. |
| 313 |
:param var: The name of the environment variable |
| 314 |
:param value: The requested new value of this environment variable |
| 315 |
""" |
| 316 |
if var not in self.pre_environment.keys(): |
| 317 |
self.pre_environment[var] = os.environ.get(var) |
| 318 |
os.environ[var] = value |
| 319 |
|
| 320 |
def sync_remote_attr(self): |
| 321 |
""" |
| 322 |
Sync the MR remote attributes between the server and the client. |
| 323 |
""" |
| 324 |
self.server.rkey = self.client.mr.rkey |
| 325 |
self.server.raddr = self.client.mr.buf |
| 326 |
self.client.rkey = self.server.mr.rkey |
| 327 |
self.client.raddr = self.server.mr.buf |
| 328 |
|
| 329 |
def pre_run(self): |
| 330 |
""" |
| 331 |
Configure Resources before running traffic. |
| 332 |
pre_run() must be implemented by the client and server. |
| 333 |
""" |
| 334 |
self.client.pre_run(self.server.psns, self.server.qps_num) |
| 335 |
self.server.pre_run(self.client.psns, self.client.qps_num) |
| 336 |
|
| 337 |
def create_players(self, resource, sync_attrs=True, **resource_arg): |
| 338 |
""" |
| 339 |
Init test resources. |
| 340 |
:param resource: The RDMA resources to use. |
| 341 |
:param sync_attrs: If True, sync remote attrs such as rkey and raddr |
| 342 |
:param resource_arg: Dict of args that specify the resource specific |
| 343 |
attributes. |
| 344 |
""" |
| 345 |
try: |
| 346 |
self.client = resource(**self.dev_info, **resource_arg) |
| 347 |
self.server = resource(**self.dev_info, **resource_arg) |
| 348 |
except PyverbsRDMAError as ex: |
| 349 |
if ex.error_code == errno.EOPNOTSUPP: |
| 350 |
raise unittest.SkipTest(f'Create player of {resource.__name__} is not supported') |
| 351 |
raise ex |
| 352 |
self.pre_run() |
| 353 |
if sync_attrs: |
| 354 |
self.sync_remote_attr() |
| 355 |
self.traffic_args = {'client': self.client, 'server': self.server, |
| 356 |
'iters': self.iters, 'gid_idx': self.gid_index, |
| 357 |
'port': self.ib_port} |
| 358 |
|
| 359 |
def tearDown(self): |
| 360 |
""" |
| 361 |
Restore the previous environment variables values before ending the test. |
| 362 |
""" |
| 363 |
for k, v in self.pre_environment.items(): |
| 364 |
if v is None: |
| 365 |
os.environ.pop(k) |
| 366 |
else: |
| 367 |
os.environ[k] = v |
| 368 |
if self.server: |
| 369 |
self.server.ctx.close() |
| 370 |
if self.client: |
| 371 |
self.client.ctx.close() |
| 372 |
super().tearDown() |
| 373 |
|
| 374 |
|
| 375 |
class RDMACMBaseTest(RDMATestCase): |
| 376 |
""" |
| 377 |
Base RDMACM test class. |
| 378 |
This class does not include any test, but rather implements generic |
| 379 |
connection and traffic methods that are needed by RDMACM tests in general. |
| 380 |
Each RDMACM test should have a class that inherits this class and extends |
| 381 |
its functionalities if needed. |
| 382 |
""" |
| 383 |
def setUp(self): |
| 384 |
super().setUp() |
| 385 |
if not self.ip_addr: |
| 386 |
raise unittest.SkipTest('Device {} doesn\'t have net interface' |
| 387 |
.format(self.dev_name)) |
| 388 |
is_gid_available(self.gid_index) |
| 389 |
|
| 390 |
def two_nodes_rdmacm_traffic(self, connection_resources, test_flow, bad_flow=False, |
| 391 |
**resource_kwargs): |
| 392 |
""" |
| 393 |
Init and manage the rdmacm test processes. The exit code of the |
| 394 |
test processes indicates if exception was thrown. |
| 395 |
{0: pass, 2: exception was thrown, 5: skip test} |
| 396 |
If needed, terminate those processes and raise an exception. |
| 397 |
:param connection_resources: The CMConnection resources to use. |
| 398 |
:param test_flow: The target RDMACM flow method to run. |
| 399 |
:param bad_flow: If true, traffic is expected to fail. |
| 400 |
:param resource_kwargs: Dict of args that specify the CMResources |
| 401 |
specific attributes. Each test case can pass |
| 402 |
here as key words the specific CMResources |
| 403 |
attributes that are requested. |
| 404 |
:return: None |
| 405 |
""" |
| 406 |
if resource_kwargs.get('port_space', None) == rdma_port_space.RDMA_PS_UDP and \ |
| 407 |
self.is_eth_and_has_roce_hw_bug(): |
| 408 |
raise unittest.SkipTest('Device {} doesn\'t support UDP with RoCEv2' |
| 409 |
.format(self.dev_name)) |
| 410 |
ctx = mp.get_context('fork') |
| 411 |
self.syncer = ctx.Barrier(NUM_OF_PROCESSES, timeout=15) |
| 412 |
self.notifier = ctx.Queue() |
| 413 |
passive = ctx.Process(target=test_flow, |
| 414 |
kwargs={'connection_resources': connection_resources, |
| 415 |
'passive':True, **resource_kwargs}) |
| 416 |
active = ctx.Process(target=test_flow, |
| 417 |
kwargs={'connection_resources': connection_resources, |
| 418 |
'passive':False, **resource_kwargs}) |
| 419 |
passive.start() |
| 420 |
active.start() |
| 421 |
repeat_times=150 if not bad_flow else 3 |
| 422 |
proc_res = {} |
| 423 |
for _ in range(repeat_times): |
| 424 |
for proc in [passive, active]: |
| 425 |
proc.join(0.1) |
| 426 |
# Write the exit code of the proc. |
| 427 |
if not proc.is_alive(): |
| 428 |
side = 'passive' if proc == passive else 'active' |
| 429 |
if side not in proc_res.keys(): |
| 430 |
proc_res[side] = proc.exitcode |
| 431 |
# If the processes is still alive kill them and fail the test. |
| 432 |
proc_killed = False |
| 433 |
for proc in [passive, active]: |
| 434 |
if proc.is_alive(): |
| 435 |
proc.terminate() |
| 436 |
proc_killed = True |
| 437 |
# Check if need to skip this test |
| 438 |
for side in proc_res.keys(): |
| 439 |
if proc_res[side] == 5: |
| 440 |
raise unittest.SkipTest(f'SkipTest occurred on {side} side') |
| 441 |
# Raise exeption if the test proceses was terminate. |
| 442 |
if bad_flow and not proc_killed: |
| 443 |
raise Exception('Bad flow: traffic passed which is not expected') |
| 444 |
if not bad_flow and proc_killed: |
| 445 |
raise Exception('RDMA CM test procces is stuck, kill the test') |
| 446 |
|
| 447 |
# Check if the test processes raise exceptions. |
| 448 |
for side in proc_res: |
| 449 |
if proc_res[side] != 0: |
| 450 |
raise Exception(f'Exception in {side} side occurred') |
| 451 |
|
| 452 |
def rdmacm_traffic(self, connection_resources=None, passive=None, **kwargs): |
| 453 |
""" |
| 454 |
Run RDMACM traffic between two CMIDs. |
| 455 |
:param connection_resources: The connection resources to use. |
| 456 |
:param passive: Indicate if this CMID is this the passive side. |
| 457 |
:param kwargs: Arguments to be passed to the connection_resources. |
| 458 |
:return: None |
| 459 |
""" |
| 460 |
try: |
| 461 |
player = connection_resources(ip_addr=self.ip_addr, |
| 462 |
syncer=self.syncer, |
| 463 |
notifier=self.notifier, |
| 464 |
passive=passive, **kwargs) |
| 465 |
player.establish_connection() |
| 466 |
if kwargs.get('reject_conn'): |
| 467 |
return |
| 468 |
player.rdmacm_traffic() |
| 469 |
player.disconnect() |
| 470 |
except Exception as ex: |
| 471 |
self._rdmacm_exception_handler(passive, ex) |
| 472 |
|
| 473 |
def rdmacm_multicast_traffic(self, connection_resources=None, passive=None, |
| 474 |
extended=False, leave_test=False, **kwargs): |
| 475 |
""" |
| 476 |
Run RDMACM multicast traffic between two CMIDs. |
| 477 |
:param connection_resources: The connection resources to use. |
| 478 |
:param passive: Indicate if this CMID is the passive side. |
| 479 |
:param extended: Use exteneded multicast join request. This request |
| 480 |
allows CMID to join with specific join flags. |
| 481 |
:param leave_test: Perform traffic after leaving the multicast group to |
| 482 |
ensure leave works. |
| 483 |
:param kwargs: Arguments to be passed to the connection_resources. |
| 484 |
:return: None |
| 485 |
""" |
| 486 |
try: |
| 487 |
player = connection_resources(ip_addr=self.ip_addr, syncer=self.syncer, |
| 488 |
notifier=self.notifier, passive=False, |
| 489 |
**kwargs) |
| 490 |
mc_addr = MC_IP_PREFIX + self.ip_addr[self.ip_addr.find('.'):] |
| 491 |
player.join_to_multicast(src_addr=self.ip_addr, mc_addr=mc_addr, |
| 492 |
extended=extended) |
| 493 |
player.rdmacm_traffic(server=passive, multicast=True) |
| 494 |
player.leave_multicast(mc_addr=mc_addr) |
| 495 |
if leave_test: |
| 496 |
player.rdmacm_traffic(server=passive, multicast=True) |
| 497 |
except Exception as ex: |
| 498 |
self._rdmacm_exception_handler(passive, ex) |
| 499 |
|
| 500 |
def rdmacm_remote_traffic(self, connection_resources=None, passive=None, |
| 501 |
remote_op='write', **kwargs): |
| 502 |
""" |
| 503 |
Run RDMACM remote traffic between two CMIDs. |
| 504 |
:param connection_resources: The connection resources to use. |
| 505 |
:param passive: Indicate if this CMID is the passive side. |
| 506 |
:param remote_op: The remote operation in the traffic. |
| 507 |
:param kwargs: Arguments to be passed to the connection_resources. |
| 508 |
:return: None |
| 509 |
""" |
| 510 |
try: |
| 511 |
player = connection_resources(ip_addr=self.ip_addr, |
| 512 |
syncer=self.syncer, |
| 513 |
notifier=self.notifier, |
| 514 |
passive=passive, |
| 515 |
remote_op=remote_op, **kwargs) |
| 516 |
player.establish_connection() |
| 517 |
player.remote_traffic(passive=passive, remote_op=remote_op) |
| 518 |
player.disconnect() |
| 519 |
except Exception as ex: |
| 520 |
self._rdmacm_exception_handler(passive, ex) |
| 521 |
|
| 522 |
@staticmethod |
| 523 |
def _rdmacm_exception_handler(passive, exception): |
| 524 |
if isinstance(exception, PyverbsRDMAError): |
| 525 |
if exception.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]: |
| 526 |
sys.exit(5) |
| 527 |
if isinstance(exception, unittest.case.SkipTest): |
| 528 |
sys.exit(5) |
| 529 |
side = 'passive' if passive else 'active' |
| 530 |
print(f'Player {side} got: {exception}') |
| 531 |
sys.exit(2) |
| 532 |
|
| 533 |
|
| 534 |
def catch_skiptest(func): |
| 535 |
""" |
| 536 |
Decorator to catch unittest.SkipTest in __init__ resource functions. |
| 537 |
It gracefully closes the context and all of its underlying resources. |
| 538 |
""" |
| 539 |
@functools.wraps(func) |
| 540 |
def wrapper(self, *args, **kwargs): |
| 541 |
try: |
| 542 |
func(self, *args, **kwargs) |
| 543 |
except unittest.SkipTest as e: |
| 544 |
if hasattr(self, 'ctx') and self.ctx: |
| 545 |
self.ctx.close() |
| 546 |
raise e |
| 547 |
return wrapper |
| 548 |
|
| 549 |
|
| 550 |
class SkipTestMeta(type): |
| 551 |
""" |
| 552 |
Metaclass to automatically wrap __init__ in catch_skiptest. |
| 553 |
It should only be used in resource classes, such as those inheriting from |
| 554 |
BaseResources. |
| 555 |
""" |
| 556 |
def __new__(cls, name, bases, dct): |
| 557 |
if "__init__" in dct: |
| 558 |
dct["__init__"] = catch_skiptest(dct["__init__"]) |
| 559 |
return super().__new__(cls, name, bases, dct) |
| 560 |
|
| 561 |
|
| 562 |
class BaseResources(object, metaclass=SkipTestMeta): |
| 563 |
""" |
| 564 |
BaseResources class is a base aggregator object which contains basic |
| 565 |
resources like Context and PD. It opens a context over the given device |
| 566 |
and port and allocates a PD. |
| 567 |
""" |
| 568 |
def __init__(self, dev_name, ib_port, gid_index): |
| 569 |
""" |
| 570 |
Initializes a BaseResources object. |
| 571 |
:param dev_name: Device name to be used (default: 'ibp0s8f0') |
| 572 |
:param ib_port: IB port of the device to use (default: 1) |
| 573 |
:param gid_index: Which GID index to use (default: 0) |
| 574 |
""" |
| 575 |
self.dev_name = dev_name |
| 576 |
self.gid_index = gid_index |
| 577 |
self.ib_port = ib_port |
| 578 |
self.ctx = None |
| 579 |
self.create_context() |
| 580 |
self.create_pd() |
| 581 |
|
| 582 |
def create_context(self): |
| 583 |
self.ctx = Context(name=self.dev_name) |
| 584 |
|
| 585 |
def create_pd(self): |
| 586 |
self.pd = PD(self.ctx) |
| 587 |
|
| 588 |
def mem_write(self, data, size, offset=0): |
| 589 |
self.mr.write(data, size, offset) |
| 590 |
|
| 591 |
def mem_read(self, size=None, offset=0): |
| 592 |
size_ = self.msg_size if size is None else size |
| 593 |
return self.mr.read(size_, offset) |
| 594 |
|
| 595 |
|
| 596 |
class TrafficResources(BaseResources): |
| 597 |
""" |
| 598 |
Basic traffic class. It provides the basic RDMA resources and operations |
| 599 |
needed for traffic. |
| 600 |
""" |
| 601 |
def __init__(self, dev_name, ib_port, gid_index, with_srq=False, |
| 602 |
qp_count=1, msg_size=1024): |
| 603 |
""" |
| 604 |
Initializes a TrafficResources object with the given values and creates |
| 605 |
basic RDMA resources. |
| 606 |
:param dev_name: Device name to be used |
| 607 |
:param ib_port: IB port of the device to use |
| 608 |
:param gid_index: Which GID index to use |
| 609 |
:param with_srq: If True, create SRQ and attach to QPs |
| 610 |
:param qp_count: Number of QPs to create |
| 611 |
:param msg_size: Size of resource msg. If None, use 1024 as default. |
| 612 |
""" |
| 613 |
super(TrafficResources, self).__init__(dev_name=dev_name, |
| 614 |
ib_port=ib_port, |
| 615 |
gid_index=gid_index) |
| 616 |
self.msg_size = msg_size |
| 617 |
self.num_msgs = 1000 |
| 618 |
self.port_attr = None |
| 619 |
self.mr = None |
| 620 |
self.use_mr_prefetch = None |
| 621 |
self.srq = None |
| 622 |
self.cq = None |
| 623 |
self.qps = [] |
| 624 |
self.qps_num = [] |
| 625 |
self.psns = [] |
| 626 |
self.rqps_num = None |
| 627 |
self.rpsns = None |
| 628 |
self.with_srq = with_srq |
| 629 |
self.qp_count = qp_count |
| 630 |
self.init_resources() |
| 631 |
|
| 632 |
@property |
| 633 |
def qp(self): |
| 634 |
return self.qps[0] |
| 635 |
|
| 636 |
@property |
| 637 |
def mr_lkey(self): |
| 638 |
if self.mr: |
| 639 |
return self.mr.lkey |
| 640 |
|
| 641 |
def init_resources(self): |
| 642 |
""" |
| 643 |
Initializes a CQ, MR and an RC QP. |
| 644 |
:return: None |
| 645 |
""" |
| 646 |
self.port_attr = self.ctx.query_port(self.ib_port) |
| 647 |
self.create_cq() |
| 648 |
if self.with_srq: |
| 649 |
self.create_srq() |
| 650 |
self.create_mr() |
| 651 |
self.create_qps() |
| 652 |
|
| 653 |
def create_cq(self): |
| 654 |
""" |
| 655 |
Initializes self.cq with a CQ of depth <num_msgs> - defined by each |
| 656 |
test. |
| 657 |
:return: None |
| 658 |
""" |
| 659 |
self.cq = CQ(self.ctx, self.num_msgs, None, None, 0) |
| 660 |
|
| 661 |
def create_mr(self): |
| 662 |
""" |
| 663 |
Initializes self.mr with an MR of length <msg_size> - defined by each |
| 664 |
test. |
| 665 |
:return: None |
| 666 |
""" |
| 667 |
self.mr = MR(self.pd, self.msg_size, ibv_access_flags.IBV_ACCESS_LOCAL_WRITE) |
| 668 |
|
| 669 |
def create_qp_cap(self): |
| 670 |
return QPCap(max_recv_wr=self.num_msgs) |
| 671 |
|
| 672 |
def create_qp_init_attr(self): |
| 673 |
return QPInitAttr(qp_type=ibv_qp_type.IBV_QPT_RC, scq=self.cq, rcq=self.cq, |
| 674 |
srq=self.srq, cap=self.create_qp_cap()) |
| 675 |
|
| 676 |
def create_qp_attr(self): |
| 677 |
return QPAttr(port_num=self.ib_port) |
| 678 |
|
| 679 |
def create_qps(self): |
| 680 |
""" |
| 681 |
Initializes self.qps with RC QPs. |
| 682 |
:return: None |
| 683 |
""" |
| 684 |
qp_init_attr = self.create_qp_init_attr() |
| 685 |
qp_attr = self.create_qp_attr() |
| 686 |
for _ in range(self.qp_count): |
| 687 |
try: |
| 688 |
qp = QP(self.pd, qp_init_attr, qp_attr) |
| 689 |
self.qps.append(qp) |
| 690 |
self.qps_num.append(qp.qp_num) |
| 691 |
self.psns.append(random.getrandbits(24)) |
| 692 |
except PyverbsRDMAError as ex: |
| 693 |
if ex.error_code == errno.EOPNOTSUPP: |
| 694 |
raise unittest.SkipTest(f'Create QP type {qp_init_attr.qp_type} is not supported') |
| 695 |
raise ex |
| 696 |
|
| 697 |
def create_srq_attr(self): |
| 698 |
return SrqAttr(max_wr=self.num_msgs*self.qp_count) |
| 699 |
|
| 700 |
def create_srq_init_attr(self): |
| 701 |
return SrqInitAttr(self.create_srq_attr()) |
| 702 |
|
| 703 |
def create_srq(self): |
| 704 |
srq_init_attr = self.create_srq_init_attr() |
| 705 |
try: |
| 706 |
self.srq = SRQ(self.pd, srq_init_attr) |
| 707 |
except PyverbsRDMAError as ex: |
| 708 |
if ex.error_code == errno.EOPNOTSUPP: |
| 709 |
raise unittest.SkipTest('Create SRQ is not supported') |
| 710 |
raise ex |
| 711 |
|
| 712 |
def pre_run(self, rpsns, rqps_num): |
| 713 |
""" |
| 714 |
Configure resources before running traffic and modifies the QP to RTS |
| 715 |
if required. |
| 716 |
:param rpsns: Remote PSNs (packet serial numbers) |
| 717 |
:param rqps_num: Remote QPs Number |
| 718 |
""" |
| 719 |
self.rpsns = rpsns |
| 720 |
self.rqps_num = rqps_num |
| 721 |
self.to_rts() |
| 722 |
|
| 723 |
def to_rts(self): |
| 724 |
""" |
| 725 |
Modify the QP's states to RTS and initialize it to be ready for traffic. |
| 726 |
If not required, can be "passed" but must be implemented. |
| 727 |
""" |
| 728 |
raise NotImplementedError() |
| 729 |
|
| 730 |
|
| 731 |
class RoCETrafficResources(TrafficResources): |
| 732 |
def __init__(self, dev_name, ib_port, gid_index, **kwargs): |
| 733 |
is_gid_available(gid_index) |
| 734 |
super(RoCETrafficResources, self).__init__(dev_name, ib_port, gid_index, **kwargs) |
| 735 |
|
| 736 |
|
| 737 |
class RCResources(RoCETrafficResources): |
| 738 |
def __init__(self, dev_name, ib_port, gid_index, |
| 739 |
max_dest_rd_atomic=MAX_DEST_RD_ATOMIC, max_rd_atomic=MAX_RD_ATOMIC, **kwargs): |
| 740 |
""" |
| 741 |
Initializes a TrafficResources object with the given values and creates |
| 742 |
basic RDMA resources. |
| 743 |
:param dev_name: Device name to be used |
| 744 |
:param ib_port: IB port of the device to use |
| 745 |
:param gid_index: Which GID index to use |
| 746 |
:param max_dest_rd_atomic: Number of responder incoming rd_atomic operations that can |
| 747 |
be handled |
| 748 |
:param max_rd_atomic: Number of outstanding destination rd_atomic operations |
| 749 |
""" |
| 750 |
self.max_dest_rd_atomic = max_dest_rd_atomic |
| 751 |
self.max_rd_atomic = max_rd_atomic |
| 752 |
super().__init__(dev_name, ib_port, gid_index, **kwargs) |
| 753 |
|
| 754 |
def to_rts(self): |
| 755 |
""" |
| 756 |
Set the QP attributes' values to arbitrary values (same values used in |
| 757 |
ibv_rc_pingpong). |
| 758 |
:return: None |
| 759 |
""" |
| 760 |
attr = self.create_qp_attr() |
| 761 |
attr.path_mtu = PATH_MTU |
| 762 |
attr.max_dest_rd_atomic = self.max_dest_rd_atomic |
| 763 |
set_rnr_attributes(attr) |
| 764 |
attr.max_rd_atomic = self.max_rd_atomic |
| 765 |
gr = GlobalRoute(dgid=self.ctx.query_gid(self.ib_port, self.gid_index), |
| 766 |
sgid_index=self.gid_index) |
| 767 |
ah_attr = AHAttr(port_num=self.ib_port, is_global=1, gr=gr, |
| 768 |
dlid=self.port_attr.lid) |
| 769 |
attr.ah_attr = ah_attr |
| 770 |
for i in range(self.qp_count): |
| 771 |
attr.dest_qp_num = self.rqps_num[i] |
| 772 |
attr.rq_psn = self.psns[i] |
| 773 |
attr.sq_psn = self.rpsns[i] |
| 774 |
self.qps[i].to_rts(attr) |
| 775 |
|
| 776 |
|
| 777 |
class UDResources(RoCETrafficResources): |
| 778 |
UD_QKEY = 0x11111111 |
| 779 |
UD_PKEY_INDEX = 0 |
| 780 |
GRH_SIZE = 40 |
| 781 |
|
| 782 |
def create_mr(self): |
| 783 |
self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE, |
| 784 |
ibv_access_flags.IBV_ACCESS_LOCAL_WRITE) |
| 785 |
|
| 786 |
def create_qp_init_attr(self): |
| 787 |
return QPInitAttr(qp_type=ibv_qp_type.IBV_QPT_UD, scq=self.cq, |
| 788 |
rcq=self.cq, srq=self.srq, cap=self.create_qp_cap()) |
| 789 |
|
| 790 |
def create_qps(self): |
| 791 |
qp_init_attr = self.create_qp_init_attr() |
| 792 |
qp_attr = self.create_qp_attr() |
| 793 |
qp_attr.qkey = self.UD_QKEY |
| 794 |
qp_attr.pkey_index = self.UD_PKEY_INDEX |
| 795 |
for _ in range(self.qp_count): |
| 796 |
try: |
| 797 |
qp = QP(self.pd, qp_init_attr, qp_attr) |
| 798 |
self.qps.append(qp) |
| 799 |
self.qps_num.append(qp.qp_num) |
| 800 |
self.psns.append(random.getrandbits(24)) |
| 801 |
except PyverbsRDMAError as ex: |
| 802 |
if ex.error_code == errno.EOPNOTSUPP: |
| 803 |
raise unittest.SkipTest(f'Create QP type {qp_init_attr.qp_type} is not supported') |
| 804 |
raise ex |
| 805 |
|
| 806 |
def to_rts(self): |
| 807 |
pass |
| 808 |
|
| 809 |
|
| 810 |
class RawResources(TrafficResources): |
| 811 |
def create_qp_init_attr(self): |
| 812 |
return QPInitAttr(qp_type=ibv_qp_type.IBV_QPT_RAW_PACKET, scq=self.cq, |
| 813 |
rcq=self.cq, srq=self.srq, cap=self.create_qp_cap()) |
| 814 |
|
| 815 |
def pre_run(self, rpsns=None, rqps_num=None): |
| 816 |
pass |
| 817 |
|
| 818 |
|
| 819 |
class XRCResources(RoCETrafficResources): |
| 820 |
def __init__(self, dev_name, ib_port, gid_index, qp_count=2): |
| 821 |
self.temp_file = None |
| 822 |
self.xrcd_fd = -1 |
| 823 |
self.xrcd = None |
| 824 |
self.sqp_lst = [] |
| 825 |
self.rqp_lst = [] |
| 826 |
super(XRCResources, self).__init__(dev_name, ib_port, gid_index, |
| 827 |
qp_count=qp_count) |
| 828 |
|
| 829 |
def close(self): |
| 830 |
os.close(self.xrcd_fd) |
| 831 |
self.temp_file.close() |
| 832 |
|
| 833 |
@property |
| 834 |
def qp(self): |
| 835 |
return self.sqp_lst[0] |
| 836 |
|
| 837 |
def create_qps(self): |
| 838 |
""" |
| 839 |
Initializes self.qps with an XRC SEND/RECV QPs. |
| 840 |
:return: None |
| 841 |
""" |
| 842 |
qp_attr = QPAttr(port_num=self.ib_port) |
| 843 |
qp_attr.pkey_index = 0 |
| 844 |
|
| 845 |
for _ in range(self.qp_count): |
| 846 |
attr_ex = QPInitAttrEx(qp_type=ibv_qp_type.IBV_QPT_XRC_RECV, |
| 847 |
comp_mask=ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_XRCD, |
| 848 |
xrcd=self.xrcd) |
| 849 |
qp_attr.qp_access_flags = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | \ |
| 850 |
ibv_access_flags.IBV_ACCESS_REMOTE_READ | \ |
| 851 |
ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \ |
| 852 |
ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC |
| 853 |
recv_qp = QP(self.ctx, attr_ex, qp_attr) |
| 854 |
self.rqp_lst.append(recv_qp) |
| 855 |
|
| 856 |
qp_caps = QPCap(max_send_wr=self.num_msgs, max_recv_sge=0, |
| 857 |
max_recv_wr=0) |
| 858 |
attr_ex = QPInitAttrEx(qp_type=ibv_qp_type.IBV_QPT_XRC_SEND, sq_sig_all=1, |
| 859 |
comp_mask=ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD, |
| 860 |
pd=self.pd, scq=self.cq, cap=qp_caps) |
| 861 |
qp_attr.qp_access_flags = 0 |
| 862 |
send_qp =QP(self.ctx, attr_ex, qp_attr) |
| 863 |
self.sqp_lst.append(send_qp) |
| 864 |
self.qps_num.append((recv_qp.qp_num, send_qp.qp_num)) |
| 865 |
self.psns.append(random.getrandbits(24)) |
| 866 |
|
| 867 |
def create_xrcd(self): |
| 868 |
""" |
| 869 |
Initializes self.xrcd with an XRC Domain object. |
| 870 |
:return: None |
| 871 |
""" |
| 872 |
self.temp_file = tempfile.NamedTemporaryFile() |
| 873 |
self.xrcd_fd = os.open(self.temp_file.name, os.O_RDONLY | os.O_CREAT, |
| 874 |
stat.S_IRUSR | stat.S_IRGRP) |
| 875 |
init = XRCDInitAttr( |
| 876 |
ibv_xrcd_init_attr_mask.IBV_XRCD_INIT_ATTR_FD | ibv_xrcd_init_attr_mask.IBV_XRCD_INIT_ATTR_OFLAGS, |
| 877 |
os.O_CREAT, self.xrcd_fd) |
| 878 |
try: |
| 879 |
self.xrcd = XRCD(self.ctx, init) |
| 880 |
except PyverbsRDMAError as ex: |
| 881 |
if ex.error_code == errno.EOPNOTSUPP: |
| 882 |
raise unittest.SkipTest('Create XRCD is not supported') |
| 883 |
raise ex |
| 884 |
|
| 885 |
def create_srq(self): |
| 886 |
""" |
| 887 |
Initializes self.srq with a Shared Receive QP object. |
| 888 |
:return: None |
| 889 |
""" |
| 890 |
srq_attr = SrqInitAttrEx(max_wr=self.qp_count*self.num_msgs) |
| 891 |
srq_attr.srq_type = ibv_srq_type.IBV_SRQT_XRC |
| 892 |
srq_attr.pd = self.pd |
| 893 |
srq_attr.xrcd = self.xrcd |
| 894 |
srq_attr.cq = self.cq |
| 895 |
srq_attr.comp_mask = ibv_srq_init_attr_mask.IBV_SRQ_INIT_ATTR_TYPE | ibv_srq_init_attr_mask.IBV_SRQ_INIT_ATTR_PD | \ |
| 896 |
ibv_srq_init_attr_mask.IBV_SRQ_INIT_ATTR_CQ | ibv_srq_init_attr_mask.IBV_SRQ_INIT_ATTR_XRCD |
| 897 |
self.srq = SRQ(self.ctx, srq_attr) |
| 898 |
|
| 899 |
def to_rts(self): |
| 900 |
gid = self.ctx.query_gid(self.ib_port, self.gid_index) |
| 901 |
gr = GlobalRoute(dgid=gid, sgid_index=self.gid_index) |
| 902 |
ah_attr = AHAttr(port_num=self.ib_port, is_global=True, |
| 903 |
gr=gr, dlid=self.port_attr.lid) |
| 904 |
qp_attr = QPAttr() |
| 905 |
qp_attr.max_rd_atomic = MAX_RD_ATOMIC |
| 906 |
qp_attr.max_dest_rd_atomic = MAX_DEST_RD_ATOMIC |
| 907 |
qp_attr.path_mtu = PATH_MTU |
| 908 |
set_rnr_attributes(qp_attr) |
| 909 |
qp_attr.ah_attr = ah_attr |
| 910 |
for i in range(self.qp_count): |
| 911 |
qp_attr.dest_qp_num = self.rqps_num[i][1] |
| 912 |
qp_attr.rq_psn = self.psns[i] |
| 913 |
qp_attr.sq_psn = self.rpsns[i] |
| 914 |
self.rqp_lst[i].to_rts(qp_attr) |
| 915 |
qp_attr.dest_qp_num = self.rqps_num[i][0] |
| 916 |
self.sqp_lst[i].to_rts(qp_attr) |
| 917 |
|
| 918 |
def init_resources(self): |
| 919 |
self.create_xrcd() |
| 920 |
super(XRCResources, self).init_resources() |
| 921 |
self.create_srq() |