| 1 |
import unittest |
| 2 |
import errno |
| 3 |
|
| 4 |
from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \ |
| 5 |
Mlx5DVCQInitAttr, Mlx5CQ, context_flags_to_str, cqe_comp_to_str |
| 6 |
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError |
| 7 |
from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_cq_init_attr_mask, mlx5dv_cqe_comp_res_format, \ |
| 8 |
mlx5dv_context_flags, mlx5dv_cq_init_attr_flags, mlx5dv_qp_create_flags |
| 9 |
from tests.mlx5_base import Mlx5RDMATestCase |
| 10 |
from tests.mlx5_base import Mlx5DcResources |
| 11 |
from pyverbs.cq import CqInitAttrEx |
| 12 |
from tests.base import RCResources |
| 13 |
from pyverbs.libibverbs_enums import ibv_qp_create_send_ops_flags, ibv_wr_opcode |
| 14 |
import tests.utils as u |
| 15 |
|
| 16 |
|
| 17 |
def create_dv_cq(res): |
| 18 |
""" |
| 19 |
Create Mlx5 DV CQ. |
| 20 |
:param res: An instance of BaseResources. |
| 21 |
:return: None |
| 22 |
""" |
| 23 |
dvcq_init_attr = Mlx5DVCQInitAttr() |
| 24 |
if res.cqe_comp_res_format: |
| 25 |
dvcq_init_attr.cqe_comp_res_format = res.cqe_comp_res_format |
| 26 |
dvcq_init_attr.comp_mask |= mlx5dv_cq_init_attr_mask.MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE |
| 27 |
# Check CQE compression capability |
| 28 |
cqe_comp_caps = res.ctx.query_mlx5_device().cqe_comp_caps |
| 29 |
if not (cqe_comp_caps['supported_format'] & res.cqe_comp_res_format) or \ |
| 30 |
not cqe_comp_caps['max_num']: |
| 31 |
cqe_comp_str = cqe_comp_to_str(res.cqe_comp_res_format) |
| 32 |
raise unittest.SkipTest(f'CQE compression {cqe_comp_str} is not supported') |
| 33 |
if res.flags: |
| 34 |
dvcq_init_attr.flags = res.flags |
| 35 |
dvcq_init_attr.comp_mask |= mlx5dv_cq_init_attr_mask.MLX5DV_CQ_INIT_ATTR_MASK_FLAGS |
| 36 |
if res.cqe_size: |
| 37 |
dvcq_init_attr.cqe_size = res.cqe_size |
| 38 |
dvcq_init_attr.comp_mask |= mlx5dv_cq_init_attr_mask.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE |
| 39 |
try: |
| 40 |
res.cq = Mlx5CQ(res.ctx, CqInitAttrEx(), dvcq_init_attr) |
| 41 |
except PyverbsRDMAError as ex: |
| 42 |
if ex.error_code == errno.EOPNOTSUPP: |
| 43 |
raise unittest.SkipTest('Create Mlx5DV CQ is not supported') |
| 44 |
raise ex |
| 45 |
|
| 46 |
|
| 47 |
class Mlx5CQRes(RCResources): |
| 48 |
def __init__(self, dev_name, ib_port, gid_index, cqe_comp_res_format=None, |
| 49 |
flags=None, cqe_size=None, msg_size=1024, requested_dev_cap=None): |
| 50 |
""" |
| 51 |
Initialize Mlx5 DV CQ resources based on RC resources that include RC |
| 52 |
QP. |
| 53 |
:param dev_name: Device name to be used |
| 54 |
:param ib_port: IB port of the device to use |
| 55 |
:param gid_index: Which GID index to use |
| 56 |
:param cqe_comp_res_format: Type of compression to use |
| 57 |
:param flags: DV CQ specific flags |
| 58 |
:param cqe_size: The CQE size |
| 59 |
:param msg_size: The resource msg size |
| 60 |
:param requested_dev_cap: A necessary device cap. If it's not supported |
| 61 |
by the device, the test will be skipped. |
| 62 |
""" |
| 63 |
self.cqe_comp_res_format = cqe_comp_res_format |
| 64 |
self.flags = flags |
| 65 |
self.cqe_size = cqe_size |
| 66 |
self.requested_dev_cap = requested_dev_cap |
| 67 |
super().__init__(dev_name, ib_port, gid_index, msg_size=msg_size) |
| 68 |
|
| 69 |
def create_context(self): |
| 70 |
mlx5dv_attr = Mlx5DVContextAttr() |
| 71 |
try: |
| 72 |
self.ctx = Mlx5Context(mlx5dv_attr, name=self.dev_name) |
| 73 |
except PyverbsUserError as ex: |
| 74 |
raise unittest.SkipTest(f'Could not open mlx5 context ({ex})') |
| 75 |
except PyverbsRDMAError: |
| 76 |
raise unittest.SkipTest('Opening mlx5 context is not supported') |
| 77 |
if self.requested_dev_cap: |
| 78 |
if not self.ctx.query_mlx5_device().flags & self.requested_dev_cap: |
| 79 |
miss_caps = context_flags_to_str(self.requested_dev_cap) |
| 80 |
raise unittest.SkipTest(f'Device caps doesn\'t support {miss_caps}') |
| 81 |
|
| 82 |
def create_cq(self): |
| 83 |
create_dv_cq(self) |
| 84 |
|
| 85 |
|
| 86 |
class Mlx5DvCqDcRes(Mlx5DcResources): |
| 87 |
def __init__(self, dev_name, ib_port, gid_index, cqe_comp_res_format=None, |
| 88 |
flags=None, cqe_size=None, create_flags=None): |
| 89 |
""" |
| 90 |
Initialize Mlx5 DV CQ resources based on RC resources that include RC |
| 91 |
QP. |
| 92 |
:param dev_name: Device name to be used |
| 93 |
:param ib_port: IB port of the device to use |
| 94 |
:param gid_index: Which GID index to use |
| 95 |
:param cqe_comp_res_format: Type of compression to use |
| 96 |
:param flags: DV CQ specific flags |
| 97 |
:param cqe_size: The CQ's CQe size |
| 98 |
:param create_flags: DV QP specific flags |
| 99 |
""" |
| 100 |
self.cqe_comp_res_format = cqe_comp_res_format |
| 101 |
self.flags = flags |
| 102 |
self.cqe_size = cqe_size |
| 103 |
super().__init__(dev_name, ib_port, gid_index, |
| 104 |
send_ops_flags=ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, |
| 105 |
create_flags=create_flags) |
| 106 |
|
| 107 |
def create_cq(self): |
| 108 |
create_dv_cq(self) |
| 109 |
|
| 110 |
|
| 111 |
class DvCqTest(Mlx5RDMATestCase): |
| 112 |
def setUp(self): |
| 113 |
super().setUp() |
| 114 |
self.iters = 10 |
| 115 |
self.server = None |
| 116 |
self.client = None |
| 117 |
self.traffic_args = None |
| 118 |
|
| 119 |
def create_players(self, resource, **resource_arg): |
| 120 |
""" |
| 121 |
Init DV CQ tests resources. |
| 122 |
:param resource: The RDMA resources to use. |
| 123 |
:param resource_arg: Dict of args that specify the resource specific |
| 124 |
attributes. |
| 125 |
:return: None |
| 126 |
""" |
| 127 |
super().create_players(resource, **resource_arg) |
| 128 |
if resource == Mlx5DvCqDcRes: |
| 129 |
self.client.remote_dct_num = self.server.dct_qp.qp_num |
| 130 |
self.server.remote_dct_num = self.client.dct_qp.qp_num |
| 131 |
|
| 132 |
def test_dv_cq_traffic(self): |
| 133 |
""" |
| 134 |
Run SEND traffic using DC CQ. |
| 135 |
""" |
| 136 |
self.create_players(Mlx5CQRes) |
| 137 |
u.traffic(**self.traffic_args, is_cq_ex=True) |
| 138 |
|
| 139 |
def test_dv_cq_compression_flags(self): |
| 140 |
""" |
| 141 |
Create DV CQ with different types of CQE compression formats. The test |
| 142 |
also does bad flow and try to use more than one compression formats. |
| 143 |
""" |
| 144 |
# Create DV CQ with all legal compression flags. |
| 145 |
for comp_type in [mlx5dv_cqe_comp_res_format.MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX, |
| 146 |
mlx5dv_cqe_comp_res_format.MLX5DV_CQE_RES_FORMAT_CSUM, |
| 147 |
mlx5dv_cqe_comp_res_format.MLX5DV_CQE_RES_FORMAT_HASH]: |
| 148 |
self.create_players(Mlx5CQRes, cqe_comp_res_format=comp_type, |
| 149 |
requested_dev_cap=mlx5dv_context_flags.MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP) |
| 150 |
u.traffic(**self.traffic_args, is_cq_ex=True) |
| 151 |
|
| 152 |
# Try to create DV CQ with more than one compression flags. |
| 153 |
cqe_multi_format = mlx5dv_cqe_comp_res_format.MLX5DV_CQE_RES_FORMAT_HASH | \ |
| 154 |
mlx5dv_cqe_comp_res_format.MLX5DV_CQE_RES_FORMAT_CSUM |
| 155 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 156 |
self.create_players(Mlx5CQRes, cqe_comp_res_format=cqe_multi_format) |
| 157 |
self.assertEqual(ex.exception.error_code, errno.EINVAL) |
| 158 |
|
| 159 |
def test_dv_cq_padding(self): |
| 160 |
""" |
| 161 |
Create DV CQ with padding flag. |
| 162 |
""" |
| 163 |
self.create_players(Mlx5CQRes, cqe_size=128, |
| 164 |
flags=mlx5dv_cq_init_attr_flags.MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD, |
| 165 |
requested_dev_cap=mlx5dv_context_flags.MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD) |
| 166 |
u.traffic(**self.traffic_args, is_cq_ex=True) |
| 167 |
|
| 168 |
def test_dv_cq_padding_not_aligned_cqe_size(self): |
| 169 |
""" |
| 170 |
Create DV CQ with padding flag when CQE size is not 128B. The creation |
| 171 |
should fail because padding is supported only with CQE size of 128B. |
| 172 |
""" |
| 173 |
# Padding flag works only when the cqe size is 128. |
| 174 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 175 |
self.create_players(Mlx5CQRes, cqe_size=64, |
| 176 |
flags=mlx5dv_cq_init_attr_flags.MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD, |
| 177 |
requested_dev_cap=mlx5dv_context_flags.MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD) |
| 178 |
self.assertEqual(ex.exception.error_code, errno.EINVAL) |
| 179 |
|
| 180 |
def test_dv_cq_cqe_size_128(self): |
| 181 |
""" |
| 182 |
Test multiple sizes of msg using CQE size of 128B. |
| 183 |
""" |
| 184 |
msg_sizes = [60, # Lower than 64B |
| 185 |
70, # In range of 64B - 128B |
| 186 |
140] # Bigger than 128B |
| 187 |
for size in msg_sizes: |
| 188 |
self.create_players(Mlx5CQRes, cqe_size=128, msg_size=size) |
| 189 |
u.traffic(**self.traffic_args, is_cq_ex=True) |
| 190 |
|
| 191 |
def test_dv_cq_cqe_size_64(self): |
| 192 |
""" |
| 193 |
Test multiple sizes of msg using CQE size of 64B. |
| 194 |
""" |
| 195 |
msg_sizes = [16, # Lower than 32B |
| 196 |
60, # In range of 32B - 64B |
| 197 |
70] # Bigger than 64B |
| 198 |
for size in msg_sizes: |
| 199 |
self.create_players(Mlx5CQRes, cqe_size=64, msg_size=size) |
| 200 |
u.traffic(**self.traffic_args, is_cq_ex=True) |
| 201 |
|
| 202 |
def test_dv_cq_cqe_size_with_bad_size(self): |
| 203 |
""" |
| 204 |
Create CQ with ilegal cqe_size value. |
| 205 |
""" |
| 206 |
# Set the CQE size in the CQE creation. |
| 207 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 208 |
self.create_players(Mlx5CQRes, cqe_size=100) |
| 209 |
self.assertEqual(ex.exception.error_code, errno.EINVAL) |
| 210 |
|
| 211 |
# Set the CQE size using the environment value. |
| 212 |
self.set_env_variable('MLX5_CQE_SIZE', '100') |
| 213 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 214 |
self.create_players(Mlx5CQRes) |
| 215 |
self.assertEqual(ex.exception.error_code, errno.EINVAL) |
| 216 |
|
| 217 |
def test_dv_cq_cqe_size_environment_var(self): |
| 218 |
""" |
| 219 |
Create DV CQs with all the legal cqe_size values using the environment |
| 220 |
variable mechanism. |
| 221 |
""" |
| 222 |
for cqe_size in ['64', '128']: |
| 223 |
self.set_env_variable('MLX5_CQE_SIZE', cqe_size) |
| 224 |
self.create_players(Mlx5CQRes) |
| 225 |
|
| 226 |
def test_scatter_to_cqe_control_by_qp(self): |
| 227 |
""" |
| 228 |
Create QP with specific SCATTER_TO_CQE flags. The test set different |
| 229 |
values in the scatter2cqe environment variable and create the QP with |
| 230 |
enable/disable flags. The QP should ignore the environment variable |
| 231 |
value and behave according to the specific creation flag. |
| 232 |
""" |
| 233 |
for s2c_env_val in ['0', '1']: |
| 234 |
for qp_s2c_value in [mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE, |
| 235 |
mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE]: |
| 236 |
self.set_env_variable('MLX5_SCATTER_TO_CQE', s2c_env_val) |
| 237 |
self.create_players(Mlx5DvCqDcRes, create_flags=qp_s2c_value) |
| 238 |
u.traffic(**self.traffic_args, new_send=True, |
| 239 |
send_op=ibv_wr_opcode.IBV_WR_SEND, is_cq_ex=True) |