| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright (c) 2020 Nvidia Corporation. All rights reserved. See COPYING file |
| 3 |
|
| 4 |
import unittest |
| 5 |
import errno |
| 6 |
|
| 7 |
from pyverbs.providers.mlx5.mlx5dv import Mlx5DVQPInitAttr, Mlx5QP, \ |
| 8 |
Mlx5DVDCInitAttr, Mlx5Context |
| 9 |
from tests.test_rdmacm import CMAsyncConnection |
| 10 |
from tests.mlx5_base import Mlx5PyverbsAPITestCase, Mlx5RDMACMBaseTest |
| 11 |
from pyverbs.pyverbs_error import PyverbsRDMAError |
| 12 |
from pyverbs.srq import SRQ, SrqInitAttr, SrqAttr |
| 13 |
from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_qp_init_attr_mask, mlx5dv_dc_type |
| 14 |
from tests.base_rdmacm import AsyncCMResources |
| 15 |
from pyverbs.qp import QPCap, QPInitAttrEx |
| 16 |
from pyverbs.cmid import ConnParam |
| 17 |
from tests.base import DCT_KEY |
| 18 |
from pyverbs.addr import AH |
| 19 |
from pyverbs.libibverbs_enums import ibv_wr_opcode, ibv_qp_create_send_ops_flags, ibv_qp_init_attr_mask, \ |
| 20 |
ibv_qp_type, ibv_qp_state |
| 21 |
from pyverbs.cq import CQ |
| 22 |
import tests.utils as u |
| 23 |
|
| 24 |
|
| 25 |
class DcCMConnection(CMAsyncConnection): |
| 26 |
""" |
| 27 |
Implement RDMACM connection management for asynchronous CMIDs using DC as |
| 28 |
an external QP. |
| 29 |
""" |
| 30 |
def create_cm_res(self, ip_addr, passive, **kwargs): |
| 31 |
self.cm_res = DcCMResources(addr=ip_addr, passive=passive, **kwargs) |
| 32 |
if passive: |
| 33 |
self.cm_res.create_cmid() |
| 34 |
else: |
| 35 |
for conn_idx in range(self.num_conns): |
| 36 |
self.cm_res.create_cmid(conn_idx) |
| 37 |
self.get_and_update_addrinfo(passive, ip_addr) |
| 38 |
|
| 39 |
def _ext_qp_server_traffic(self): |
| 40 |
recv_wr = u.get_recv_wr(self.cm_res) |
| 41 |
for _ in range(self.cm_res.num_msgs): |
| 42 |
u.post_recv(self.cm_res, recv_wr) |
| 43 |
self.syncer.wait() |
| 44 |
for _ in range(self.cm_res.num_msgs): |
| 45 |
u.poll_cq(self.cm_res.cq) |
| 46 |
|
| 47 |
def _ext_qp_client_traffic(self): |
| 48 |
self.cm_res.remote_dct_num = self.cm_res.remote_qpn |
| 49 |
_, send_wr = u.get_send_elements(self.cm_res, self.cm_res.passive) |
| 50 |
ah = AH(self.cm_res.cmid.pd, attr=self.cm_res.remote_ah) |
| 51 |
self.syncer.wait() |
| 52 |
for send_idx in range(self.cm_res.num_msgs): |
| 53 |
dci_idx = send_idx % len(self.cm_res.qps) |
| 54 |
u.post_send_ex(self.cm_res, send_wr, ibv_wr_opcode.IBV_WR_SEND, ah=ah, |
| 55 |
qp_idx=dci_idx) |
| 56 |
u.poll_cq(self.cm_res.cq) |
| 57 |
|
| 58 |
def disconnect(self): |
| 59 |
if self.cm_res.reserved_qp_num and self.cm_res.passive: |
| 60 |
Mlx5Context.reserved_qpn_dealloc(self.cm_res.child_id.context, |
| 61 |
self.cm_res.reserved_qp_num) |
| 62 |
self.cm_res.reserved_qp_num = 0 |
| 63 |
super().disconnect() |
| 64 |
|
| 65 |
|
| 66 |
class DcCMResources(AsyncCMResources): |
| 67 |
""" |
| 68 |
DcCMResources class contains resources for RDMA CM asynchronous |
| 69 |
communication using DC as an external QP. |
| 70 |
""" |
| 71 |
def __init__(self, addr=None, passive=None, **kwargs): |
| 72 |
""" |
| 73 |
Init DcCMResources instance. |
| 74 |
:param passive: Indicate if this CM is the passive CM. |
| 75 |
""" |
| 76 |
super().__init__(passive=passive, **kwargs) |
| 77 |
self.srq = None |
| 78 |
self.remote_dct_num = None |
| 79 |
self.reserved_qp_num = 0 |
| 80 |
|
| 81 |
def create_qp(self, conn_idx=0): |
| 82 |
""" |
| 83 |
Create an RDMACM QP. If self.with_ext_qp is set, then an external CQ and |
| 84 |
DC QP will be created. In case that CQ is already created, it is used |
| 85 |
for the newly created QP. |
| 86 |
""" |
| 87 |
try: |
| 88 |
if not self.passive: |
| 89 |
# Create the DCI QPs. |
| 90 |
cmid = self.cmids[conn_idx] |
| 91 |
self.create_cq(cmid) |
| 92 |
qp_init_attr = self.create_qp_init_attr(cmid, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND) |
| 93 |
attr = Mlx5DVQPInitAttr(comp_mask=mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_DC, |
| 94 |
dc_init_attr=Mlx5DVDCInitAttr()) |
| 95 |
self.qps[conn_idx] = Mlx5QP(cmid.context, qp_init_attr, attr) |
| 96 |
|
| 97 |
if self.passive and conn_idx == 0: |
| 98 |
# Create the DCT QP only for the first connection. |
| 99 |
cmid = self.child_id |
| 100 |
self.create_cq(cmid) |
| 101 |
self.create_srq(cmid) |
| 102 |
qp_init_attr = self.create_qp_init_attr(cmid) |
| 103 |
dc_attr = Mlx5DVDCInitAttr(dc_type=mlx5dv_dc_type.MLX5DV_DCTYPE_DCT, |
| 104 |
dct_access_key=DCT_KEY) |
| 105 |
attr = Mlx5DVQPInitAttr(comp_mask=mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_DC, |
| 106 |
dc_init_attr=dc_attr) |
| 107 |
self.qps[conn_idx] = Mlx5QP(cmid.context, qp_init_attr, attr) |
| 108 |
except PyverbsRDMAError as ex: |
| 109 |
if ex.error_code == errno.EOPNOTSUPP: |
| 110 |
raise unittest.SkipTest('Create DC QP is not supported') |
| 111 |
raise ex |
| 112 |
|
| 113 |
def create_qp_cap(self): |
| 114 |
return QPCap(self.num_msgs, 0, 1, 0) |
| 115 |
|
| 116 |
def create_qp_init_attr(self, cmid, send_ops_flags=0): |
| 117 |
comp_mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD |
| 118 |
if send_ops_flags: |
| 119 |
comp_mask |= ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS |
| 120 |
return QPInitAttrEx(cap=self.create_qp_cap(), pd=cmid.pd, scq=self.cq, |
| 121 |
rcq=self.cq, srq=self.srq, qp_type=ibv_qp_type.IBV_QPT_DRIVER, |
| 122 |
send_ops_flags=send_ops_flags, comp_mask=comp_mask, |
| 123 |
sq_sig_all=1) |
| 124 |
|
| 125 |
def create_srq(self, cmid): |
| 126 |
srq_init_attr = SrqInitAttr(SrqAttr(max_wr=self.num_msgs)) |
| 127 |
try: |
| 128 |
self.srq = SRQ(cmid.pd, srq_init_attr) |
| 129 |
except PyverbsRDMAError as ex: |
| 130 |
if ex.error_code == errno.EOPNOTSUPP: |
| 131 |
raise unittest.SkipTest('Create SRQ is not supported') |
| 132 |
raise ex |
| 133 |
|
| 134 |
def modify_ext_qp_to_rts(self, conn_idx=0): |
| 135 |
cmids = self.child_ids if self.passive else self.cmids |
| 136 |
if not self.passive or not conn_idx: |
| 137 |
qp = self.qps[conn_idx] |
| 138 |
attr, _ = cmids[conn_idx].init_qp_attr(ibv_qp_state.IBV_QPS_INIT) |
| 139 |
qp.to_init(attr) |
| 140 |
attr, _ = cmids[conn_idx].init_qp_attr(ibv_qp_state.IBV_QPS_RTR) |
| 141 |
qp.to_rtr(attr) |
| 142 |
if not self.passive: |
| 143 |
# The passive QP is DCT which should stay in RTR state. |
| 144 |
self.remote_ah = attr.ah_attr |
| 145 |
attr, _ = cmids[conn_idx].init_qp_attr(ibv_qp_state.IBV_QPS_RTS) |
| 146 |
qp.to_rts(attr) |
| 147 |
|
| 148 |
def create_conn_param(self, qp_num=0, conn_idx=0): |
| 149 |
if conn_idx and self.passive: |
| 150 |
try: |
| 151 |
ctx = self.child_id.context |
| 152 |
self.reserved_qp_num = Mlx5Context.reserved_qpn_alloc(ctx) |
| 153 |
except PyverbsRDMAError as ex: |
| 154 |
if ex.error_code == errno.EOPNOTSUPP: |
| 155 |
raise unittest.SkipTest('Alloc reserved QP number is not supported') |
| 156 |
raise ex |
| 157 |
qp_num = self.reserved_qp_num |
| 158 |
else: |
| 159 |
qp_num = self.qps[conn_idx].qp_num |
| 160 |
return ConnParam(qp_num=qp_num) |
| 161 |
|
| 162 |
|
| 163 |
class Mlx5CMTestCase(Mlx5RDMACMBaseTest): |
| 164 |
""" |
| 165 |
Mlx5 RDMACM test class. |
| 166 |
""" |
| 167 |
def test_rdmacm_async_traffic_dc_external_qp(self): |
| 168 |
""" |
| 169 |
Connect multiple RDMACM connections using DC as an external QP for |
| 170 |
traffic. |
| 171 |
""" |
| 172 |
self.two_nodes_rdmacm_traffic(DcCMConnection, self.rdmacm_traffic, |
| 173 |
with_ext_qp=True, num_conns=2) |
| 174 |
|
| 175 |
|
| 176 |
class ReservedQPTest(Mlx5PyverbsAPITestCase): |
| 177 |
|
| 178 |
def test_reservered_qpn(self): |
| 179 |
""" |
| 180 |
Alloc reserved qpn multiple times and then dealloc the qpns. In addition, |
| 181 |
the test includes bad flows where a fake qpn gets deallocated, and a |
| 182 |
real qpn gets deallocated twice. |
| 183 |
""" |
| 184 |
try: |
| 185 |
# Alloc qp number multiple times. |
| 186 |
qpns = [] |
| 187 |
for i in range(1000): |
| 188 |
qpns.append(Mlx5Context.reserved_qpn_alloc(self.ctx)) |
| 189 |
for i in range(1000): |
| 190 |
Mlx5Context.reserved_qpn_dealloc(self.ctx, qpns[i]) |
| 191 |
|
| 192 |
# Dealloc qp number that was not allocated. |
| 193 |
qpn = Mlx5Context.reserved_qpn_alloc(self.ctx) |
| 194 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 195 |
fake_qpn = qpn - 1 |
| 196 |
Mlx5Context.reserved_qpn_dealloc(self.ctx, fake_qpn) |
| 197 |
self.assertEqual(ex.exception.error_code, errno.EINVAL) |
| 198 |
|
| 199 |
# Try to dealloc same qp number twice. |
| 200 |
Mlx5Context.reserved_qpn_dealloc(self.ctx, qpn) |
| 201 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 202 |
Mlx5Context.reserved_qpn_dealloc(self.ctx, qpn) |
| 203 |
self.assertEqual(ex.exception.error_code, errno.EINVAL) |
| 204 |
|
| 205 |
except PyverbsRDMAError as ex: |
| 206 |
if ex.error_code == errno.EOPNOTSUPP: |
| 207 |
raise unittest.SkipTest('Alloc reserved QP number is not supported') |
| 208 |
raise ex |