| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file |
| 3 |
|
| 4 |
from enum import Enum |
| 5 |
import unittest |
| 6 |
|
| 7 |
from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr |
| 8 |
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError |
| 9 |
from tests.mlx5_base import Mlx5RDMATestCase, Mlx5RcResources |
| 10 |
from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_qp_create_send_ops_flags, mlx5dv_wc_opcode, \ |
| 11 |
mlx5dv_context_comp_mask |
| 12 |
from pyverbs.pd import PD |
| 13 |
from pyverbs.mr import MR |
| 14 |
from pyverbs.libibverbs_enums import ibv_qp_create_send_ops_flags, ibv_access_flags, ibv_send_flags, \ |
| 15 |
ibv_wc_status |
| 16 |
import tests.utils as u |
| 17 |
|
| 18 |
|
| 19 |
class BadFlowType(Enum): |
| 20 |
DIFFERENT_PD = 1 |
| 21 |
MR_ILLEGAL_ACCESS = 2 |
| 22 |
|
| 23 |
|
| 24 |
class Mlx5DmaResources(Mlx5RcResources): |
| 25 |
def create_send_ops_flags(self): |
| 26 |
self.dv_send_ops_flags = mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MEMCPY |
| 27 |
self.send_ops_flags = ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND |
| 28 |
|
| 29 |
|
| 30 |
class DmaGgaMemcpy(Mlx5RDMATestCase): |
| 31 |
def create_resources(self, bad_flow_type=0, **resource_arg): |
| 32 |
""" |
| 33 |
Creates DmaGga test resources that include a "server" resource that |
| 34 |
can be used to send the MEMCPY WR, and a destination MR to copy data to. |
| 35 |
The destination MR can be created on a different PD or with insufficient |
| 36 |
access permissions, according to the bad_flow_type. |
| 37 |
:param bad_flow_type: (Optional) An enum of BadFlowType that indicates |
| 38 |
the bad flow type (default: 0 - good flow) |
| 39 |
:param resource_arg: Dict of args that specify the resource specific |
| 40 |
attributes. |
| 41 |
:return: None |
| 42 |
""" |
| 43 |
self.server = Mlx5DmaResources(**self.dev_info, **resource_arg) |
| 44 |
self.dest_pd = self.server.pd |
| 45 |
dest_mr_access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE |
| 46 |
if bad_flow_type == BadFlowType.DIFFERENT_PD: |
| 47 |
self.dest_pd = PD(self.server.ctx) |
| 48 |
elif bad_flow_type == BadFlowType.MR_ILLEGAL_ACCESS: |
| 49 |
dest_mr_access = ibv_access_flags.IBV_ACCESS_REMOTE_READ |
| 50 |
self.dest_mr = MR(self.dest_pd, self.server.msg_size, dest_mr_access) |
| 51 |
# No need to connect the QPs |
| 52 |
self.server.pre_run([0], [0]) |
| 53 |
|
| 54 |
def dma_memcpy(self, msg_size=1024, bad_flow=False): |
| 55 |
""" |
| 56 |
Creates resources and posts a memcpy WR. |
| 57 |
After posting the WR, the WC opcode and the data are verified. |
| 58 |
:param msg_size: Size of the data to be copied (in Bytes) |
| 59 |
:param bad_flow: If True, do not fill data in the MRs (default: False) |
| 60 |
:return: None |
| 61 |
""" |
| 62 |
self.create_resources(msg_size=msg_size) |
| 63 |
if not bad_flow: |
| 64 |
self.dest_mr.write('0' * msg_size, msg_size) |
| 65 |
self.server.mr.write('s' * msg_size, msg_size) |
| 66 |
self.server.qp.wr_start() |
| 67 |
self.server.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED |
| 68 |
self.server.qp.wr_memcpy(self.dest_mr.lkey, self.dest_mr.buf, self.server.mr.lkey, |
| 69 |
self.server.mr.buf, msg_size) |
| 70 |
self.server.qp.wr_complete() |
| 71 |
u.poll_cq_ex(self.server.cq) |
| 72 |
wc_opcode = self.server.cq.read_opcode() |
| 73 |
self.assertEqual(wc_opcode, mlx5dv_wc_opcode.MLX5DV_WC_MEMCPY, |
| 74 |
'WC opcode validation failed') |
| 75 |
self.assertEqual(self.dest_mr.read(msg_size, 0), |
| 76 |
self.server.mr.read(msg_size, 0)) |
| 77 |
|
| 78 |
def dma_memcpy_bad_protection_flow(self, bad_flow_type): |
| 79 |
""" |
| 80 |
Creates resources with bad protection and posts a memcpy WR. |
| 81 |
The bad protection is either a destination MR created on a different PD |
| 82 |
or a destination MR created with insufficient access permissions. |
| 83 |
:param bad_flow_type: An enum of BadFlowType that indicates the bad flow type |
| 84 |
:return: None |
| 85 |
""" |
| 86 |
self.create_resources(bad_flow_type) |
| 87 |
self.server.qp.wr_start() |
| 88 |
self.server.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED |
| 89 |
self.server.qp.wr_memcpy(self.dest_mr.lkey, self.dest_mr.buf, |
| 90 |
self.server.mr.lkey, |
| 91 |
self.server.mr.buf, self.server.msg_size) |
| 92 |
self.server.qp.wr_complete() |
| 93 |
with self.assertRaises(PyverbsRDMAError): |
| 94 |
u.poll_cq_ex(self.server.cq) |
| 95 |
self.assertEqual(self.server.cq.status, ibv_wc_status.IBV_WC_LOC_PROT_ERR, |
| 96 |
'Expected CQE with Local Protection Error') |
| 97 |
|
| 98 |
def test_dma_memcpy_data(self): |
| 99 |
self.dma_memcpy() |
| 100 |
|
| 101 |
def test_dma_memcpy_different_pd_bad_flow(self): |
| 102 |
self.dma_memcpy_bad_protection_flow(BadFlowType.DIFFERENT_PD) |
| 103 |
|
| 104 |
def test_dma_memcpy_protection_bad_flow(self): |
| 105 |
self.dma_memcpy_bad_protection_flow(BadFlowType.MR_ILLEGAL_ACCESS) |
| 106 |
|
| 107 |
def test_dma_memcpy_large_data_bad_flow(self): |
| 108 |
""" |
| 109 |
Bad flow test, testing DMA memcpy with data larger than the maximum |
| 110 |
allowed size, according to the HCA capabilities. |
| 111 |
:return: None |
| 112 |
""" |
| 113 |
try: |
| 114 |
ctx = Mlx5Context(Mlx5DVContextAttr(), name=self.dev_name) |
| 115 |
except PyverbsUserError as ex: |
| 116 |
raise unittest.SkipTest(f'Could not open mlx5 context ({ex})') |
| 117 |
except PyverbsRDMAError: |
| 118 |
raise unittest.SkipTest('Opening mlx5 context is not supported') |
| 119 |
max_size = ctx.query_mlx5_device( |
| 120 |
mlx5dv_context_comp_mask.MLX5DV_CONTEXT_MASK_WR_MEMCPY_LENGTH).max_wr_memcpy_length |
| 121 |
max_size = max_size if max_size else 1024 |
| 122 |
|
| 123 |
with self.assertRaises(PyverbsRDMAError): |
| 124 |
self.dma_memcpy(max_size + 1, bad_flow=True) |