| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file |
| 3 |
|
| 4 |
from pyverbs.providers.mlx5.mlx5dv import Wqe, WqeDataSeg, WqeCtrlSeg |
| 5 |
from pyverbs.pyverbs_error import PyverbsError |
| 6 |
from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_qp_create_send_ops_flags, MLX5_WQE_CTRL_CQ_UPDATE, MLX5_OPCODE_SEND |
| 7 |
from tests.mlx5_base import Mlx5RDMATestCase, Mlx5RcResources |
| 8 |
from pyverbs.qp import QPCap |
| 9 |
from pyverbs.wr import SGE |
| 10 |
from pyverbs.libibverbs_enums import ibv_qp_create_send_ops_flags, ibv_send_flags, ibv_wc_opcode |
| 11 |
import tests.utils as u |
| 12 |
|
| 13 |
|
| 14 |
class Mlx5RawWqeResources(Mlx5RcResources): |
| 15 |
def create_send_ops_flags(self): |
| 16 |
self.dv_send_ops_flags = mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_RAW_WQE |
| 17 |
self.send_ops_flags = ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND |
| 18 |
|
| 19 |
def create_qp_cap(self): |
| 20 |
""" |
| 21 |
Create QPCap such that work queue elements will wrap around the send |
| 22 |
work queue, this happens due to the iteration count being higher |
| 23 |
than the max_send_wr. |
| 24 |
:return: |
| 25 |
""" |
| 26 |
return QPCap(max_send_wr=1, max_recv_wr=4, max_recv_sge=2, max_send_sge=2) |
| 27 |
|
| 28 |
|
| 29 |
class RawWqeTest(Mlx5RDMATestCase): |
| 30 |
def setUp(self): |
| 31 |
super().setUp() |
| 32 |
self.iters = 10 |
| 33 |
self.server = None |
| 34 |
self.client = None |
| 35 |
|
| 36 |
def prepare_send_elements(self): |
| 37 |
mr = self.client.mr |
| 38 |
sge_count = 2 |
| 39 |
unit_size = mr.length / 2 |
| 40 |
data_segs = [WqeDataSeg(unit_size, mr.lkey, mr.buf + i * unit_size) for |
| 41 |
i in range(sge_count)] |
| 42 |
ctrl_seg = WqeCtrlSeg() |
| 43 |
ctrl_seg.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE |
| 44 |
segment_num = 1 + len(data_segs) |
| 45 |
ctrl_seg.opmod_idx_opcode = MLX5_OPCODE_SEND |
| 46 |
ctrl_seg.qpn_ds = segment_num | int(self.client.qp.qp_num) << 8 |
| 47 |
self.raw_send_wqe = Wqe([ctrl_seg] + data_segs) |
| 48 |
self.regular_send_sge = SGE(mr.buf, mr.length, mr.lkey) |
| 49 |
|
| 50 |
def mixed_traffic(self): |
| 51 |
s_recv_wr = u.get_recv_wr(self.server) |
| 52 |
u.post_recv(self.server, s_recv_wr) |
| 53 |
self.prepare_send_elements() |
| 54 |
|
| 55 |
for i in range(self.iters): |
| 56 |
self.client.qp.wr_start() |
| 57 |
if i % 2: |
| 58 |
self.client.mr.write('c' * self.client.mr.length, self.client.mr.length) |
| 59 |
self.client.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED |
| 60 |
self.client.qp.wr_send() |
| 61 |
self.client.qp.wr_set_sge(self.regular_send_sge) |
| 62 |
else: |
| 63 |
self.client.mr.write('s' * self.client.mr.length, self.client.mr.length) |
| 64 |
self.client.qp.wr_raw_wqe(self.raw_send_wqe) |
| 65 |
self.client.qp.wr_complete() |
| 66 |
u.poll_cq_ex(self.client.cq) |
| 67 |
u.poll_cq_ex(self.server.cq) |
| 68 |
u.post_recv(self.server, s_recv_wr) |
| 69 |
expected_opcode = ibv_wc_opcode.IBV_WC_SEND if i % 2 else ibv_wc_opcode.IBV_WC_DRIVER2 |
| 70 |
|
| 71 |
if self.client.cq.read_opcode() != expected_opcode: |
| 72 |
raise PyverbsError('Opcode validation failed: expected ' |
| 73 |
f'{expected_opcode}, received {self.client.cq.read_opcode()}') |
| 74 |
|
| 75 |
act_buffer = self.server.mr.read(self.server.mr.length, 0) |
| 76 |
u.validate(act_buffer, i % 2, self.server.mr.length) |
| 77 |
|
| 78 |
def test_mixed_raw_wqe_traffic(self): |
| 79 |
""" |
| 80 |
Runs traffic with a mix of SEND opcode regular WQEs and SEND opcode RAW |
| 81 |
WQEs. |
| 82 |
""" |
| 83 |
self.create_players(Mlx5RawWqeResources) |
| 84 |
self.mixed_traffic() |