| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright 2020-2025 Amazon.com, Inc. or its affiliates. All rights reserved. |
| 3 |
|
| 4 |
import unittest |
| 5 |
import errno |
| 6 |
|
| 7 |
from pyverbs.cq import CQ, CompChannel |
| 8 |
from pyverbs.libibverbs_enums import ibv_qp_create_send_ops_flags, ibv_wr_opcode, ibv_qp_attr_mask |
| 9 |
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError |
| 10 |
import pyverbs.providers.efa.efa_enums as efa_enums |
| 11 |
import pyverbs.providers.efa.efadv as efa |
| 12 |
|
| 13 |
from tests.efa_base import EfaRDMATestCase |
| 14 |
from tests.efa_base import SRDResources |
| 15 |
import tests.utils as u |
| 16 |
|
| 17 |
|
| 18 |
class CqEventsSRD(SRDResources): |
| 19 |
def __init__(self, dev_name, ib_port, gid_index): |
| 20 |
super().__init__(dev_name, ib_port, gid_index, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND) |
| 21 |
|
| 22 |
def create_cq(self): |
| 23 |
self.comp_channel = CompChannel(self.ctx) |
| 24 |
self.cq = CQ(self.ctx, self.num_msgs, None, self.comp_channel) |
| 25 |
self.cq.req_notify() |
| 26 |
|
| 27 |
|
| 28 |
class CqEventsSRDTestCase(EfaRDMATestCase): |
| 29 |
def setUp(self): |
| 30 |
super().setUp() |
| 31 |
self.iters = 100 |
| 32 |
|
| 33 |
def test_cq_events_srd(self): |
| 34 |
for use_new_send in [False, True]: |
| 35 |
with self.subTest(): |
| 36 |
super().create_players(CqEventsSRD) |
| 37 |
u.traffic(**self.traffic_args, new_send=use_new_send) |
| 38 |
|
| 39 |
|
| 40 |
class QPSRDTestCase(EfaRDMATestCase): |
| 41 |
def setUp(self): |
| 42 |
super().setUp() |
| 43 |
self.iters = 100 |
| 44 |
self.server = None |
| 45 |
self.client = None |
| 46 |
|
| 47 |
def create_players(self, send_ops_flags=0, qp_count=8, dev_cap=None, wc_flags=0, qp_flags=0): |
| 48 |
super().create_players(SRDResources, send_ops_flags=send_ops_flags, qp_count=qp_count, |
| 49 |
required_dev_cap=dev_cap, wc_flags=wc_flags, qp_flags=qp_flags) |
| 50 |
|
| 51 |
def full_sq_bad_flow(self): |
| 52 |
""" |
| 53 |
Check post_send while qp's sq is full. |
| 54 |
- Find qp's sq length |
| 55 |
- Fill the qp with work requests until overflow |
| 56 |
""" |
| 57 |
qp_idx = 0 |
| 58 |
send_op = ibv_wr_opcode.IBV_WR_SEND |
| 59 |
ah = u.get_global_ah(self.client, self.gid_index, self.ib_port) |
| 60 |
qp_attr, _ = self.client.qps[qp_idx].query(ibv_qp_attr_mask.IBV_QP_CAP) |
| 61 |
max_send_wr = qp_attr.cap.max_send_wr |
| 62 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 63 |
for _ in range (max_send_wr + 1): |
| 64 |
_, c_sg = u.get_send_elements(self.client, False) |
| 65 |
u.send(self.client, c_sg, send_op, new_send=True, qp_idx=qp_idx, ah=ah) |
| 66 |
self.assertEqual(ex.exception.error_code, errno.ENOMEM) |
| 67 |
|
| 68 |
def unsolicited_rdma_write_traffic(self, client, server, iters, gid_idx, port): |
| 69 |
""" |
| 70 |
Runs unsolicited rdma write traffic between two sides |
| 71 |
""" |
| 72 |
ah_client = u.get_global_ah(client, gid_idx, port) |
| 73 |
ah_server = u.get_global_ah(server, gid_idx, port) |
| 74 |
poll = u.poll_cq_ex |
| 75 |
imm_data = u.IMM_DATA |
| 76 |
send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM |
| 77 |
read_offset = 0 |
| 78 |
for _ in range(iters): |
| 79 |
for qp_idx in range(server.qp_count): |
| 80 |
_, c_send_object = u.get_send_elements(client, False, send_op) |
| 81 |
u.send(client, c_send_object, send_op, True, qp_idx, ah_client, is_imm=True) |
| 82 |
poll(client.cq) |
| 83 |
poll(server.cq, data=imm_data) |
| 84 |
# Validate that the CQE is marked as unsolicited |
| 85 |
if not server.cq.is_unsolicited(): |
| 86 |
raise PyverbsError('Completion was not marked as unsolicited') |
| 87 |
msg_received_list = u.get_msg_received(server, read_offset) |
| 88 |
for msg in msg_received_list: |
| 89 |
u.validate(msg, True, server.msg_size) |
| 90 |
|
| 91 |
def test_qp_ex_srd_send(self): |
| 92 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND) |
| 93 |
u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND) |
| 94 |
|
| 95 |
def test_qp_ex_srd_send_imm(self): |
| 96 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND_WITH_IMM) |
| 97 |
u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND_WITH_IMM) |
| 98 |
|
| 99 |
def test_qp_ex_srd_rdma_read(self): |
| 100 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_READ) |
| 101 |
self.server.mr.write('s' * self.server.msg_size, self.server.msg_size) |
| 102 |
u.rdma_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_READ) |
| 103 |
|
| 104 |
def test_qp_ex_srd_rdma_write(self): |
| 105 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE) |
| 106 |
u.rdma_traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE) |
| 107 |
|
| 108 |
def test_qp_ex_srd_rdma_write_with_imm(self): |
| 109 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM) |
| 110 |
u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM) |
| 111 |
|
| 112 |
def test_qp_ex_srd_rdma_unsolicited_write_with_imm(self): |
| 113 |
wc_flag = efa_enums.EFADV_WC_EX_WITH_IS_UNSOLICITED |
| 114 |
dev_cap = efa_enums.EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV |
| 115 |
qp_flags = efa_enums.EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV |
| 116 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM, |
| 117 |
dev_cap=dev_cap, wc_flags=wc_flag, qp_flags=qp_flags) |
| 118 |
self.unsolicited_rdma_write_traffic(**self.traffic_args) |
| 119 |
|
| 120 |
def test_qp_ex_srd_old_send(self): |
| 121 |
self.create_players() |
| 122 |
u.traffic(**self.traffic_args, new_send=False) |
| 123 |
|
| 124 |
def test_qp_ex_srd_old_send_imm(self): |
| 125 |
self.create_players() |
| 126 |
u.traffic(**self.traffic_args, new_send=False, send_op=ibv_wr_opcode.IBV_WR_SEND_WITH_IMM) |
| 127 |
|
| 128 |
def test_qp_ex_srd_zero_size(self): |
| 129 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND) |
| 130 |
self.client.msg_size = 0 |
| 131 |
self.server.msg_size = 0 |
| 132 |
u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND) |
| 133 |
|
| 134 |
def test_post_receive_qp_state_bad_flow(self): |
| 135 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1) |
| 136 |
u.post_rq_state_bad_flow(self) |
| 137 |
|
| 138 |
def test_post_send_qp_state_bad_flow(self): |
| 139 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1) |
| 140 |
u.post_sq_state_bad_flow(self) |
| 141 |
|
| 142 |
def test_full_rq_bad_flow(self): |
| 143 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1) |
| 144 |
u.full_rq_bad_flow(self) |
| 145 |
|
| 146 |
def test_full_sq_bad_flow(self): |
| 147 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1) |
| 148 |
self.full_sq_bad_flow() |
| 149 |
|
| 150 |
def test_rq_with_larger_sgl_bad_flow(self): |
| 151 |
self.create_players(ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND, qp_count=1) |
| 152 |
u.create_rq_with_larger_sgl_bad_flow(self) |