| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright 2020-2025 Amazon.com, Inc. or its affiliates. All rights reserved. |
| 3 |
|
| 4 |
import unittest |
| 5 |
import random |
| 6 |
import errno |
| 7 |
|
| 8 |
from pyverbs.pyverbs_error import PyverbsRDMAError |
| 9 |
from pyverbs.cq import CqInitAttrEx |
| 10 |
from pyverbs.qp import QPAttr, QPCap, QPInitAttrEx |
| 11 |
import pyverbs.providers.efa.efa_enums as efa_enums |
| 12 |
import pyverbs.providers.efa.efadv as efa |
| 13 |
import pyverbs.device as d |
| 14 |
from pyverbs.libibverbs_enums import ibv_qp_init_attr_mask, ibv_qp_type, ibv_qp_create_send_ops_flags,\ |
| 15 |
ibv_access_flags, IBV_WC_STANDARD_FLAGS |
| 16 |
|
| 17 |
from tests.base import PyverbsAPITestCase |
| 18 |
from tests.base import TrafficResources |
| 19 |
from tests.base import RDMATestCase |
| 20 |
import tests.utils |
| 21 |
|
| 22 |
|
| 23 |
AMAZON_VENDOR_ID = 0x1d0f |
| 24 |
|
| 25 |
|
| 26 |
def is_efa_dev(ctx): |
| 27 |
dev_attrs = ctx.query_device() |
| 28 |
return dev_attrs.vendor_id == AMAZON_VENDOR_ID |
| 29 |
|
| 30 |
|
| 31 |
def skip_if_not_efa_dev(ctx): |
| 32 |
if not is_efa_dev(ctx): |
| 33 |
raise unittest.SkipTest('Can not run the test over non EFA device') |
| 34 |
|
| 35 |
|
| 36 |
class EfaAPITestCase(PyverbsAPITestCase): |
| 37 |
def setUp(self): |
| 38 |
super().setUp() |
| 39 |
skip_if_not_efa_dev(self.ctx) |
| 40 |
|
| 41 |
|
| 42 |
class EfaRDMATestCase(RDMATestCase): |
| 43 |
def setUp(self): |
| 44 |
super().setUp() |
| 45 |
skip_if_not_efa_dev(d.Context(name=self.dev_name)) |
| 46 |
|
| 47 |
|
| 48 |
class SRDResources(TrafficResources): |
| 49 |
SRD_QKEY = 0x11111111 |
| 50 |
SRD_PKEY_INDEX = 0 |
| 51 |
def __init__(self, dev_name, ib_port, gid_index, send_ops_flags, |
| 52 |
qp_count=1, required_dev_cap=None, wc_flags=0, qp_flags=0): |
| 53 |
self.send_ops_flags = send_ops_flags |
| 54 |
self.qp_flags = qp_flags |
| 55 |
self.required_dev_cap = required_dev_cap |
| 56 |
self.efa_wc_flags = wc_flags |
| 57 |
super().__init__(dev_name, ib_port, gid_index, qp_count=qp_count) |
| 58 |
|
| 59 |
def create_context(self): |
| 60 |
super().create_context() |
| 61 |
if self.required_dev_cap: |
| 62 |
with efa.EfaContext(name=self.ctx.name) as efa_ctx: |
| 63 |
device_caps = efa_ctx.query_efa_device().device_caps |
| 64 |
if (device_caps & self.required_dev_cap) != self.required_dev_cap: |
| 65 |
miss_caps = efa.dev_cap_to_str(self.required_dev_cap) |
| 66 |
raise unittest.SkipTest(f"Device caps doesn't support {miss_caps}") |
| 67 |
|
| 68 |
def create_qp_attr(self): |
| 69 |
attr = QPAttr(port_num=self.ib_port) |
| 70 |
attr.qkey = self.SRD_QKEY |
| 71 |
attr.pkey_index = self.SRD_PKEY_INDEX |
| 72 |
return attr |
| 73 |
|
| 74 |
def to_rts(self): |
| 75 |
attr = self.create_qp_attr() |
| 76 |
for i in range(self.qp_count): |
| 77 |
attr.dest_qp_num = self.rqps_num[i] |
| 78 |
attr.sq_psn = self.rpsns[i] |
| 79 |
self.qps[i].to_rts(attr) |
| 80 |
|
| 81 |
def create_qps(self): |
| 82 |
qp_cap = QPCap(max_recv_wr=self.num_msgs, max_send_wr=self.num_msgs, max_recv_sge=1, |
| 83 |
max_send_sge=1) |
| 84 |
comp_mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD |
| 85 |
if self.send_ops_flags: |
| 86 |
comp_mask |= ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS |
| 87 |
qp_init_attr_ex = QPInitAttrEx(cap=qp_cap, qp_type=ibv_qp_type.IBV_QPT_DRIVER, scq=self.cq, |
| 88 |
rcq=self.cq, pd=self.pd, send_ops_flags=self.send_ops_flags, |
| 89 |
comp_mask=comp_mask) |
| 90 |
efa_init_attr_ex = efa.EfaQPInitAttr() |
| 91 |
efa_init_attr_ex.driver_qp_type = efa_enums.EFADV_QP_DRIVER_TYPE_SRD |
| 92 |
efa_init_attr_ex.flags |= self.qp_flags |
| 93 |
try: |
| 94 |
for _ in range(self.qp_count): |
| 95 |
qp = efa.SRDQPEx(self.ctx, qp_init_attr_ex, efa_init_attr_ex) |
| 96 |
self.qps.append(qp) |
| 97 |
self.qps_num.append(qp.qp_num) |
| 98 |
self.psns.append(random.getrandbits(24)) |
| 99 |
except PyverbsRDMAError as ex: |
| 100 |
if ex.error_code == errno.EOPNOTSUPP: |
| 101 |
raise unittest.SkipTest('Extended SRD QP is not supported on this device') |
| 102 |
raise ex |
| 103 |
|
| 104 |
def create_mr(self): |
| 105 |
additional_access_flags = 0 |
| 106 |
if self.send_ops_flags == ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_READ: |
| 107 |
additional_access_flags = ibv_access_flags.IBV_ACCESS_REMOTE_READ |
| 108 |
elif self.send_ops_flags in [ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE, ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM]: |
| 109 |
additional_access_flags = ibv_access_flags.IBV_ACCESS_REMOTE_WRITE |
| 110 |
self.mr = tests.utils.create_custom_mr(self, additional_access_flags) |
| 111 |
|
| 112 |
def create_cq(self): |
| 113 |
if self.efa_wc_flags == 0: |
| 114 |
super().create_cq() |
| 115 |
else: |
| 116 |
cia = CqInitAttrEx(wc_flags=IBV_WC_STANDARD_FLAGS) |
| 117 |
efa_cia = efa.EfaDVCQInitAttr(wc_flags=self.efa_wc_flags) |
| 118 |
try: |
| 119 |
self.cq = efa.EfaCQ(self.ctx, cia, efa_cia) |
| 120 |
except PyverbsRDMAError as ex: |
| 121 |
if ex.error_code == errno.EOPNOTSUPP: |
| 122 |
raise unittest.SkipTest('Create EFA DV CQ is not supported') |
| 123 |
raise ex |