| 1 |
import unittest |
| 2 |
import errno |
| 3 |
|
| 4 |
from tests.base import RCResources, RDMATestCase, XRCResources |
| 5 |
from pyverbs.pyverbs_error import PyverbsRDMAError |
| 6 |
from pyverbs.qp import QPAttr, QPInitAttr |
| 7 |
import pyverbs.device as d |
| 8 |
from pyverbs.libibverbs_enums import ibv_access_flags, ibv_qp_type, ibv_atomic_cap, ibv_wr_opcode |
| 9 |
from pyverbs.mr import MR |
| 10 |
import tests.utils as u |
| 11 |
|
| 12 |
|
| 13 |
class RCAtomic(RCResources): |
| 14 |
def __init__(self, dev_name, ib_port, gid_index, msg_size=8, qp_access=None, |
| 15 |
mr_access=None): |
| 16 |
""" |
| 17 |
Initialize an RCAtomic Resource object. |
| 18 |
:param dev_name: Device name to be used |
| 19 |
:param ib_port: IB port of the device to use |
| 20 |
:param gid_index: Which GID index to use |
| 21 |
:param msg_size: Message size for all resources memory actions |
| 22 |
:param qp_access: The QP access to use when modifying the resource's QP |
| 23 |
:param mr_access: The MR access to use when registering the resource's MR |
| 24 |
""" |
| 25 |
atomic_access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | \ |
| 26 |
ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC |
| 27 |
self.qp_access = qp_access if qp_access else atomic_access |
| 28 |
self.mr_access = mr_access if mr_access else atomic_access |
| 29 |
super().__init__(dev_name=dev_name, ib_port=ib_port, |
| 30 |
gid_index=gid_index) |
| 31 |
self.msg_size = msg_size |
| 32 |
self.new_mr_lkey = None |
| 33 |
|
| 34 |
def create_mr(self): |
| 35 |
try: |
| 36 |
self.mr = MR(self.pd, self.msg_size, self.mr_access) |
| 37 |
except PyverbsRDMAError as ex: |
| 38 |
if ex.error_code == errno.EOPNOTSUPP: |
| 39 |
raise unittest.SkipTest(f'Reg MR with access ({self.mr_access}) is not supported') |
| 40 |
raise ex |
| 41 |
|
| 42 |
def create_qp_init_attr(self): |
| 43 |
return QPInitAttr(qp_type=ibv_qp_type.IBV_QPT_RC, scq=self.cq, sq_sig_all=0, |
| 44 |
rcq=self.cq, srq=self.srq, cap=self.create_qp_cap()) |
| 45 |
|
| 46 |
def create_qp_attr(self): |
| 47 |
qp_attr = QPAttr(port_num=self.ib_port) |
| 48 |
qp_attr.qp_access_flags = self.qp_access |
| 49 |
return qp_attr |
| 50 |
|
| 51 |
@property |
| 52 |
def mr_lkey(self): |
| 53 |
return self.new_mr_lkey if self.new_mr_lkey is not None else self.mr.lkey |
| 54 |
|
| 55 |
|
| 56 |
class XRCAtomic(XRCResources): |
| 57 |
def create_mr(self): |
| 58 |
try: |
| 59 |
atomic_access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | \ |
| 60 |
ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC |
| 61 |
self.mr = MR(self.pd, self.msg_size, atomic_access) |
| 62 |
except PyverbsRDMAError as ex: |
| 63 |
if ex.error_code == errno.EOPNOTSUPP: |
| 64 |
raise unittest.SkipTest(f'Reg MR with access ({atomic_access}) is not supported') |
| 65 |
raise ex |
| 66 |
|
| 67 |
|
| 68 |
class AtomicTest(RDMATestCase): |
| 69 |
""" |
| 70 |
Test various functionalities of the DM class. |
| 71 |
""" |
| 72 |
def setUp(self): |
| 73 |
super().setUp() |
| 74 |
self.iters = 10 |
| 75 |
self.server = None |
| 76 |
self.client = None |
| 77 |
self.traffic_args = None |
| 78 |
ctx = d.Context(name=self.dev_name) |
| 79 |
if ctx.query_device().atomic_caps == ibv_atomic_cap.IBV_ATOMIC_NONE: |
| 80 |
raise unittest.SkipTest('Atomic operations are not supported') |
| 81 |
|
| 82 |
def test_atomic_cmp_and_swap(self): |
| 83 |
self.create_players(RCAtomic) |
| 84 |
u.atomic_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_ATOMIC_CMP_AND_SWP) |
| 85 |
u.atomic_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_ATOMIC_CMP_AND_SWP, |
| 86 |
receiver_val=1, sender_val=1) |
| 87 |
|
| 88 |
def test_atomic_fetch_and_add(self): |
| 89 |
self.create_players(RCAtomic) |
| 90 |
u.atomic_traffic(**self.traffic_args, |
| 91 |
send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD) |
| 92 |
|
| 93 |
def test_xrc_atomic_fetch_and_add(self): |
| 94 |
self.create_players(XRCAtomic) |
| 95 |
u.atomic_traffic(**self.traffic_args, |
| 96 |
send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD) |
| 97 |
|
| 98 |
def test_xrc_atomic_cmp_and_swap(self): |
| 99 |
self.create_players(XRCAtomic) |
| 100 |
u.atomic_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_ATOMIC_CMP_AND_SWP) |
| 101 |
u.atomic_traffic(**self.traffic_args, send_op=ibv_wr_opcode.IBV_WR_ATOMIC_CMP_AND_SWP, |
| 102 |
receiver_val=1, sender_val=1) |
| 103 |
|
| 104 |
def test_atomic_invalid_qp_access(self): |
| 105 |
self.create_players(RCAtomic, qp_access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE) |
| 106 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 107 |
u.atomic_traffic(**self.traffic_args, |
| 108 |
send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD) |
| 109 |
|
| 110 |
def test_atomic_invalid_mr_access(self): |
| 111 |
self.create_players(RCAtomic, mr_access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE) |
| 112 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 113 |
u.atomic_traffic(**self.traffic_args, |
| 114 |
send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD) |
| 115 |
|
| 116 |
def test_atomic_non_aligned_addr(self): |
| 117 |
self.create_players(RCAtomic, msg_size=9) |
| 118 |
self.client.raddr += 1 |
| 119 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 120 |
u.atomic_traffic(**self.traffic_args, |
| 121 |
send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD) |
| 122 |
|
| 123 |
def test_atomic_invalid_lkey(self): |
| 124 |
self.create_players(RCAtomic) |
| 125 |
self.client.new_mr_lkey = self.client.mr.lkey + 1 |
| 126 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 127 |
u.atomic_traffic(**self.traffic_args, |
| 128 |
send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD) |
| 129 |
|
| 130 |
def test_atomic_invalid_rkey(self): |
| 131 |
self.create_players(RCAtomic) |
| 132 |
self.client.rkey += 1 |
| 133 |
with self.assertRaises(PyverbsRDMAError) as ex: |
| 134 |
u.atomic_traffic(**self.traffic_args, |
| 135 |
send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD) |