| 1 |
import unittest |
| 2 |
import datetime |
| 3 |
import errno |
| 4 |
|
| 5 |
from tests.base import RCResources, RDMATestCase, PyverbsAPITestCase |
| 6 |
from pyverbs.providers.mlx5.mlx5dv import Mlx5Context |
| 7 |
from pyverbs.pyverbs_error import PyverbsRDMAError |
| 8 |
from pyverbs.cq import CqInitAttrEx, CQEX |
| 9 |
from tests.test_flow import FlowRes |
| 10 |
from pyverbs.qp import QPInitAttr |
| 11 |
from pyverbs.cq import PollCqAttr |
| 12 |
from pyverbs.libibverbs_enums import ibv_qp_type, ibv_wc_status, ibv_wr_opcode, ibv_create_cq_wc_flags, \ |
| 13 |
IBV_WC_STANDARD_FLAGS |
| 14 |
import tests.utils as u |
| 15 |
|
| 16 |
|
| 17 |
GIGA = 1000000000 |
| 18 |
|
| 19 |
|
| 20 |
def convert_ts_to_ns(ctx, device_ts): |
| 21 |
""" |
| 22 |
Convert device timestamp from HCA core clock units to corresponding |
| 23 |
nanosecond counts. |
| 24 |
:param ctx: The context that gets this timestamp. |
| 25 |
:param device_ts: The device timestamp to translate. |
| 26 |
:return: Timestamp in nanoseconds |
| 27 |
""" |
| 28 |
try: |
| 29 |
timestamp_in_ns = Mlx5Context.device_timestamp_to_ns(ctx, device_ts) |
| 30 |
except PyverbsRDMAError as ex: |
| 31 |
if ex.error_code == errno.EOPNOTSUPP: |
| 32 |
raise unittest.SkipTest('Converting timestamp to nanoseconds is not supported') |
| 33 |
raise ex |
| 34 |
return timestamp_in_ns |
| 35 |
|
| 36 |
|
| 37 |
def timestamp_res_cls(base_class): |
| 38 |
""" |
| 39 |
This is a factory function which creates a class that inherits base_class of |
| 40 |
any BaseResources type. |
| 41 |
:param base_class: The base resources class to inherit from. |
| 42 |
:return: TimeStampRes class. |
| 43 |
""" |
| 44 |
class TimeStampRes(base_class): |
| 45 |
def __init__(self, dev_name, ib_port, gid_index, qp_type, send_ts=None, |
| 46 |
recv_ts=None): |
| 47 |
self.qp_type = qp_type |
| 48 |
self.send_ts = send_ts |
| 49 |
self.recv_ts = recv_ts |
| 50 |
self.timestamp = None |
| 51 |
self.scq = None |
| 52 |
self.rcq = None |
| 53 |
super().__init__(dev_name=dev_name, ib_port=ib_port, gid_index=gid_index) |
| 54 |
|
| 55 |
def create_cq(self): |
| 56 |
self.scq = self._create_ex_cq(self.send_ts) |
| 57 |
self.rcq = self._create_ex_cq(self.recv_ts) |
| 58 |
|
| 59 |
def _create_ex_cq(self, timestamp=None): |
| 60 |
""" |
| 61 |
Create an Extended CQ. |
| 62 |
:param timestamp: If set, the timestamp type to use. |
| 63 |
""" |
| 64 |
wc_flags = IBV_WC_STANDARD_FLAGS |
| 65 |
if timestamp: |
| 66 |
wc_flags |= timestamp |
| 67 |
cia = CqInitAttrEx(cqe=self.num_msgs, wc_flags=wc_flags) |
| 68 |
try: |
| 69 |
cq = CQEX(self.ctx, cia) |
| 70 |
except PyverbsRDMAError as ex: |
| 71 |
if ex.error_code == errno.EOPNOTSUPP: |
| 72 |
raise unittest.SkipTest('Create Extended CQ is not supported') |
| 73 |
raise ex |
| 74 |
return cq |
| 75 |
|
| 76 |
def create_qp_init_attr(self): |
| 77 |
return QPInitAttr(qp_type=self.qp_type, scq=self.scq, |
| 78 |
rcq=self.rcq, srq=self.srq, cap=self.create_qp_cap()) |
| 79 |
|
| 80 |
return TimeStampRes |
| 81 |
|
| 82 |
|
| 83 |
class TimeStampTest(RDMATestCase): |
| 84 |
""" |
| 85 |
Test various types of timestamping formats. |
| 86 |
""" |
| 87 |
def setUp(self): |
| 88 |
super().setUp() |
| 89 |
self.send_ts = None |
| 90 |
self.recv_ts = None |
| 91 |
self.qp_type = None |
| 92 |
|
| 93 |
@property |
| 94 |
def resource_arg(self): |
| 95 |
return {'send_ts': self.send_ts, 'recv_ts': self.recv_ts, |
| 96 |
'qp_type': self.qp_type} |
| 97 |
|
| 98 |
def test_timestamp_free_running_rc_traffic(self): |
| 99 |
""" |
| 100 |
Test free running timestamp on RC traffic. |
| 101 |
""" |
| 102 |
self.qp_type = ibv_qp_type.IBV_QPT_RC |
| 103 |
self.send_ts = self.recv_ts = ibv_create_cq_wc_flags.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP |
| 104 |
self.create_players(timestamp_res_cls(RCResources), **self.resource_arg) |
| 105 |
self.ts_traffic() |
| 106 |
timestamp = convert_ts_to_ns(self.client.ctx, self.client.timestamp) |
| 107 |
self.verify_ts(timestamp) |
| 108 |
|
| 109 |
def test_timestamp_real_time_rc_traffic(self): |
| 110 |
""" |
| 111 |
Test real time timestamp on RC traffic. |
| 112 |
""" |
| 113 |
self.qp_type = ibv_qp_type.IBV_QPT_RC |
| 114 |
self.send_ts = self.recv_ts = ibv_create_cq_wc_flags.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK |
| 115 |
self.create_players(timestamp_res_cls(RCResources), **self.resource_arg) |
| 116 |
self.ts_traffic() |
| 117 |
self.verify_ts(self.client.timestamp) |
| 118 |
|
| 119 |
def test_timestamp_free_running_send_raw_traffic(self): |
| 120 |
""" |
| 121 |
Test timestamping on RAW traffic only on the send completions. |
| 122 |
""" |
| 123 |
self.qp_type = ibv_qp_type.IBV_QPT_RAW_PACKET |
| 124 |
self.send_ts = ibv_create_cq_wc_flags.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP |
| 125 |
self.create_players(timestamp_res_cls(FlowRes), **self.resource_arg) |
| 126 |
self.flow = self.server.create_flow([self.server.create_eth_spec()]) |
| 127 |
self.ts_traffic() |
| 128 |
timestamp = convert_ts_to_ns(self.client.ctx, self.client.timestamp) |
| 129 |
self.verify_ts(timestamp) |
| 130 |
|
| 131 |
def test_timestamp_free_running_recv_raw_traffic(self): |
| 132 |
""" |
| 133 |
Test timestamping on RAW traffic only on the recv completions. |
| 134 |
""" |
| 135 |
self.qp_type = ibv_qp_type.IBV_QPT_RAW_PACKET |
| 136 |
self.recv_ts = ibv_create_cq_wc_flags.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP |
| 137 |
self.create_players(timestamp_res_cls(FlowRes), **self.resource_arg) |
| 138 |
self.flow = self.server.create_flow([self.server.create_eth_spec()]) |
| 139 |
self.ts_traffic() |
| 140 |
timestamp = convert_ts_to_ns(self.server.ctx, self.server.timestamp) |
| 141 |
self.verify_ts(timestamp) |
| 142 |
|
| 143 |
def test_timestamp_real_time_raw_traffic(self): |
| 144 |
""" |
| 145 |
Test real time timestamp on RAW traffic. |
| 146 |
""" |
| 147 |
self.qp_type = ibv_qp_type.IBV_QPT_RAW_PACKET |
| 148 |
self.send_ts = self.recv_ts = ibv_create_cq_wc_flags.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK |
| 149 |
self.create_players(timestamp_res_cls(FlowRes), **self.resource_arg) |
| 150 |
self.flow = self.server.create_flow([self.server.create_eth_spec()]) |
| 151 |
self.ts_traffic() |
| 152 |
self.verify_ts(self.client.timestamp) |
| 153 |
|
| 154 |
@staticmethod |
| 155 |
def verify_ts(timestamp): |
| 156 |
""" |
| 157 |
Verify that the timestamp is a valid value of time. |
| 158 |
""" |
| 159 |
datetime.datetime.fromtimestamp(timestamp/GIGA) |
| 160 |
|
| 161 |
@staticmethod |
| 162 |
def poll_cq_ex_ts(cqex, ts_type=None): |
| 163 |
""" |
| 164 |
Poll completion from the extended CQ. |
| 165 |
:param cqex: CQEX to poll from |
| 166 |
:param ts_type: If set, read the CQE timestamp in this format |
| 167 |
:return: The CQE timestamp if it requested. |
| 168 |
""" |
| 169 |
polling_timeout = 10 |
| 170 |
start = datetime.datetime.now() |
| 171 |
ts = 0 |
| 172 |
|
| 173 |
poll_attr = PollCqAttr() |
| 174 |
ret = cqex.start_poll(poll_attr) |
| 175 |
while ret == 2 and (datetime.datetime.now() - start).seconds < polling_timeout: |
| 176 |
ret = cqex.start_poll(poll_attr) |
| 177 |
if ret == 2: |
| 178 |
raise PyverbsRDMAError('Failed to poll CQEX - Got timeout') |
| 179 |
if ret != 0: |
| 180 |
raise PyverbsRDMAError('Failed to poll CQEX') |
| 181 |
if cqex.status != ibv_wc_status.IBV_WC_SUCCESS: |
| 182 |
raise PyverbsRDMAError('Completion status is {cqex.status}') |
| 183 |
if ts_type == ibv_create_cq_wc_flags.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP: |
| 184 |
ts = cqex.read_timestamp() |
| 185 |
if ts_type == ibv_create_cq_wc_flags.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK: |
| 186 |
ts = cqex.read_completion_wallclock_ns() |
| 187 |
cqex.end_poll() |
| 188 |
return ts |
| 189 |
|
| 190 |
def ts_traffic(self): |
| 191 |
""" |
| 192 |
Run RDMA traffic and read the completions timestamps. |
| 193 |
""" |
| 194 |
s_recv_wr = u.get_recv_wr(self.server) |
| 195 |
u.post_recv(self.server, s_recv_wr) |
| 196 |
if self.qp_type == ibv_qp_type.IBV_QPT_RAW_PACKET: |
| 197 |
c_send_wr, _, _ = u.get_send_elements_raw_qp(self.client) |
| 198 |
else: |
| 199 |
c_send_wr, _ = u.get_send_elements(self.client, False) |
| 200 |
u.send(self.client, c_send_wr, ibv_wr_opcode.IBV_WR_SEND, False, 0) |
| 201 |
self.client.timestamp = self.poll_cq_ex_ts(self.client.scq, ts_type=self.send_ts) |
| 202 |
self.server.timestamp = self.poll_cq_ex_ts(self.server.rcq, ts_type=self.recv_ts) |
| 203 |
|
| 204 |
|
| 205 |
class TimeAPITest(PyverbsAPITestCase): |
| 206 |
def test_query_rt_values(self): |
| 207 |
""" |
| 208 |
Test the ibv_query_rt_values_ex API. |
| 209 |
Query the device real-time values, convert them to ns and verify that |
| 210 |
the timestamp is a valid value of time.. |
| 211 |
""" |
| 212 |
try: |
| 213 |
_, hw_time = self.ctx.query_rt_values_ex() |
| 214 |
time_in_ns = convert_ts_to_ns(self.ctx, hw_time) |
| 215 |
datetime.datetime.fromtimestamp(time_in_ns/GIGA) |
| 216 |
except PyverbsRDMAError as ex: |
| 217 |
if ex.error_code == errno.EOPNOTSUPP: |
| 218 |
raise unittest.SkipTest('Query device real time is not supported') |
| 219 |
raise ex |