Debian Codegraph @

tests/test_mlx5_dc.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2020 NVIDIA Corporation . All rights reserved. See COPYING file
3
4 import unittest
5 import errno
6 import time
7
8 from tests.mlx5_base import Mlx5DcResources, Mlx5RDMATestCase, Mlx5DcStreamsRes
9 from pyverbs.pyverbs_error import PyverbsRDMAError
10 from pyverbs.providers.mlx5.mlx5dv import Mlx5QP
11 from pyverbs.libibverbs_enums import ibv_access_flags, ibv_qp_create_send_ops_flags, ibv_wr_opcode, \
12 ibv_odp_transport_cap_bits, ibv_qp_attr_mask, ibv_qp_state
13 import tests.utils as u
14
15
16 class OdpDc(Mlx5DcResources):
17 def create_mr(self):
18 try:
19 self.mr = u.create_custom_mr(self, ibv_access_flags.IBV_ACCESS_ON_DEMAND)
20 except PyverbsRDMAError as ex:
21 if ex.error_code == errno.EOPNOTSUPP:
22 raise unittest.SkipTest('Reg ODP MR is not supported')
23 raise ex
24
25
26 class DCTest(Mlx5RDMATestCase):
27 def setUp(self):
28 super().setUp()
29 self.iters = 10
30 self.server = None
31 self.client = None
32 self.traffic_args = None
33
34 def sync_remote_attr(self):
35 """
36 Exchange the remote attributes between the server and the client.
37 """
38 super().sync_remote_attr()
39 self.client.remote_dct_num = self.server.dct_qp.qp_num
40 self.server.remote_dct_num = self.client.dct_qp.qp_num
41
42 def test_dc_rdma_write(self):
43 self.create_players(Mlx5DcResources, qp_count=2,
44 send_ops_flags=ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE)
45 u.rdma_traffic(**self.traffic_args, new_send=True,
46 send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
47
48 def test_dc_send(self):
49 self.create_players(Mlx5DcResources, qp_count=2,
50 send_ops_flags=ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND)
51 u.traffic(**self.traffic_args, new_send=True,
52 send_op=ibv_wr_opcode.IBV_WR_SEND)
53
54 def test_dc_atomic(self):
55 self.create_players(Mlx5DcResources, qp_count=2,
56 send_ops_flags=ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
57 client_max_log = self.client.ctx.query_mlx5_device().max_dc_rd_atom
58 server_max_log = self.server.ctx.query_mlx5_device().max_dc_rd_atom
59 u.atomic_traffic(**self.traffic_args, new_send=True,
60 send_op=ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD,
61 client_wr=client_max_log, server_wr=server_max_log)
62
63 def test_dc_ah_to_qp_mapping(self):
64 self.create_players(Mlx5DcResources, qp_count=2,
65 send_ops_flags=ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND)
66 client_ah = u.get_global_ah(self.client, self.gid_index, self.ib_port)
67 try:
68 Mlx5QP.map_ah_to_qp(client_ah, self.server.qps[0].qp_num)
69 except PyverbsRDMAError as ex:
70 if ex.error_code == errno.EOPNOTSUPP:
71 raise unittest.SkipTest('Mapping AH to QP is not supported')
72 raise ex
73 u.traffic(**self.traffic_args, new_send=True,
74 send_op=ibv_wr_opcode.IBV_WR_SEND)
75
76 def check_odp_dc_support(self):
77 """
78 Check if the device supports ODP with DC.
79 :raises SkipTest: In case ODP is not supported with DC
80 """
81 dc_odp_caps = self.server.ctx.query_mlx5_device().dc_odp_caps
82 required_odp_caps = ibv_odp_transport_cap_bits.IBV_ODP_SUPPORT_SEND | \
83 ibv_odp_transport_cap_bits.IBV_ODP_SUPPORT_SRQ_RECV
84 if required_odp_caps & dc_odp_caps != required_odp_caps:
85 raise unittest.SkipTest('ODP is not supported using DC')
86
87 def test_odp_dc_traffic(self):
88 send_ops_flag = ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND
89 self.create_players(OdpDc, qp_count=2, send_ops_flags=send_ops_flag)
90 self.check_odp_dc_support()
91 u.traffic(**self.traffic_args, new_send=True,
92 send_op=ibv_wr_opcode.IBV_WR_SEND)
93
94 def test_dc_rdma_write_stream(self):
95 """
96 Check good flow of DCS.
97 Calculate stream_id for DCS test by setting same stream id
98 twice for WR and after increase it. Setting goes by loop
99 and after stream_id is more than number of concurrent
100 streams + 1 then stream_id returns to 1.
101 :raises SkipTest: In case DCI is not supported with HW
102 """
103 self.create_players(Mlx5DcStreamsRes, qp_count=2,
104 send_ops_flags=ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE)
105 u.rdma_traffic(**self.traffic_args, new_send=True,
106 send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
107
108 def test_dc_stream_qp_recovery(self):
109 """
110 Test DC QP error state transition with stream channel error accumulation.
111 Creates DC QPs with restricted MR access and generates remote access errors
112 via RDMA_WRITE operations. Verifies QP transitions to ERR state after enough
113 channels entered error mode. Validates QP recovery after reset.
114 """
115 self.create_players(Mlx5DcStreamsRes, qp_count=2,
116 send_ops_flags=ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE,
117 mr_access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
118 qp_idx = 0
119 error_threshold = self.client.dcis[qp_idx]['errored']
120 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND)
121 for _ in range(error_threshold):
122 with self.assertRaisesRegex(PyverbsRDMAError, r'Remote access error'):
123 u.rdma_traffic(**self.traffic_args, new_send=True,
124 send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
125 # Poll QP state with timeout: it takes time to transition QP to ERR after errors
126 qp_in_err_state = False
127 start = time.perf_counter()
128 while time.perf_counter() - start < 1.0:
129 qp_attr, _ = self.client.qps[qp_idx].query(ibv_qp_attr_mask.IBV_QP_STATE)
130 if qp_attr.cur_qp_state == ibv_qp_state.IBV_QPS_ERR:
131 qp_in_err_state = True
132 break
133 if not qp_in_err_state:
134 raise PyverbsRDMAError(f'QP is not in ERR state after {error_threshold} errors')
135 for qp_idx in range(self.client.qp_count):
136 self.client.reset_qp(qp_idx)
137 for qp_idx in range(self.server.qp_count):
138 self.server.reset_qp(qp_idx)
139 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND)
140
141 def test_dc_stream_ids_recovery(self):
142 """
143 Test DC stream ID reset functionality after remote access errors.
144 Creates DC QPs with restricted MR access and generates
145 remote access errors via RDMA_WRITE operations. After each error, resets
146 the stream ID and verifies QP remains functional.
147 Validates normal SEND traffic continues to work after stream resets.
148 """
149 self.create_players(Mlx5DcStreamsRes, qp_count=2,
150 send_ops_flags=ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE,
151 mr_access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
152 qp_idx = 0
153 error_threshold = self.client.dcis[qp_idx]['errored']
154 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND)
155 for _ in range(error_threshold):
156 with self.assertRaisesRegex(PyverbsRDMAError, r'Remote access error'):
157 u.rdma_traffic(**self.traffic_args, new_send=True,
158 send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
159 self.client.dci_reset_stream_id(qp_idx)
160 qp_attr, _ = self.client.qps[qp_idx].query(ibv_qp_attr_mask.IBV_QP_STATE)
161 if qp_attr.cur_qp_state == ibv_qp_state.IBV_QPS_ERR:
162 raise PyverbsRDMAError('QP is in ERR state after reset stream id')
163 u.traffic(**self.traffic_args, new_send=True, send_op=ibv_wr_opcode.IBV_WR_SEND)