Debian Codegraph @

tests/test_mlx5_mkey.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2020 Nvidia, Inc. All rights reserved. See COPYING file
3
4 import unittest
5 import random
6 import errno
7
8 from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \
9 Mlx5DVQPInitAttr, Mlx5QP
10 from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError, \
11 PyverbsError
12 from pyverbs.providers.mlx5.mlx5dv_mkey import Mlx5Mkey, Mlx5MrInterleaved, \
13 Mlx5MkeyConfAttr, Mlx5SigT10Dif, Mlx5SigCrc, Mlx5SigBlockDomain, \
14 Mlx5SigBlockAttr
15 from tests.base import RCResources, RDMATestCase
16 from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_mkey_init_attr_flags, mlx5dv_qp_create_flags, \
17 mlx5dv_qp_create_send_ops_flags, mlx5dv_qp_init_attr_mask, mlx5dv_sig_t10dif_flags, \
18 mlx5dv_sig_t10dif_bg_type, mlx5dv_sig_type, mlx5dv_block_size, mlx5dv_sig_mask, mlx5dv_sig_crc_type, \
19 mlx5dv_mkey_err_type
20 from pyverbs.wr import SGE, SendWR, RecvWR
21 from pyverbs.qp import QPInitAttrEx, QPCap, QPAttr
22 from pyverbs.mr import MR
23 from pyverbs.libibverbs_enums import ibv_access_flags, ibv_qp_create_send_ops_flags, ibv_qp_init_attr_mask, \
24 ibv_qp_type, ibv_odp_transport_cap_bits, ibv_send_flags, ibv_wr_opcode, ibv_event_type, ibv_qp_state, \
25 ibv_qp_attr_mask
26 import tests.utils as u
27
28
29 class Mlx5MkeyResources(RCResources):
30 def __init__(self, dev_name, ib_port, gid_index, dv_send_ops_flags=0,
31 mkey_create_flags=mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT,
32 dv_qp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE):
33 self.dv_send_ops_flags = dv_send_ops_flags
34 self.mkey_create_flags = mkey_create_flags
35 self.dv_qp_create_flags = dv_qp_create_flags
36 if dv_send_ops_flags & mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE:
37 self.max_inline_data = 512
38 else:
39 self.max_inline_data = 0
40
41 self.qp_access_flags = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE
42 self.send_ops_flags = ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_SEND
43 # The signature pipelining tests use RDMA_WRITE. Allow RDMA_WRITE
44 # if the pipelining flag is enabled for the QP.
45 if self.dv_qp_create_flags & mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_SIG_PIPELINING:
46 self.qp_access_flags |= ibv_access_flags.IBV_ACCESS_REMOTE_WRITE
47 self.send_ops_flags |= ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_RDMA_WRITE
48
49 super().__init__(dev_name, ib_port, gid_index)
50 self.create_mkey()
51
52 def create_context(self):
53 mlx5dv_attr = Mlx5DVContextAttr()
54 try:
55 self.ctx = Mlx5Context(mlx5dv_attr, name=self.dev_name)
56 except PyverbsUserError as ex:
57 raise unittest.SkipTest(f'Could not open mlx5 context ({ex})')
58 except PyverbsRDMAError:
59 raise unittest.SkipTest('Opening mlx5 context is not supported')
60
61 def create_mkey(self):
62 try:
63 self.mkey = Mlx5Mkey(self.pd, self.mkey_create_flags, 3)
64 except PyverbsRDMAError as ex:
65 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
66 raise unittest.SkipTest('Create Mkey is not supported')
67 raise ex
68
69 def create_qp_cap(self):
70 return QPCap(max_send_wr=self.num_msgs, max_recv_wr=self.num_msgs,
71 max_inline_data=self.max_inline_data)
72
73 def create_qp_init_attr(self):
74 comp_mask = ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD | ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS
75 return QPInitAttrEx(cap=self.create_qp_cap(), pd=self.pd, scq=self.cq,
76 rcq=self.cq, qp_type=ibv_qp_type.IBV_QPT_RC,
77 send_ops_flags=self.send_ops_flags,
78 comp_mask=comp_mask)
79
80 def create_qp_attr(self):
81 attr = super().create_qp_attr()
82 attr.qp_access_flags = self.qp_access_flags
83 return attr
84
85 def create_qps(self):
86 try:
87 qp_init_attr = self.create_qp_init_attr()
88 comp_mask = mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS |\
89 mlx5dv_qp_init_attr_mask.MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS
90 attr = Mlx5DVQPInitAttr(comp_mask=comp_mask,
91 create_flags=self.dv_qp_create_flags,
92 send_ops_flags=self.dv_send_ops_flags)
93 qp = Mlx5QP(self.ctx, qp_init_attr, attr)
94 self.qps.append(qp)
95 self.qps_num.append(qp.qp_num)
96 self.psns.append(random.getrandbits(24))
97 except PyverbsRDMAError as ex:
98 if ex.error_code == errno.EOPNOTSUPP:
99 raise unittest.SkipTest('Create Mlx5DV QP is not supported')
100 raise ex
101
102
103 class Mlx5MkeyOdpRes(Mlx5MkeyResources):
104 @u.requires_odp('rc', ibv_odp_transport_cap_bits.IBV_ODP_SUPPORT_SEND | ibv_odp_transport_cap_bits.IBV_ODP_SUPPORT_RECV)
105 def create_mr(self):
106 self.mr = MR(self.pd, self.msg_size, ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | ibv_access_flags.IBV_ACCESS_ON_DEMAND)
107
108
109 class Mlx5MkeyTest(RDMATestCase):
110 """
111 Test various functionalities of the mlx5 mkeys.
112 """
113 def setUp(self):
114 super().setUp()
115 self.iters = 10
116 self.server = None
117 self.client = None
118
119 def reg_mr_list(self, configure_mkey=False):
120 """
121 Register a list of SGEs using the player's mkeys.
122 :param configure_mkey: If True, use the mkey configuration API.
123 """
124 for player in [self.server, self.client]:
125 player.qp.wr_start()
126 player.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED | ibv_send_flags.IBV_SEND_INLINE
127 sge_1 = SGE(player.mr.buf, 8, player.mr.lkey)
128 sge_2 = SGE(player.mr.buf + 64, 8, player.mr.lkey)
129 if configure_mkey:
130 player.qp.wr_mkey_configure(player.mkey, 2, Mlx5MkeyConfAttr())
131 player.qp.wr_set_mkey_access_flags(ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
132 player.qp.wr_set_mkey_layout_list([sge_1, sge_2])
133 else:
134 player.qp.wr_mr_list(player.mkey, ibv_access_flags.IBV_ACCESS_LOCAL_WRITE,
135 sge_list=[sge_1, sge_2])
136 player.qp.wr_complete()
137 u.poll_cq(player.cq)
138
139 def reg_mr_interleaved(self, configure_mkey=False):
140 """
141 Register an interleaved memory layout using the player's mkeys.
142 :param configure_mkey: Use the mkey configuration API.
143 """
144 for player in [self.server, self.client]:
145 player.qp.wr_start()
146 player.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED | ibv_send_flags.IBV_SEND_INLINE
147 mr_interleaved_1 = Mlx5MrInterleaved(addr=player.mr.buf, bytes_count=8,
148 bytes_skip=2, lkey=player.mr.lkey)
149 mr_interleaved_2 = Mlx5MrInterleaved(addr=player.mr.buf + 64, bytes_count=8,
150 bytes_skip=2, lkey=player.mr.lkey)
151 mr_interleaved_lst = [mr_interleaved_1, mr_interleaved_2]
152 mkey_access = ibv_access_flags.IBV_ACCESS_LOCAL_WRITE
153 if configure_mkey:
154 player.qp.wr_mkey_configure(player.mkey, 2, Mlx5MkeyConfAttr())
155 player.qp.wr_set_mkey_access_flags(mkey_access)
156 player.qp.wr_set_mkey_layout_interleaved(3, mr_interleaved_lst)
157 else:
158 player.qp.wr_mr_interleaved(player.mkey, ibv_access_flags.IBV_ACCESS_LOCAL_WRITE,
159 repeat_count=3, mr_interleaved_lst=mr_interleaved_lst)
160 player.qp.wr_complete()
161 u.poll_cq(player.cq)
162
163 def reg_mr_sig_t10dif(self):
164 """
165 Register the player's mkeys with T10DIF signature on the wire domain.
166 """
167 for player in [self.server, self.client]:
168 player.qp.wr_start()
169 player.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED | ibv_send_flags.IBV_SEND_INLINE
170 sge = SGE(player.mr.buf, 512, player.mr.lkey)
171 player.qp.wr_mkey_configure(player.mkey, 3, Mlx5MkeyConfAttr())
172 player.qp.wr_set_mkey_access_flags(ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
173 player.qp.wr_set_mkey_layout_list([sge])
174
175 t10dif_flags = mlx5dv_sig_t10dif_flags.MLX5DV_SIG_T10DIF_FLAG_REF_REMAP
176 sig_t10dif = Mlx5SigT10Dif(bg_type=mlx5dv_sig_t10dif_bg_type.MLX5DV_SIG_T10DIF_CRC,
177 bg=0xFFFF, app_tag=0xABCD,
178 ref_tag=0x01234567, flags=t10dif_flags)
179
180 sig_type = mlx5dv_sig_type.MLX5DV_SIG_TYPE_T10DIF
181 block_size = mlx5dv_block_size.MLX5DV_BLOCK_SIZE_512
182 sig_block_domain = Mlx5SigBlockDomain(sig_type=sig_type,
183 dif=sig_t10dif,
184 block_size=block_size)
185
186 check_mask = (mlx5dv_sig_mask.MLX5DV_SIG_MASK_T10DIF_GUARD |
187 mlx5dv_sig_mask.MLX5DV_SIG_MASK_T10DIF_APPTAG |
188 mlx5dv_sig_mask.MLX5DV_SIG_MASK_T10DIF_REFTAG)
189 sig_attr = Mlx5SigBlockAttr(wire=sig_block_domain,
190 check_mask=check_mask)
191 player.qp.wr_set_mkey_sig_block(sig_attr)
192 player.qp.wr_complete()
193 u.poll_cq(player.cq)
194
195 def reg_mr_sig_crc(self):
196 """
197 Register the player's mkeys with CRC32 signature on the wire domain.
198 """
199 for player in [self.server, self.client]:
200 player.qp.wr_start()
201 player.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED | ibv_send_flags.IBV_SEND_INLINE
202 sge = SGE(player.mr.buf, 512, player.mr.lkey)
203 player.qp.wr_mkey_configure(player.mkey, 3, Mlx5MkeyConfAttr())
204 player.qp.wr_set_mkey_access_flags(ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
205 player.qp.wr_set_mkey_layout_list([sge])
206
207 sig_crc = Mlx5SigCrc(crc_type=mlx5dv_sig_crc_type.MLX5DV_SIG_CRC_TYPE_CRC32,
208 seed=0xFFFFFFFF)
209 sig_block_domain = Mlx5SigBlockDomain(sig_type=mlx5dv_sig_type.MLX5DV_SIG_TYPE_CRC,
210 crc=sig_crc,
211 block_size=mlx5dv_block_size.MLX5DV_BLOCK_SIZE_512)
212 sig_attr = Mlx5SigBlockAttr(wire=sig_block_domain,
213 check_mask=mlx5dv_sig_mask.MLX5DV_SIG_MASK_CRC32)
214 player.qp.wr_set_mkey_sig_block(sig_attr)
215 player.qp.wr_complete()
216 u.poll_cq(player.cq)
217
218 def reg_mr_sig_err(self):
219 """
220 Register the player's mkeys with an SGE and CRC32 signature on
221 the memory domain. Data transport operation with these MKEYs will cause
222 a signature error because the test does not fill out the signature in
223 the memory buffer.
224 """
225
226 sig_crc = Mlx5SigCrc(crc_type=mlx5dv_sig_crc_type.MLX5DV_SIG_CRC_TYPE_CRC32,
227 seed=0xFFFFFFFF)
228
229 block_size = mlx5dv_block_size.MLX5DV_BLOCK_SIZE_512
230 sig_block_domain = Mlx5SigBlockDomain(sig_type=mlx5dv_sig_type.MLX5DV_SIG_TYPE_CRC,
231 crc=sig_crc,
232 block_size=block_size)
233 sig_attr = Mlx5SigBlockAttr(mem=sig_block_domain,
234 check_mask=mlx5dv_sig_mask.MLX5DV_SIG_MASK_CRC32)
235
236 # Configure the mkey on the server side
237 self.server.qp.wr_start()
238 self.server.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED | ibv_send_flags.IBV_SEND_INLINE
239 sge = SGE(self.server.mr.buf, 512, self.server.mr.lkey)
240 self.server.qp.wr_mkey_configure(self.server.mkey, 2,
241 Mlx5MkeyConfAttr())
242 self.server.qp.wr_set_mkey_access_flags(ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
243 self.server.qp.wr_set_mkey_layout_list([sge])
244 self.server.qp.wr_complete()
245 u.poll_cq(self.server.cq)
246
247 # Configure the mkey on the client side
248 self.client.qp.wr_start()
249 self.client.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED | ibv_send_flags.IBV_SEND_INLINE
250 sge = SGE(self.client.mr.buf, 512 + 4, self.client.mr.lkey)
251 self.client.qp.wr_mkey_configure(self.client.mkey, 3,
252 Mlx5MkeyConfAttr())
253 self.client.qp.wr_set_mkey_access_flags(ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
254 self.client.qp.wr_set_mkey_layout_list([sge])
255 self.client.qp.wr_set_mkey_sig_block(sig_attr)
256 self.client.qp.wr_complete()
257 u.poll_cq(self.client.cq)
258
259 def reg_mr_sig_pipelining_server(self):
260 """
261 Register mkey without signature.
262 """
263 self.server.qp.wr_start()
264 self.server.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED | ibv_send_flags.IBV_SEND_INLINE
265 sge = SGE(self.server.mr.buf, 512, self.server.mr.lkey)
266 self.server.qp.wr_mkey_configure(self.server.mkey, 2,
267 Mlx5MkeyConfAttr())
268 self.server.qp.wr_set_mkey_access_flags(ibv_access_flags.IBV_ACCESS_LOCAL_WRITE |
269 ibv_access_flags.IBV_ACCESS_REMOTE_WRITE)
270 self.server.qp.wr_set_mkey_layout_list([sge])
271 self.server.qp.wr_complete()
272 u.poll_cq(self.server.cq)
273
274 def reg_mr_sig_pipelining_client(self, check_mask=0):
275 """
276 Register mkey with CRC32 signature in memory domain and no
277 signature in wire domain.
278 :param check_mask: The mask for the signature checking.
279 """
280 self.client.qp.wr_start()
281 self.client.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED | ibv_send_flags.IBV_SEND_INLINE
282 # Add 4 bytes for CRC32 signature
283 sge = SGE(self.client.mr.buf, 512 + 4, self.client.mr.lkey)
284 self.client.qp.wr_mkey_configure(self.client.mkey, 3,
285 Mlx5MkeyConfAttr())
286 self.client.qp.wr_set_mkey_access_flags(ibv_access_flags.IBV_ACCESS_LOCAL_WRITE)
287 self.client.qp.wr_set_mkey_layout_list([sge])
288
289 sig = Mlx5SigCrc(crc_type = mlx5dv_sig_crc_type.MLX5DV_SIG_CRC_TYPE_CRC32)
290 sig_domain = Mlx5SigBlockDomain(sig_type=mlx5dv_sig_type.MLX5DV_SIG_TYPE_CRC,
291 crc=sig,
292 block_size=mlx5dv_block_size.MLX5DV_BLOCK_SIZE_512)
293 sig_attr = Mlx5SigBlockAttr(mem=sig_domain,
294 check_mask=check_mask)
295 self.client.qp.wr_set_mkey_sig_block(sig_attr)
296 self.client.qp.wr_complete()
297 u.poll_cq(self.client.cq)
298
299 def build_traffic_elements(self, sge_size):
300 """
301 Build the server and client send/recv work requests.
302 :param sge_size: The sge send size using the mkey.
303 """
304 opcode = ibv_wr_opcode.IBV_WR_SEND
305 server_sge = SGE(0, sge_size, self.server.mkey.lkey)
306 self.server_recv_wr = RecvWR(sg=[server_sge], num_sge=1)
307 client_sge = SGE(0, sge_size, self.client.mkey.lkey)
308 self.client_send_wr = SendWR(opcode=opcode, num_sge=1, sg=[client_sge])
309
310 def build_traffic_elements_sig_pipelining(self):
311 """
312 Build two WRs for data and response on client side and one WR for
313 response on server side. Transaction consists of two operations:
314 RDMA write of data and send/recv of response. Data size is
315 512 bytes, response size is 16 bytes. For simplicity the same
316 memory is used for data and response. Data is transferred using
317 the player signature mkey. Response is transferred using the
318 plain MR.
319 """
320 server_sge_resp = SGE(self.server.mr.buf, 16, self.server.mr.lkey)
321 self.server_resp_wr = RecvWR(sg=[server_sge_resp], num_sge=1)
322 client_sge_data = SGE(0, 512, self.client.mkey.lkey)
323 self.client_data_wr = SendWR(wr_id=1, opcode=ibv_wr_opcode.IBV_WR_RDMA_WRITE,
324 num_sge=1, sg=[client_sge_data],
325 send_flags=0)
326 self.client_data_wr.set_wr_rdma(self.server.mkey.rkey, 0)
327 client_sge_resp = SGE(self.client.mr.buf, 16, self.client.mr.lkey)
328 client_send_flags = ibv_send_flags.IBV_SEND_SIGNALED | ibv_send_flags.IBV_SEND_FENCE
329 self.client_resp_wr = SendWR(wr_id=1, opcode=ibv_wr_opcode.IBV_WR_SEND, num_sge=1,
330 sg=[client_sge_resp],
331 send_flags=client_send_flags)
332
333 def traffic(self, sge_size, exp_buffer):
334 """
335 Perform RC traffic using the mkey.
336 :param sge_size: The sge size using the mkey.
337 :param exp_buffer: The expected result of the receive buffer after
338 the traffic operation.
339 """
340 self.build_traffic_elements(sge_size)
341 self.server.qp.post_recv(self.server_recv_wr)
342 for _ in range(self.iters):
343 self.server.mr.write('s' * self.server.msg_size,
344 self.server.msg_size)
345 self.client.mr.write('c' * self.client.msg_size,
346 self.client.msg_size)
347 self.client.qp.post_send(self.client_send_wr)
348 u.poll_cq(self.client.cq)
349 u.poll_cq(self.server.cq)
350 self.server.qp.post_recv(self.server_recv_wr)
351 act_buffer = self.server.mr.read(len(exp_buffer), 0).decode()
352 if act_buffer != exp_buffer:
353 raise PyverbsError('Data validation failed: expected '
354 f'{exp_buffer}, received {act_buffer}')
355
356 def traffic_scattered_data(self, sge_size=16):
357 exp_buffer=((('c' * 8 + 's' * 56) *2)[:100])
358 self.traffic(sge_size=sge_size, exp_buffer=exp_buffer)
359
360 def traffic_sig(self):
361 exp_buffer=('c' * 512 + 's' * (self.server.msg_size - 512))
362 self.traffic(sge_size=512, exp_buffer=exp_buffer)
363
364 def invalidate_mkeys(self):
365 """
366 Invalidate the players mkey.
367 """
368 for player in [self.server, self.client]:
369 inv_send_wr = SendWR(opcode=ibv_wr_opcode.IBV_WR_LOCAL_INV)
370 inv_send_wr.imm_data = player.mkey.lkey
371 player.qp.post_send(inv_send_wr)
372 u.poll_cq(player.cq)
373
374 def invalidate_mkeys_remotely(self):
375 """
376 Client remotely invalidates the server's rkey
377 """
378 sge = SGE(0,0, self.server.mkey.lkey)
379 self.server.qp.post_recv(RecvWR(sg=[sge], num_sge=1))
380 self.client.qp.wr_start()
381 self.client.qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED
382 self.client.qp.wr_send_inv(self.server.mkey.rkey)
383 sge = SGE(0, 0, self.client.mkey.lkey)
384 self.client.qp.wr_set_sge(sge)
385 self.client.qp.wr_complete()
386 u.poll_cq(self.client.cq)
387 u.poll_cq(self.server.cq)
388
389 def test_mkey_remote_invalidate(self):
390 """
391 Verify remote Mkey invalidation.
392 Create Mkey, traffic using this mkey then the client invalidates
393 the server's mkey remotly.
394 """
395 self.create_players(Mlx5MkeyResources,
396 mkey_create_flags=mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT |
397 mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_REMOTE_INVALIDATE,
398 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE)
399 self.reg_mr_list(configure_mkey=True)
400 self.traffic_scattered_data()
401 self.invalidate_mkeys_remotely()
402 with self.assertRaises(PyverbsRDMAError):
403 self.traffic_scattered_data()
404
405 def check_mkey(self, player, expected=mlx5dv_mkey_err_type.MLX5DV_MKEY_NO_ERR):
406 """
407 Check the player's mkey for a signature error.
408 param player: Player to check.
409 param expected: The expected result of the checking.
410 """
411 mkey_err = player.mkey.mkey_check()
412 if mkey_err.err_type != expected:
413 raise PyverbsRDMAError('MKEY check failed: '
414 f'expected err_type: {expected}, '
415 f'actual err_type: {mkey_err.err_type}')
416
417 def test_mkey_interleaved(self):
418 """
419 Create Mkeys, register an interleaved memory layout using this mkey and
420 then perform traffic using it.
421 """
422 self.create_players(Mlx5MkeyResources,
423 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MR_INTERLEAVED)
424 self.reg_mr_interleaved()
425 self.traffic_scattered_data()
426 self.invalidate_mkeys()
427
428 def test_mkey_list(self):
429 """
430 Create Mkeys, register a memory layout using this mkey and then perform
431 traffic using this mkey.
432 """
433 self.create_players(Mlx5MkeyResources,
434 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MR_LIST)
435 self.reg_mr_list()
436 self.traffic_scattered_data()
437 self.invalidate_mkeys()
438
439 def test_mkey_list_new_api(self):
440 """
441 Create Mkeys, configure it with memory layout using the new API and
442 traffic using this mkey.
443 """
444 self.create_players(Mlx5MkeyResources,
445 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE)
446 self.reg_mr_list(configure_mkey=True)
447 self.traffic_scattered_data()
448 self.invalidate_mkeys()
449
450 def test_odp_mkey_list_new_api(self):
451 """
452 Create Mkeys above ODP MR, configure it with memory layout using the new API and
453 traffic using this mkey.
454 """
455 self.create_players(Mlx5MkeyOdpRes,
456 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE)
457 self.reg_mr_list(configure_mkey=True)
458 self.traffic_scattered_data()
459 self.invalidate_mkeys()
460
461 def test_mkey_interleaved_new_api(self):
462 """
463 Create Mkeys, configure it with interleaved memory layout using the new
464 API and then perform traffic using it.
465 """
466 self.create_players(Mlx5MkeyResources,
467 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE)
468 self.reg_mr_interleaved(configure_mkey=True)
469 self.traffic_scattered_data()
470 self.invalidate_mkeys()
471
472 def test_mkey_list_bad_flow(self):
473 """
474 Create Mkeys, register a memory layout using this mkey and then try to
475 access the memory out of the mkey defined region. Expect this case to
476 fail.
477 """
478 self.create_players(Mlx5MkeyResources,
479 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MR_LIST)
480 self.reg_mr_list()
481 with self.assertRaises(PyverbsRDMAError) as ex:
482 self.traffic_scattered_data(sge_size=100)
483
484 def test_mkey_sig_t10dif(self):
485 """
486 Create Mkeys, configure it with T10DIF signature and traffic using
487 this mkey.
488 """
489 self.create_players(Mlx5MkeyResources,
490 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE,
491 mkey_create_flags=mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT |
492 mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE)
493 self.reg_mr_sig_t10dif()
494 self.traffic_sig()
495 self.check_mkey(self.server)
496 self.check_mkey(self.client)
497 self.invalidate_mkeys()
498
499 def test_mkey_sig_crc(self):
500 """
501 Create Mkeys, configure it with CRC32 signature and traffic using
502 this mkey.
503 """
504 self.create_players(Mlx5MkeyResources,
505 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE,
506 mkey_create_flags=mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT |
507 mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE)
508 self.reg_mr_sig_crc()
509 self.traffic_sig()
510 self.check_mkey(self.server)
511 self.check_mkey(self.client)
512 self.invalidate_mkeys()
513
514 def test_mkey_sig_err(self):
515 """
516 Test the signature error handling flow. Create Mkeys, configure it
517 CRC32 signature on the memory domain but do not set a valid signature
518 in the memory buffer. Run traffic using this mkey, ensure that the
519 signature error is detected.
520 """
521 self.create_players(Mlx5MkeyResources,
522 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE,
523 mkey_create_flags=mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT |
524 mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE)
525 self.reg_mr_sig_err()
526 # The test supports only one iteration because mkey re-registration
527 # is required after each signature error.
528 self.iters = 1
529 self.traffic_sig()
530 self.check_mkey(self.client, mlx5dv_mkey_err_type.MLX5DV_MKEY_SIG_BLOCK_BAD_GUARD)
531 self.check_mkey(self.server)
532 self.invalidate_mkeys()
533
534 def test_mkey_sig_pipelining_good(self):
535 """
536 Test the good signature pipelining scenario.
537 """
538 self.create_players(Mlx5MkeyResources,
539 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE,
540 mkey_create_flags=mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT |
541 mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE,
542 dv_qp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE |
543 mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_SIG_PIPELINING)
544 self.reg_mr_sig_pipelining_client()
545 self.reg_mr_sig_pipelining_server()
546 self.build_traffic_elements_sig_pipelining()
547
548 self.server.qp.post_recv(self.server_resp_wr)
549 self.client.qp.post_send(self.client_data_wr)
550 self.client.qp.post_send(self.client_resp_wr)
551
552 u.poll_cq(self.client.cq)
553 u.poll_cq(self.server.cq)
554
555 def test_mkey_sig_pipelining_bad(self):
556 """
557 Test the bad signature pipelining scenario.
558 """
559 self.create_players(Mlx5MkeyResources,
560 dv_send_ops_flags=mlx5dv_qp_create_send_ops_flags.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE,
561 mkey_create_flags=mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT |
562 mlx5dv_mkey_init_attr_flags.MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE,
563 dv_qp_create_flags=mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE |
564 mlx5dv_qp_create_flags.MLX5DV_QP_CREATE_SIG_PIPELINING)
565 self.reg_mr_sig_pipelining_client(check_mask=mlx5dv_sig_mask.MLX5DV_SIG_MASK_CRC32)
566 self.reg_mr_sig_pipelining_server()
567 self.build_traffic_elements_sig_pipelining()
568
569 self.server.qp.post_recv(self.server_resp_wr)
570 self.client.qp.post_send(self.client_data_wr)
571 self.client.qp.post_send(self.client_resp_wr)
572
573 # Expect SQ_DRAINED event
574 event = self.client.ctx.get_async_event()
575 event.ack()
576 self.assertEqual(event.event_type, ibv_event_type.IBV_EVENT_SQ_DRAINED)
577 # No completion is expected on the client side
578 nc, _ = self.client.cq.poll(1)
579 self.assertEqual(nc, 0)
580 # No completion is expected on the server side
581 nc, _ = self.server.cq.poll(1)
582 self.assertEqual(nc, 0)
583 self.check_mkey(self.client, mlx5dv_mkey_err_type.MLX5DV_MKEY_SIG_BLOCK_BAD_GUARD)
584 self.check_mkey(self.server)
585
586 # Cancel and repost response WR
587 canceled_count = self.client.qp.cancel_posted_send_wrs(1)
588 self.assertEqual(canceled_count, 1)
589 self.client.qp.post_send(self.client_resp_wr)
590
591 # Move QP back to RTS and receive completions
592 self.client.qp.modify(QPAttr(qp_state=ibv_qp_state.IBV_QPS_RTS,
593 cur_qp_state=ibv_qp_state.IBV_QPS_SQD),
594 ibv_qp_attr_mask.IBV_QP_STATE | ibv_qp_attr_mask.IBV_QP_CUR_STATE)
595 u.poll_cq(self.client.cq)
596 u.poll_cq(self.server.cq)