Debian Codegraph @

tests/utils.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
3 # Copyright (c) 2020 Intel Corporation. All rights reserved. See COPYING file
4 """
5 Provide some useful helper function for pyverbs' tests.
6 """
7 from itertools import combinations as com
8 import errno
9 import subprocess
10 import unittest
11 import random
12 import socket
13 import struct
14 import string
15 import glob
16 import time
17 import os
18
19 from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError, PyverbsUserError
20 from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr
21 from pyverbs.qp import QPCap, QPInitAttr, QPInitAttrEx, QPAttr, QPEx, QP
22 from tests.mlx5_base import Mlx5DcResources, Mlx5DcStreamsRes
23 from tests.base import XRCResources, DCT_KEY, MLNX_VENDOR_ID
24 from pyverbs.addr import AHAttr, AH, GlobalRoute
25 from pyverbs.providers.efa.efadv import EfaCQ
26 from pyverbs.wr import SGE, SendWR, RecvWR
27 from pyverbs.base import PyverbsRDMAErrno
28 from tests.efa_base import SRDResources
29 from pyverbs.cq import PollCqAttr, CQEX
30 from pyverbs.mr import MW, MWBindInfo
31 from pyverbs.mem_alloc import madvise
32 import pyverbs.device as d
33 from pyverbs.libibverbs_enums import ibv_access_flags, ibv_odp_general_caps, ibv_device_cap_flags, \
34 ibv_atomic_cap, ibv_qp_init_attr_mask, ibv_raw_packet_caps, ibv_qp_create_flags, ibv_qp_type, \
35 ibv_wr_opcode, ibv_send_flags, ibv_mw_type, ibv_wc_status, ibv_wc_flags, ibv_wc_opcode, \
36 ibv_selectivity_level, ibv_odp_general_caps, ibv_qp_state, ibv_qp_attr_mask, ibv_qp_create_send_ops_flags, \
37 IBV_LINK_LAYER_ETHERNET, _IBV_DEVICE_RAW_SCATTER_FCS, _IBV_DEVICE_PCI_WRITE_END_PADDING, \
38 _IBV_ADVISE_MR_FLAG_FLUSH, _IBV_ADVISE_MR_ADVICE_PREFETCH_WRITE, ibv_tph_mem_type, \
39 ibv_dmah_init_attr_mask
40 from pyverbs.mr import MR, MREx, DmaHandleInitAttr
41
42
43 MAX_MR_SIZE = 4194304
44 # Some HWs limit DM address and length alignment to 4 for read and write
45 # operations. Use a minimal length and alignment that respect that.
46 # For creation purposes use random alignments. As this is log2 of address
47 # alignment, no need for large numbers.
48 MIN_DM_SIZE = 4
49 DM_ALIGNMENT = 4
50 MIN_DM_LOG_ALIGN = 0
51 MAX_DM_LOG_ALIGN = 6
52 # Raw Packet QP supports TSO header, which creates a larger send WQE.
53 MAX_RAW_PACKET_SEND_WR = 2500
54 GRH_SIZE = 40
55 IMM_DATA = 1234
56 POLL_CQ_TIMEOUT = 10 # In seconds
57
58
59 class MatchCriteriaEnable:
60 NONE = 0
61 OUTER = 1
62 MISC = 1 << 1
63 INNER = 1 << 2
64 MISC_2 = 1 << 3
65 MISC_3 = 1 << 4
66
67
68 class PacketConsts:
69 """
70 Class to hold constant packets' values.
71 """
72 ETHER_HEADER_SIZE = 14
73 IPV4_HEADER_SIZE = 20
74 IPV6_HEADER_SIZE = 40
75 UDP_HEADER_SIZE = 8
76 TCP_HEADER_SIZE = 20
77 VLAN_HEADER_SIZE = 4
78 TCP_HEADER_SIZE_WORDS = 5
79 IP_V4 = 4
80 IP_V6 = 6
81 TCP_PROTO = 'tcp'
82 UDP_PROTO = 'udp'
83 IP_V4_FLAGS = 2 # Don't fragment is set
84 TTL_HOP_LIMIT = 64
85 IHL = 5
86 # Hardcoded values for flow matchers
87 ETHER_TYPE_ETH = 0x6558
88 ETHER_TYPE_IPV4 = 0x800
89 MAC_MASK = "ff:ff:ff:ff:ff:ff"
90 ETHER_TYPE_IPV6 = 0x86DD
91 SRC_MAC = "24:8a:07:a5:28:c8"
92 # DST mac must be multicast
93 DST_MAC = "01:50:56:19:20:a7"
94 SRC_IP = "1.1.1.1"
95 DST_IP = "2.2.2.2"
96 SRC_PORT = 1234
97 DST_PORT = 5678
98 SRC_IP6 = "a0a1::a2a3:a4a5:a6a7:a8a9"
99 DST_IP6 = "b0b1::b2b3:b4b5:b6b7:b8b9"
100 SEQ_NUM = 1
101 WINDOW_SIZE = 65535
102 VXLAN_PORT = 4789
103 VXLAN_VNI = 7777777
104 VXLAN_FLAGS = 0x8
105 VXLAN_HEADER_SIZE = 8
106 VLAN_TPID = 0x8100
107 VLAN_PRIO = 5
108 VLAN_CFI = 1
109 VLAN_ID = 0xc0c
110 GRE_VER = 1
111 GRE_FLAGS = 2
112 GRE_KEY = 0x12345678
113 GENEVE_VNI = 2
114 GENEVE_OAM = 0
115 GENEVE_PORT = 6081
116 BTH_HEADER_SIZE = 16
117 BTH_OPCODE = 0x81
118 BTH_DST_QP = 0xd2
119 BTH_A = 0x1
120 BTH_PARTITION_KEY = 0xffff
121 BTH_BECN = 1
122 ROCE_PORT = 4791
123
124
125 def create_dmah_init_attr(cpu_id=0, tph_mem_type=ibv_tph_mem_type.IBV_TPH_MEM_TYPE_VM, ph=0):
126 """
127 Helper function to create DmaHandleInitAttr with standard parameters.
128
129 :param cpu_id: CPU ID (default: 0)
130 :param tph_mem_type: TPH memory type (default: IBV_TPH_MEM_TYPE_VM)
131 :param ph: Processing hint (default: 0)
132 :return: DmaHandleInitAttr object
133 """
134 return DmaHandleInitAttr(
135 comp_mask=(ibv_dmah_init_attr_mask.IBV_DMAH_INIT_ATTR_MASK_CPU_ID |
136 ibv_dmah_init_attr_mask.IBV_DMAH_INIT_ATTR_MASK_TPH_MEM_TYPE |
137 ibv_dmah_init_attr_mask.IBV_DMAH_INIT_ATTR_MASK_PH),
138 cpu_id=cpu_id,
139 tph_mem_type=tph_mem_type,
140 ph=ph)
141
142
143 def get_mr_length():
144 """
145 Provide a random value for MR length. We avoid large buffers as these
146 allocations typically fails.
147 We use random.random() instead of randrange() or randint() due to
148 performance issues when generating very large pseudo random numbers.
149 :return: A random MR length
150 """
151 return int(MAX_MR_SIZE * random.random())
152
153
154 def filter_illegal_access_flags(element):
155 """
156 Helper function to filter illegal access flags combinations
157 :param element: A list of access flags to check
158 :return: True if this list is legal, else False
159 """
160 if ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC in element or ibv_access_flags.IBV_ACCESS_REMOTE_WRITE in element:
161 if not ibv_access_flags.IBV_ACCESS_LOCAL_WRITE in element:
162 return False
163 return True
164
165
166 def get_access_flags(ctx):
167 """
168 Provide an array of random legal access flags for an MR.
169 Since remote write and remote atomic require local write permission, if
170 one of them is randomly selected without local write, local write will be
171 added as well.
172 After verifying that the flags selection is legal, it is appended to an
173 array, assuming it wasn't previously appended.
174 :param ctx: Device Context to check capabilities
175 :param num: Size of initial collection
176 :return: A random legal value for MR flags
177 """
178 attr = ctx.query_device()
179 attr_ex = ctx.query_device_ex()
180 vals = list(ibv_access_flags)
181 if not attr_ex.odp_caps.general_caps & ibv_odp_general_caps.IBV_ODP_SUPPORT:
182 vals.remove(ibv_access_flags.IBV_ACCESS_ON_DEMAND)
183 if not attr.device_cap_flags & ibv_device_cap_flags.IBV_DEVICE_MEM_WINDOW:
184 vals.remove(ibv_access_flags.IBV_ACCESS_MW_BIND)
185 if not attr.atomic_caps & ibv_atomic_cap.IBV_ATOMIC_HCA:
186 vals.remove(ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC)
187 arr = []
188 for i in range(1, len(vals)):
189 tmp = list(com(vals, i))
190 tmp = filter(filter_illegal_access_flags, tmp)
191 for t in tmp: # Iterate legal combinations and bitwise OR them
192 val = 0
193 for flag in t:
194 val += flag.value
195 arr.append(val)
196 return arr
197
198
199 def get_dmabuf_access_flags(ctx):
200 """
201 Similar to get_access_flags, except that dma-buf MR only support
202 a subset of the flags.
203 :param ctx: Device Context to check capabilities
204 :return: A random legal value for MR flags
205 """
206 attr = ctx.query_device()
207 vals = [ibv_access_flags.IBV_ACCESS_LOCAL_WRITE, ibv_access_flags.IBV_ACCESS_REMOTE_WRITE,
208 ibv_access_flags.IBV_ACCESS_REMOTE_READ, ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC,
209 ibv_access_flags.IBV_ACCESS_RELAXED_ORDERING]
210 if not attr.atomic_caps & ibv_atomic_cap.IBV_ATOMIC_HCA:
211 vals.remove(ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC)
212 arr = []
213 for i in range(1, len(vals)):
214 tmp = list(com(vals, i))
215 tmp = filter(filter_illegal_access_flags, tmp)
216 for t in tmp: # Iterate legal combinations and bitwise OR them
217 val = 0
218 for flag in t:
219 val += flag.value
220 arr.append(val)
221 return arr
222
223
224 def get_dm_attrs(dm_len):
225 """
226 Initializes an AllocDmAttr member with the given length and random
227 alignment. It currently sets comp_mask = 0 since other comp_mask values
228 are not supported.
229 :param dm_len:
230 :return: An initialized AllocDmAttr object
231 """
232 align = random.randint(MIN_DM_LOG_ALIGN, MAX_DM_LOG_ALIGN)
233 return d.AllocDmAttr(dm_len, align, 0)
234
235
236 def sample(coll):
237 """
238 Returns a random-length subset of the given collection.
239 :param coll: The collection to sample
240 :return: A subset of <collection>
241 """
242 return random.sample(coll, int((len(coll) + 1) * random.random()))
243
244
245 def random_qp_cap(attr):
246 """
247 Initializes a QPCap object with valid values based on the device's
248 attributes.
249 It doesn't check the max WR limits since they're reported for smaller WR
250 sizes.
251 :return: A QPCap object
252 """
253 # We use significantly smaller values than those in device attributes.
254 # The attributes reported by the device don't take into account possible
255 # larger WQEs that include e.g. memory window.
256 send_wr = random.randint(1, int(attr.max_qp_wr / 8))
257 recv_wr = random.randint(1, int(attr.max_qp_wr / 8))
258 send_sge = random.randint(1, int(attr.max_sge / 2))
259 recv_sge = random.randint(1, int(attr.max_sge / 2))
260 return QPCap(send_wr, recv_wr, send_sge, recv_sge)
261
262
263 def random_qp_create_mask(qpt, attr_ex):
264 """
265 Select a random sublist of ibv_qp_init_attr_mask. Some of the options are
266 not yet supported by pyverbs and will not be returned. TSO support is
267 checked for the device and the QP type. If it doesn't exist, TSO will not
268 be set.
269 :param qpt: Current QP type
270 :param attr_ex: Extended device attributes for capability checks
271 :return: A sublist of ibv_qp_init_attr_mask
272 """
273 has_tso = attr_ex.tso_caps.max_tso > 0 and \
274 attr_ex.tso_caps.supported_qpts & 1 << qpt
275 supp_flags = [ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_CREATE_FLAGS,
276 ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_MAX_TSO_HEADER]
277 # Either PD or XRCD flag is needed, XRCD is not supported yet
278 selected = sample(supp_flags)
279 selected.append(ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD)
280 if ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_MAX_TSO_HEADER in selected and not has_tso:
281 selected.remove(ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_MAX_TSO_HEADER)
282 mask = 0
283 for s in selected:
284 mask += s.value
285 return mask
286
287
288 def get_create_qp_flags_raw_packet(attr_ex):
289 """
290 Select random QP creation flags for Raw Packet QP. Filter out unsupported
291 flags prior to selection.
292 :param attr_ex: Device extended attributes to check capabilities
293 :return: A random combination of QP creation flags
294 """
295 has_fcs = attr_ex.device_cap_flags_ex & _IBV_DEVICE_RAW_SCATTER_FCS
296 has_cvlan = attr_ex.raw_packet_caps & ibv_raw_packet_caps.IBV_RAW_PACKET_CAP_CVLAN_STRIPPING
297 has_padding = attr_ex.device_cap_flags_ex & _IBV_DEVICE_PCI_WRITE_END_PADDING
298 l = list(ibv_qp_create_flags)
299 l.remove(ibv_qp_create_flags.IBV_QP_CREATE_SOURCE_QPN) # UD only
300 if not has_fcs:
301 l.remove(ibv_qp_create_flags.IBV_QP_CREATE_SCATTER_FCS)
302 if not has_cvlan:
303 l.remove(ibv_qp_create_flags.IBV_QP_CREATE_CVLAN_STRIPPING)
304 if not has_padding:
305 l.remove(ibv_qp_create_flags.IBV_QP_CREATE_PCI_WRITE_END_PADDING)
306 flags = sample(l)
307 val = 0
308 for i in flags:
309 val |= i.value
310 return val
311
312
313 def random_valid_qp_create_flags(qpt, attr, attr_ex):
314 """
315 Select a random sublist of ibv_qp_create_flags according to the QP type.
316 :param qpt: Current QP type
317 :param attr_ex: Used for Raw Packet QP to check device capabilities
318 :return: A sublist of ibv_qp_create_flags
319 """
320 # Most HCAs doesn't support any create_flags so far except mlx4/mlx5
321 if attr.vendor_id != MLNX_VENDOR_ID:
322 return 0
323
324 if qpt == ibv_qp_type.IBV_QPT_RAW_PACKET:
325 return get_create_qp_flags_raw_packet(attr_ex)
326 elif qpt == ibv_qp_type.IBV_QPT_UD:
327 # IBV_QP_CREATE_SOURCE_QPN is only supported by mlx5 driver and is not
328 # to be check in unittests.
329 return random.choice([0, 2]) # IBV_QP_CREATE_BLOCK_SELF_MCAST_LB
330 else:
331 return 0
332
333
334 def random_qp_init_attr_ex(attr_ex, attr, qpt=None):
335 """
336 Create a random-valued QPInitAttrEx object with the given QP type.
337 QP type affects QP capabilities, so allow users to set it and still get
338 valid attributes.
339 :param attr_ex: Extended device attributes for capability checks
340 :param attr: Device attributes for capability checks
341 :param qpt: Requested QP type
342 :return: A valid initialized QPInitAttrEx object
343 """
344 max_tso = 0
345 if qpt is None:
346 qpt = random.choice([ibv_qp_type.IBV_QPT_RC, ibv_qp_type.IBV_QPT_UC, ibv_qp_type.IBV_QPT_UD,
347 ibv_qp_type.IBV_QPT_RAW_PACKET])
348 qp_cap = random_qp_cap(attr)
349 if qpt == ibv_qp_type.IBV_QPT_RAW_PACKET and \
350 qp_cap.max_send_wr > MAX_RAW_PACKET_SEND_WR:
351 qp_cap.max_send_wr = MAX_RAW_PACKET_SEND_WR
352 sig = random.randint(0, 1)
353 mask = random_qp_create_mask(qpt, attr_ex)
354 if mask & ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_CREATE_FLAGS:
355 cflags = random_valid_qp_create_flags(qpt, attr, attr_ex)
356 else:
357 cflags = 0
358 if mask & ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_MAX_TSO_HEADER:
359 if qpt != ibv_qp_type.IBV_QPT_RAW_PACKET:
360 mask -= ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_MAX_TSO_HEADER
361 else:
362 max_tso = \
363 random.randint(16, int(attr_ex.tso_caps.max_tso / 800))
364 qia = QPInitAttrEx(qp_type=qpt, cap=qp_cap, sq_sig_all=sig, comp_mask=mask,
365 create_flags=cflags, max_tso_header=max_tso)
366 if mask & ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_MAX_TSO_HEADER:
367 # TSO increases send WQE size, let's be on the safe side
368 qia.cap.max_send_sge = 2
369 return qia
370
371
372 def get_qp_init_attr(cq, attr):
373 """
374 Creates a QPInitAttr object with a QP type of the provided <qpts> array and
375 other random values.
376 :param cq: CQ to be used as send and receive CQ
377 :param attr: Device attributes for capability checks
378 :return: An initialized QPInitAttr object
379 """
380 qp_cap = random_qp_cap(attr)
381 sig = random.randint(0, 1)
382 return QPInitAttr(scq=cq, rcq=cq, cap=qp_cap, sq_sig_all=sig)
383
384
385 def create_qp_ex(agr_obj, qp_type, send_flags):
386 if qp_type == ibv_qp_type.IBV_QPT_XRC_SEND:
387 cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=0, max_recv_sge=0,
388 max_send_sge=1)
389 else:
390 cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=agr_obj.num_msgs,
391 max_recv_sge=1, max_send_sge=1)
392 qia = QPInitAttrEx(cap=cap, qp_type=qp_type, scq=agr_obj.cq,
393 rcq=agr_obj.cq, pd=agr_obj.pd, send_ops_flags=send_flags,
394 comp_mask=ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_PD |
395 ibv_qp_init_attr_mask.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)
396 qp_attr = QPAttr(port_num=agr_obj.ib_port)
397 if qp_type == ibv_qp_type.IBV_QPT_UD:
398 qp_attr.qkey = agr_obj.UD_QKEY
399 qp_attr.pkey_index = agr_obj.UD_PKEY_INDEX
400 if qp_type == ibv_qp_type.IBV_QPT_RC:
401 qp_attr.qp_access_flags = ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \
402 ibv_access_flags.IBV_ACCESS_REMOTE_READ | \
403 ibv_access_flags.IBV_ACCESS_REMOTE_ATOMIC | \
404 ibv_access_flags.IBV_ACCESS_FLUSH_GLOBAL | \
405 ibv_access_flags.IBV_ACCESS_FLUSH_PERSISTENT
406 try:
407 # We don't have capability bits for this
408 qp = QPEx(agr_obj.ctx, qia, qp_attr)
409 except PyverbsRDMAError as ex:
410 if ex.error_code == errno.EOPNOTSUPP:
411 raise unittest.SkipTest('Extended QP is not supported on this device')
412 raise ex
413 if qp_type != ibv_qp_type.IBV_QPT_XRC_SEND:
414 agr_obj.qps.append(qp)
415 agr_obj.qps_num.append(qp.qp_num)
416 agr_obj.psns.append(random.getrandbits(24))
417 else:
418 return qp
419
420
421 def wc_status_to_str(status):
422 try:
423 return \
424 {0: 'Success', 1: 'Local length error',
425 2: 'local QP operation error', 3: 'Local EEC operation error',
426 4: 'Local protection error', 5: 'WR flush error',
427 6: 'Memory window bind error', 7: 'Bad response error',
428 8: 'Local access error', 9: 'Remote invalidate request error',
429 10: 'Remote access error', 11: 'Remote operation error',
430 12: 'Retry exceeded', 13: 'RNR retry exceeded',
431 14: 'Local RDD violation error',
432 15: 'Remote invalidate RD request error',
433 16: 'Remote aort error', 17: 'Invalidate EECN error',
434 18: 'Invalidate EEC state error', 19: 'Fatal error',
435 20: 'Response timeout error', 21: 'General error'}[status]
436 except KeyError:
437 return 'Unknown WC status ({s})'.format(s=status)
438
439
440 def create_custom_mr(agr_obj, additional_access_flags=0, size=None, user_addr=None, mr_ex=False):
441 """
442 Creates a memory region using the aggregation object's PD.
443 If size is None, the agr_obj's message size is used to set the MR's size.
444 The access flags are local write and the additional_access_flags.
445 :param agr_obj: The aggregation object that creates the MR
446 :param additional_access_flags: Addition access flags to set in the MR
447 :param size: MR's length. If None, agr_obj.msg_size is used.
448 :param user_addr: The MR's buffer address. If None, the buffer will be allocated by pyverbs.
449 :param mr_ex: Whether to create an MREx MR instead of a regular MR
450 """
451 mr_length = size if size else agr_obj.msg_size
452 try:
453 if mr_ex:
454 return MREx(agr_obj.pd, mr_length,
455 ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | additional_access_flags,
456 address=user_addr)
457 return MR(agr_obj.pd, mr_length,
458 ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | additional_access_flags, address=user_addr)
459 except PyverbsRDMAError as ex:
460 if ex.error_code == errno.EOPNOTSUPP:
461 raise unittest.SkipTest(f'Create custom mr with additional access flags {additional_access_flags} is not supported')
462 raise ex
463
464 # Traffic helpers
465
466 def get_send_elements(agr_obj, is_server, opcode=ibv_wr_opcode.IBV_WR_SEND):
467 """
468 Creates a single SGE and a single Send WR for agr_obj's QP type. The content
469 of the message is either 's' for server side or 'c' for client side.
470 :param agr_obj: Aggregation object which contains all resources necessary
471 :param is_server: Indicates whether this is server or client side
472 :return: send wr and its SGE
473 """
474 if hasattr(agr_obj, 'use_mixed_mr') and agr_obj.use_mixed_mr:
475 return get_send_elements_mixed_mr(agr_obj, is_server, opcode)
476 if opcode == ibv_wr_opcode.IBV_WR_ATOMIC_WRITE:
477 atomic_wr = agr_obj.msg_size * (b's' if is_server else b'c')
478 return None, atomic_wr
479
480 qp_type = agr_obj.sqp_lst[0].qp_type if isinstance(agr_obj, XRCResources) \
481 else agr_obj.qp.qp_type
482 offset = GRH_SIZE if qp_type == ibv_qp_type.IBV_QPT_UD else 0
483 msg = (agr_obj.msg_size + offset) * ('s' if is_server else 'c')
484 agr_obj.mem_write(msg, agr_obj.msg_size + offset)
485 sge = SGE(agr_obj.mr.buf + offset, agr_obj.msg_size, agr_obj.mr_lkey)
486 send_wr = SendWR(opcode=opcode, num_sge=1, sg=[sge])
487 if opcode in [ibv_wr_opcode.IBV_WR_RDMA_WRITE, ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM, ibv_wr_opcode.IBV_WR_RDMA_READ]:
488 send_wr.set_wr_rdma(int(agr_obj.rkey), int(agr_obj.raddr))
489 return send_wr, sge
490
491
492 def get_send_elements_mixed_mr(agr_obj, is_server, opcode=ibv_wr_opcode.IBV_WR_SEND):
493 """
494 Creates 2 SGEs and a single Send WR for agr_obj's QP type. There are 2 messages,
495 one for each MR. The content of the message is either 's' for server side or 'c'
496 for client side.
497 :param agr_obj: Aggregation object which contains all resources necessary
498 :param is_server: Indicates whether this is server or client side
499 :param opcode: send WR opcode
500 :return: send wr and its SG list
501 """
502 msg = (agr_obj.msg_size) * ('s' if is_server else 'c')
503 agr_obj.mr.write(msg, agr_obj.msg_size)
504 agr_obj.non_odp_mr.write(msg, agr_obj.msg_size)
505 sge1 = SGE(agr_obj.mr.buf, agr_obj.msg_size, agr_obj.mr.lkey)
506 sge2 = SGE(agr_obj.non_odp_mr.buf, agr_obj.msg_size, agr_obj.non_odp_mr.lkey)
507 send_wr = SendWR(opcode=opcode, num_sge=2, sg=[sge1, sge2])
508 return send_wr, [sge1, sge2]
509
510
511 def get_recv_wr(agr_obj):
512 """
513 Creates a single SGE Recv WR for agr_obj's QP type. In case of mixed MRs,
514 creates 2 SGEs accordingly.
515 :param agr_obj: Aggregation object which contains all resources necessary
516 :return: recv wr
517 """
518 qp_type = agr_obj.rqp_lst[0].qp_type if isinstance(agr_obj, XRCResources) \
519 else agr_obj.qp.qp_type if isinstance(agr_obj.qp, QP) else None
520 mr = agr_obj.mr
521 length = agr_obj.msg_size + GRH_SIZE if qp_type == ibv_qp_type.IBV_QPT_UD \
522 else agr_obj.msg_size
523 recv_sgl = [SGE(mr.buf, length, mr.lkey)]
524 if hasattr(agr_obj, 'use_mixed_mr') and agr_obj.use_mixed_mr:
525 sec_mr = agr_obj.non_odp_mr
526 recv_sgl.append(SGE(sec_mr.buf,length,sec_mr.lkey))
527 return RecvWR(sg=recv_sgl, num_sge=len(recv_sgl))
528
529
530 def get_global_ah(agr_obj, gid_index, port):
531 gr = GlobalRoute(dgid=agr_obj.ctx.query_gid(port, gid_index),
532 sgid_index=gid_index)
533 ah_attr = AHAttr(port_num=port, is_global=1, gr=gr,
534 dlid=agr_obj.port_attr.lid)
535 return AH(agr_obj.pd, attr=ah_attr)
536
537
538 def get_global_route(ctx, gid_index=0, port_num=1):
539 """
540 Queries the provided Context's gid <gid_index> and creates a GlobalRoute
541 object with sgid_index <gid_index> and the queried GID as dgid.
542 :param ctx: Context object to query
543 :param gid_index: GID index to query and use. Default: 0, as it's always
544 valid
545 :param port_num: Number of the port to query. Default: 1
546 :return: GlobalRoute object
547 """
548 if ctx.query_port(port_num).gid_tbl_len == 0:
549 raise unittest.SkipTest(f'Not supported without GID table')
550 gid = ctx.query_gid(port_num, gid_index)
551 gr = GlobalRoute(dgid=gid, sgid_index=gid_index)
552 return gr
553
554
555 def xrc_post_send(agr_obj, qp_num, send_object, send_op=None):
556 agr_obj.qps = agr_obj.sqp_lst
557 if send_op:
558 post_send_ex(agr_obj, send_object, send_op)
559 else:
560 post_send(agr_obj, send_object)
561
562
563 def post_send_ex(agr_obj, send_object, send_op=None, qp_idx=0, ah=None, **kwargs):
564 qp = agr_obj.qps[qp_idx]
565 qp_type = qp.qp_type
566 qp.wr_start()
567 qp.wr_id = 0x123
568 qp.wr_flags = ibv_send_flags.IBV_SEND_SIGNALED
569 if send_op == ibv_wr_opcode.IBV_WR_SEND:
570 qp.wr_send()
571 elif send_op == ibv_wr_opcode.IBV_WR_RDMA_WRITE:
572 qp.wr_rdma_write(agr_obj.rkey, agr_obj.raddr)
573 elif send_op == ibv_wr_opcode.IBV_WR_SEND_WITH_IMM:
574 qp.wr_send_imm(IMM_DATA)
575 elif send_op == ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM:
576 qp.wr_rdma_write_imm(agr_obj.rkey, agr_obj.raddr, IMM_DATA)
577 elif send_op == ibv_wr_opcode.IBV_WR_ATOMIC_WRITE:
578 qp.wr_atomic_write(agr_obj.rkey, agr_obj.raddr, send_object)
579 elif send_op == ibv_wr_opcode.IBV_WR_FLUSH:
580 qp.wr_flush(agr_obj.rkey, agr_obj.raddr, agr_obj.msg_size,
581 agr_obj.ptype, agr_obj.level)
582 elif send_op == ibv_wr_opcode.IBV_WR_RDMA_READ:
583 qp.wr_rdma_read(agr_obj.rkey, agr_obj.raddr)
584 elif send_op == ibv_wr_opcode.IBV_WR_ATOMIC_CMP_AND_SWP:
585 cmp_add = kwargs.get('cmp_add')
586 swp = kwargs.get('swap')
587 qp.wr_atomic_cmp_swp(agr_obj.rkey, agr_obj.raddr,
588 int8b_from_int(cmp_add), int8b_from_int(swp))
589 elif send_op == ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD:
590 cmp_add = kwargs.get('cmp_add')
591 qp.wr_atomic_fetch_add(agr_obj.rkey, agr_obj.raddr,
592 int8b_from_int(cmp_add))
593 elif send_op == ibv_wr_opcode.IBV_WR_BIND_MW:
594 bind_info = MWBindInfo(agr_obj.mr, agr_obj.mr.buf, agr_obj.mr.rkey,
595 ibv_access_flags.IBV_ACCESS_REMOTE_WRITE)
596 mw = MW(agr_obj.pd, mw_type=ibv_mw_type.IBV_MW_TYPE_2)
597 # A new rkey is needed to be set into bind_info, modify rkey
598 qp.wr_bind_mw(mw, agr_obj.mr.rkey + 12, bind_info)
599 qp.wr_send()
600 if qp_type == ibv_qp_type.IBV_QPT_UD:
601 qp.wr_set_ud_addr(ah, agr_obj.rqps_num[qp_idx], agr_obj.UD_QKEY)
602 if isinstance(agr_obj, SRDResources):
603 qp.wr_set_ud_addr(ah, agr_obj.rqps_num[qp_idx], agr_obj.SRD_QKEY)
604 if qp_type == ibv_qp_type.IBV_QPT_XRC_SEND:
605 qp.wr_set_xrc_srqn(agr_obj.remote_srqn)
606 if hasattr(agr_obj, 'remote_dct_num'):
607 if isinstance(agr_obj, Mlx5DcStreamsRes):
608 stream_id = agr_obj.generate_stream_id(qp_idx)
609 qp.wr_set_dc_addr_stream(ah, agr_obj.remote_dct_num, DCT_KEY,
610 stream_id)
611 else:
612 qp.wr_set_dc_addr(ah, agr_obj.remote_dct_num, DCT_KEY)
613 if send_op != ibv_wr_opcode.IBV_WR_ATOMIC_WRITE and \
614 send_op != ibv_wr_opcode.IBV_WR_FLUSH:
615 qp.wr_set_sge(send_object)
616 qp.wr_complete()
617
618
619 def post_send(agr_obj, send_wr, qp_idx=0, ah=None, is_imm=False):
620 """
621 Post a single send WR to the QP. Post_send's second parameter (send bad wr)
622 is ignored for simplicity. For UD traffic an address vector is added as
623 well.
624 :param agr_obj: aggregation object which contains all resources necessary
625 :param send_wr: Send work request to post send
626 :param qp_idx: QP index to use
627 :param ah: The destination address handle
628 :param is_imm: If True, send with imm_data, relevant for old post send API
629 :return: None
630 """
631 qp_type = agr_obj.qp.qp_type
632 if is_imm:
633 send_wr.imm_data = socket.htonl(IMM_DATA)
634 if qp_type == ibv_qp_type.IBV_QPT_UD:
635 send_wr.set_wr_ud(ah, agr_obj.rqps_num[qp_idx], agr_obj.UD_QKEY)
636 if isinstance(agr_obj, SRDResources):
637 send_wr.set_wr_ud(ah, agr_obj.rqps_num[qp_idx], agr_obj.SRD_QKEY)
638 agr_obj.qps[qp_idx].post_send(send_wr, None)
639
640
641 def post_recv(agr_obj, recv_wr, qp_idx=0 ,num_wqes=1):
642 """
643 Call the QP's post_recv() method <num_wqes> times. Post_recv's second
644 parameter (recv bad wr) is ignored for simplicity.
645 :param recv_wr: Receive work request to post
646 :param qp_idx: QP index which posts receive work request
647 :param num_wqes: Number of WQEs to post
648 :return: None
649 """
650 receive_queue = agr_obj.srq if agr_obj.srq else agr_obj.qps[qp_idx]
651 for _ in range(num_wqes):
652 if isinstance(receive_queue, QPEx) and receive_queue.ind_table:
653 for wq in receive_queue.ind_table.wqs:
654 wq.post_recv(recv_wr, None)
655 else:
656 receive_queue.post_recv(recv_wr, None)
657
658
659 def _poll_cq(cq, count=1, data=None):
660 """
661 Poll <count> completions from the CQ.
662 Note: This function calls the blocking poll() method of the CQ
663 until <count> completions were received. Alternatively, gets a
664 single CQ event when events are used.
665 :param cq: CQ to poll from
666 :param count: How many completions to poll
667 :param data: In case of a work request with immediate, the immediate data
668 to be compared after poll
669 :return: An array of work completions of length <count>, None
670 when events are used
671 """
672 wcs = []
673 channel = cq.comp_channel
674 start_poll_t = time.perf_counter()
675 while count > 0 and (time.perf_counter() - start_poll_t < POLL_CQ_TIMEOUT):
676 if channel:
677 channel.get_cq_event(cq)
678 cq.req_notify()
679 nc, tmp_wcs = cq.poll(count)
680 for wc in tmp_wcs:
681 if wc.status != ibv_wc_status.IBV_WC_SUCCESS:
682 wcs.append(wc)
683 return wcs
684 if data:
685 if wc.wc_flags & ibv_wc_flags.IBV_WC_WITH_IMM == 0:
686 raise PyverbsRDMAError('Completion without immediate')
687 assert socket.ntohl(wc.imm_data) == data
688 count -= nc
689 wcs.extend(tmp_wcs)
690
691 if count > 0:
692 raise PyverbsError(f'Got timeout on polling ({count} CQEs remaining)')
693
694 return wcs
695
696
697 def poll_cq(cq, count=1, data=None):
698 """
699 Poll <count> completions from the CQ.
700 Note: This function calls the blocking poll() method of the CQ
701 until <count> completions were received. Alternatively, gets a
702 single CQ event when events are used.
703 :param cq: CQ to poll from
704 :param count: How many completions to poll
705 :param data: In case of a work request with immediate, the immediate data
706 to be compared after poll
707 :return: An array of work completions of length <count>, None
708 when events are used
709 """
710 wcs = _poll_cq(cq, count, data)
711 if wcs[0].status != ibv_wc_status.IBV_WC_SUCCESS:
712 raise PyverbsRDMAError(f'Completion status is {wc_status_to_str(wcs[0].status)}')
713
714 return wcs
715
716
717 def poll_cq_ex(cqex, count=1, data=None, sgid=None):
718 """
719 Poll <count> completions from the extended CQ.
720 :param cq: CQEX to poll from
721 :param count: How many completions to poll
722 :param data: In case of a work request with immediate, the immediate data
723 to be compared after poll, either a list of immediate data or
724 one value
725 :param sgid: In case of EFA receive completion, the sgid to be compared
726 after poll
727 :return: WR id order received
728 """
729 wr_id_order = []
730 iters = count
731 try:
732 start_poll_t = time.perf_counter()
733 poll_attr = PollCqAttr()
734 ret = cqex.start_poll(poll_attr)
735 while ret == 2 and (time.perf_counter() - start_poll_t < POLL_CQ_TIMEOUT):
736 ret = cqex.start_poll(poll_attr)
737 if ret != 0:
738 raise PyverbsRDMAErrno('Failed to poll CQ')
739 wr_id_order.append(cqex.wr_id)
740 count -= 1
741 if cqex.status != ibv_wc_status.IBV_WC_SUCCESS:
742 raise PyverbsRDMAErrno('Completion status is {s}'.
743 format(s=cqex.status))
744 if data:
745 imm_data = data if not isinstance(data, list) else data[iters - count - 1]
746 assert imm_data == socket.ntohl(cqex.read_imm_data())
747
748 if isinstance(cqex, EfaCQ):
749 if sgid is not None and cqex.read_opcode() == ibv_wc_opcode.IBV_WC_RECV:
750 assert sgid.gid == cqex.read_sgid().gid
751 # Now poll the rest of the packets
752 while count > 0 and (time.perf_counter() - start_poll_t < POLL_CQ_TIMEOUT):
753 ret = cqex.poll_next()
754 while ret == 2:
755 ret = cqex.poll_next()
756 if ret != 0:
757 raise PyverbsRDMAErrno('Failed to poll CQ')
758 if cqex.status != ibv_wc_status.IBV_WC_SUCCESS:
759 raise PyverbsRDMAErrno('Completion status is {s}'.
760 format(s=cqex.status))
761 count -= 1
762 wr_id_order.append(cqex.wr_id)
763 if data:
764 imm_data = data if not isinstance(data, list) else data[iters - count - 1]
765 assert imm_data == socket.ntohl(cqex.read_imm_data())
766
767 if isinstance(cqex, EfaCQ):
768 if sgid is not None and cqex.read_opcode() == ibv_wc_opcode.IBV_WC_RECV:
769 assert sgid.gid == cqex.read_sgid().gid
770 if count > 0:
771 raise PyverbsError(f'Got timeout on polling ({count} CQEs remaining)')
772 finally:
773 cqex.end_poll()
774 return wr_id_order
775
776
777 def validate(received_str, is_server, msg_size):
778 """
779 Validates the received buffer against the expected result.
780 The application should set client's send buffer to 'c's and the
781 server's send buffer to 's's.
782 If the expected buffer is different than the actual, an exception will
783 be raised.
784 :param received_str: The received buffer to check
785 :param is_server: Indicates whether this is the server (receiver) or
786 client side
787 :param msg_size: the message size of the received packet
788 :return: None
789 """
790 expected_str = msg_size * ('c' if is_server else 's')
791 received_str = received_str.decode()
792 if received_str[0:msg_size] == \
793 expected_str[0:msg_size]:
794 return
795 else:
796 raise PyverbsError(
797 'Data validation failure: expected {exp}, received {rcv}'.
798 format(exp=expected_str, rcv=received_str))
799
800
801 def send(agr_obj, send_object, send_op=None, new_send=False, qp_idx=0, ah=None, is_imm=False,
802 **kwargs):
803 if isinstance(agr_obj, XRCResources):
804 agr_obj.qps = agr_obj.sqp_lst
805 if new_send:
806 return post_send_ex(agr_obj, send_object, send_op, qp_idx, ah, **kwargs)
807 return post_send(agr_obj, send_object, qp_idx, ah, is_imm)
808
809
810 def traffic_poll_at_once(test, msg_size, iterations=10, opcode=ibv_wr_opcode.IBV_WR_SEND):
811 """
812 Execute traffic between two peers iterations times:
813 - Create receive resources and then post recv
814 - Create send resources and then post send
815 - Poll client and server all cqes at once
816 - Compare received buffer with the expected one and validate
817 :param test: Test object with client and server.
818 :param msg_size: Size of a packet.
819 :param iterations: Number of packets to send/recv.
820 :param opcode: Send opcode. Currently it supports SEND and
821 RDMA_WRITE_WITH_IMM opcodes only.
822 """
823 imm_data_exp = []
824
825 for i in range(iterations):
826 recv_sge = SGE(test.server.mr.buf + (msg_size * i), msg_size, test.server.mr.lkey)
827 recv_wr = RecvWR(sg=[recv_sge], num_sge=1, wr_id=i)
828 test.server.mr.write(str(iterations - i - 1) * msg_size, msg_size, i * msg_size)
829 test.server.qp.post_recv(recv_wr)
830
831 for i in range(iterations):
832 send_sge = SGE(test.client.mr.buf + (msg_size * i), msg_size, test.client.mr.lkey)
833 send_wr = SendWR(opcode=opcode, num_sge=1, sg=[send_sge], wr_id=i)
834 if opcode == ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM:
835 send_wr.imm_data = socket.htonl(i + iterations)
836 imm_data_exp.append(i + iterations)
837 send_wr.set_wr_rdma(int(test.client.rkey), int(test.client.raddr + i * msg_size))
838 test.client.mr.write(str(i % 10) * msg_size, msg_size, i * msg_size)
839 test.client.qp.post_send(send_wr)
840
841 send_id_order = poll_cq_ex(test.client.cq, iterations)
842 recv_id_order = poll_cq_ex(test.server.cq, iterations, data=imm_data_exp)
843 # With opcodes that don't consume Recv WQEs (e.g. WRITE and WRITE_WITH_IMM) the buffer
844 # scattering offsets are defined by the sender.
845 recv_id_order = send_id_order if opcode == ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM else recv_id_order
846
847 for i, j in zip(recv_id_order, send_id_order):
848 exp_buff = bytearray((str(j % 10) * msg_size), 'utf-8')
849 recv_buff = test.server.mr.read(msg_size, i * msg_size)
850 if recv_buff != exp_buff:
851 raise PyverbsRDMAError(f'Data validation failed: expected {exp_buff},'
852 f' received {recv_buff}')
853
854
855 def traffic(client, server, iters, gid_idx, port, is_cq_ex=False, send_op=ibv_wr_opcode.IBV_WR_SEND,
856 new_send=False, force_page_faults=False):
857 """
858 Runs basic traffic between two sides
859 :param client: client side, clients base class is BaseTraffic
860 :param server: server side, servers base class is BaseTraffic
861 :param iters: number of traffic iterations
862 :param gid_idx: local gid index
863 :param port: IB port
864 :param is_cq_ex: If True, use poll_cq_ex() rather than poll_cq()
865 :param send_op: The send_wr opcode.
866 :param new_send: If True use new post send API.
867 :param force_page_faults: If True, use madvise to hint that we don't need the MR's buffer to
868 force page faults (useful for ODP testing).
869 :return:
870 """
871 if is_datagram_qp(client):
872 ah_client = get_global_ah(client, gid_idx, port)
873 ah_server = get_global_ah(server, gid_idx, port)
874 else:
875 ah_client = None
876 ah_server = None
877 poll = poll_cq_ex if is_cq_ex else poll_cq
878
879 imm_data = None
880 if send_op in [ibv_wr_opcode.IBV_WR_SEND_WITH_IMM, ibv_wr_opcode.IBV_WR_RDMA_WRITE_WITH_IMM]:
881 imm_data = IMM_DATA
882
883 s_recv_wr = get_recv_wr(server)
884 c_recv_wr = get_recv_wr(client)
885 for qp_idx in range(server.qp_count):
886 # prepare the receive queue with RecvWR
887 post_recv(client, c_recv_wr, qp_idx=qp_idx)
888 post_recv(server, s_recv_wr, qp_idx=qp_idx)
889 read_offset = GRH_SIZE if client.qp.qp_type == ibv_qp_type.IBV_QPT_UD else 0
890 for _ in range(iters):
891 for qp_idx in range(server.qp_count):
892 if force_page_faults:
893 madvise(client.mr.buf, client.msg_size)
894 madvise(server.mr.buf, server.msg_size)
895 c_send_wr, c_sg = get_send_elements(client, False, send_op)
896 if client.use_mr_prefetch:
897 flags = _IBV_ADVISE_MR_FLAG_FLUSH
898 if client.use_mr_prefetch == 'async':
899 flags = 0
900 prefetch_mrs(client, [c_sg], advice=client.prefetch_advice,
901 flags=flags)
902 c_send_object = c_sg if new_send else c_send_wr
903 send(client, c_send_object, send_op, new_send, qp_idx,
904 ah_client, is_imm=(imm_data != None))
905 poll(client.cq)
906 poll(server.cq, data=imm_data)
907 post_recv(server, s_recv_wr, qp_idx=qp_idx)
908 msg_received_list = get_msg_received(server, read_offset)
909 for msg in msg_received_list:
910 validate(msg, True, server.msg_size)
911 s_send_wr, s_sg = get_send_elements(server, True, send_op)
912 if server.use_mr_prefetch:
913 flags = _IBV_ADVISE_MR_FLAG_FLUSH
914 if server.use_mr_prefetch == 'async':
915 flags = 0
916 prefetch_mrs(server, [s_sg], advice=server.prefetch_advice,
917 flags=flags)
918 s_send_object = s_sg if new_send else s_send_wr
919 send(server, s_send_object, send_op, new_send, qp_idx,
920 ah_server, is_imm=(imm_data != None))
921 poll(server.cq)
922 poll(client.cq, data=imm_data)
923 post_recv(client, c_recv_wr, qp_idx=qp_idx)
924 msg_received_list = get_msg_received(client,read_offset)
925 for msg in msg_received_list:
926 validate(msg, False, server.msg_size)
927
928
929 def get_msg_received(agr_obj, read_offset):
930 msg_received_list = [agr_obj.mr.read(agr_obj.msg_size, read_offset)]
931 if hasattr(agr_obj, 'use_mixed_mr') and agr_obj.use_mixed_mr:
932 msg_received_list.append(agr_obj.non_odp_mr.read(agr_obj.msg_size, read_offset))
933 return msg_received_list
934
935
936 def gen_ethernet_header(dst_mac=PacketConsts.DST_MAC, src_mac=PacketConsts.SRC_MAC,
937 ether_type=PacketConsts.ETHER_TYPE_IPV4):
938 """
939 Generates Ethernet header using the values from the PacketConst class by default.
940 :param dst_mac: Destination mac address
941 :param src_mac: Source mac address
942 :param ether_type: Ether type of next header
943 :return: Ethernet header
944 """
945 header = struct.pack('!6s6s',
946 bytes.fromhex(dst_mac.replace(':', '')),
947 bytes.fromhex(src_mac.replace(':', '')))
948 header += ether_type.to_bytes(2, 'big')
949 return header
950
951
952 def gen_ipv4_header(packet_len, next_proto=socket.IPPROTO_UDP, src_ip=PacketConsts.SRC_IP,
953 dst_ip=PacketConsts.DST_IP):
954 """
955 Generates IPv4 header using the values from the PacketConst class by default.
956 :param packet_len: Length of all fields following the IP header
957 :param next_proto: protocol type of next header
958 :param src_ip: Source mac address
959 :param dst_ip: Destination mac address
960 :return: IPv4 header
961 """
962 ip_total_len = packet_len + PacketConsts.IPV4_HEADER_SIZE
963 return struct.pack('!2B3H2BH4s4s', (PacketConsts.IP_V4 << 4) +
964 PacketConsts.IHL, 0, ip_total_len, 0,
965 PacketConsts.IP_V4_FLAGS << 13,
966 PacketConsts.TTL_HOP_LIMIT, next_proto, 0,
967 socket.inet_aton(src_ip),
968 socket.inet_aton(dst_ip))
969
970
971 def gen_udp_header(packet_len, src_port=PacketConsts.SRC_PORT, dst_port=PacketConsts.DST_PORT):
972 """
973 Generates UDP header using the values from the PacketConst class by default.
974 :param packet_len: Length of all fields following the UDP header
975 :param src_port: Source port
976 :param dst_port: Destination port
977 :return: UDP header
978 """
979 udp_total_len = packet_len + PacketConsts.UDP_HEADER_SIZE
980 return struct.pack('!4H', src_port, dst_port, udp_total_len, 0)
981
982
983 def gen_gre_header(ether_type=PacketConsts.ETHER_TYPE_IPV4):
984 """
985 Generates GRE header using the values from the PacketConst class by default.
986 :param ether_type: Ether type of tunneled next header
987 :return: GRE header
988 """
989 return struct.pack('!2BHI', PacketConsts.GRE_FLAGS << 4, PacketConsts.GRE_VER,
990 ether_type, PacketConsts.GRE_KEY)
991
992
993 def gen_vxlan_header():
994 """
995 Generates VXLAN header using the values from the PacketConst class by default.
996 :return: VXLAN header
997 """
998 return struct.pack('!II', PacketConsts.VXLAN_FLAGS << 24, PacketConsts.VXLAN_VNI << 8)
999
1000
1001 def gen_geneve_header(vni=PacketConsts.GENEVE_VNI, oam=PacketConsts.GENEVE_OAM,
1002 proto=PacketConsts.ETHER_TYPE_ETH):
1003 """
1004 Generates Geneve header using the values from the PacketConst class by default.
1005 :param vni: geneve vni
1006 :param oam: geneve oam
1007 :param proto: Ether type of next header inside the tunnel
1008 :return: Geneve header
1009 """
1010 return struct.pack('!BBHL', (0 << 6) + 0, (oam << 7) + (0 << 6) + 0, proto, (vni << 8) + 0)
1011
1012 def gen_bth_header(opcode=PacketConsts.BTH_OPCODE, dst_qp=PacketConsts.BTH_DST_QP, a=PacketConsts.BTH_A):
1013 """
1014 Generates ROCE BTH header using the values from the PacketConst class by default.
1015 :param opcode: BTH opcode
1016 :param dst_qp: BTH dst QP
1017 :param a: BTH acknowledgment bit
1018 :return: ROCE BTH header
1019 """
1020 return struct.pack('!2BH2BH2L', opcode, 0, PacketConsts.BTH_PARTITION_KEY,
1021 PacketConsts.BTH_BECN << 6, dst_qp >> 16, dst_qp & 0xffff, a << 31, 0)
1022
1023
1024 def gen_packet(msg_size, l3=PacketConsts.IP_V4, l4=PacketConsts.UDP_PROTO, with_vlan=False, **kwargs):
1025 """
1026 Generates a Eth | IPv4 or IPv6 | UDP or TCP packet with hardcoded values in
1027 the headers and randomized payload.
1028 :param msg_size: total packet size
1029 :param l3: Packet layer 3 type: 4 for IPv4 or 6 for IPv6
1030 :param l4: Packet layer 4 type: 'tcp' or 'udp'
1031 :param with_vlan: if True add VLAN header to the packet
1032 :param kwargs: Arguments:
1033 * *src_mac*
1034 Source MAC address to use in the packet.
1035 * *src_ipv4*
1036 Source IPv4 address to use in the packet.
1037 :return: packet
1038 """
1039 l3_header_size = getattr(PacketConsts, f'IPV{str(l3)}_HEADER_SIZE')
1040 l4_header_size = getattr(PacketConsts, f'{l4.upper()}_HEADER_SIZE')
1041 payload_size = max(0, msg_size - l3_header_size - l4_header_size -
1042 PacketConsts.ETHER_HEADER_SIZE)
1043 next_hdr = getattr(socket, f'IPPROTO_{l4.upper()}')
1044 ip_total_len = msg_size - PacketConsts.ETHER_HEADER_SIZE
1045
1046 # Ethernet header
1047 src_mac = kwargs.get('src_mac', bytes.fromhex(PacketConsts.SRC_MAC.replace(':', '')))
1048 packet = struct.pack('!6s6s',
1049 bytes.fromhex(PacketConsts.DST_MAC.replace(':', '')), src_mac)
1050 if with_vlan:
1051 packet += struct.pack('!HH', PacketConsts.VLAN_TPID, (PacketConsts.VLAN_PRIO << 13) +
1052 (PacketConsts.VLAN_CFI << 12) + PacketConsts.VLAN_ID)
1053 payload_size -= PacketConsts.VLAN_HEADER_SIZE
1054 ip_total_len -= PacketConsts.VLAN_HEADER_SIZE
1055
1056 if l3 == PacketConsts.IP_V4:
1057 packet += PacketConsts.ETHER_TYPE_IPV4.to_bytes(2, 'big')
1058 else:
1059 packet += PacketConsts.ETHER_TYPE_IPV6.to_bytes(2, 'big')
1060
1061 if l3 == PacketConsts.IP_V4:
1062 # IPv4 header
1063 src_ipv4 = kwargs.get('src_ipv4', PacketConsts.SRC_IP)
1064 packet += struct.pack('!2B3H2BH4s4s', (PacketConsts.IP_V4 << 4) +
1065 PacketConsts.IHL, 0, ip_total_len, 0,
1066 PacketConsts.IP_V4_FLAGS << 13,
1067 PacketConsts.TTL_HOP_LIMIT, next_hdr, 0,
1068 socket.inet_aton(src_ipv4),
1069 socket.inet_aton(PacketConsts.DST_IP))
1070 else:
1071 # IPv6 header
1072 packet += struct.pack('!IH2B16s16s', (PacketConsts.IP_V6 << 28),
1073 ip_total_len, next_hdr, PacketConsts.TTL_HOP_LIMIT,
1074 socket.inet_pton(socket.AF_INET6, PacketConsts.SRC_IP6),
1075 socket.inet_pton(socket.AF_INET6, PacketConsts.DST_IP6))
1076
1077 if l4 == PacketConsts.UDP_PROTO:
1078 # UDP header
1079 packet += struct.pack('!4H', PacketConsts.SRC_PORT,
1080 PacketConsts.DST_PORT,
1081 payload_size + PacketConsts.UDP_HEADER_SIZE, 0)
1082 else:
1083 # TCP header
1084 packet += struct.pack('!2H2I4H', PacketConsts.SRC_PORT,
1085 PacketConsts.DST_PORT, 0, 0,
1086 PacketConsts.TCP_HEADER_SIZE_WORDS << 12,
1087 PacketConsts.WINDOW_SIZE, 0, 0)
1088 # Payload
1089 packet += str.encode('a' * payload_size)
1090 return packet
1091
1092
1093 def get_send_elements_raw_qp(agr_obj, l3=PacketConsts.IP_V4,
1094 l4=PacketConsts.UDP_PROTO, with_vlan=False,
1095 packet_to_send=None, **packet_args):
1096 """
1097 Creates a single SGE and a single Send WR for agr_obj's RAW QP type. The
1098 content of the message is Eth | Ipv4 | UDP packet.
1099 :param agr_obj: Aggregation object which contains all resources necessary
1100 :param l3: Packet layer 3 type: 4 for IPv4 or 6 for IPv6
1101 :param l4: Packet layer 4 type: 'tcp' or 'udp'
1102 :param with_vlan: if True add VLAN header to the packet
1103 :param packet_to_send: If passed, the other packet related parameters would
1104 be ignored, and this will be the packet to send.
1105 :param packet_args: Pass packet_args to gen_packets method.
1106 :return: send wr, its SGE, and message
1107 """
1108 mr = agr_obj.mr
1109 msg = packet_to_send if packet_to_send is not None else \
1110 gen_packet(agr_obj.msg_size, l3, l4, with_vlan, **packet_args)
1111 mr.write(msg, agr_obj.msg_size)
1112 sge = SGE(mr.buf, agr_obj.msg_size, mr.lkey)
1113 send_wr = SendWR(opcode=ibv_wr_opcode.IBV_WR_SEND, num_sge=1, sg=[sge])
1114 return send_wr, sge, msg
1115
1116
1117 def validate_raw(msg_received, msg_expected, skip_idxs):
1118 size = len(msg_expected)
1119 for i in range(size):
1120 if (msg_received[i] != msg_expected[i]) and i not in skip_idxs:
1121 err_msg = f'Data validation failure:\nexpected {msg_expected}\n\nreceived {msg_received}'
1122 raise PyverbsError(err_msg)
1123
1124
1125 def sampler_traffic(client, server, iters, l3=PacketConsts.IP_V4, l4=PacketConsts.UDP_PROTO):
1126 """
1127 Send raw ethernet traffic
1128 :param client: client side, clients base class is BaseTraffic
1129 :param server: server side, servers base class is BaseTraffic
1130 :param iters: number of traffic iterations
1131 :param l3: Packet layer 3 type: 4 for IPv4 or 6 for IPv6
1132 :param l4: Packet layer 4 type: 'tcp' or 'udp'
1133 """
1134 s_recv_wr = get_recv_wr(server)
1135 c_recv_wr = get_recv_wr(client)
1136 for qp_idx in range(server.qp_count):
1137 # Prepare the receive queue with RecvWR
1138 post_recv(client, c_recv_wr, qp_idx=qp_idx)
1139 post_recv(server, s_recv_wr, qp_idx=qp_idx)
1140 poll = poll_cq_ex if isinstance(client.cq, CQEX) else poll_cq
1141 for _ in range(iters):
1142 for qp_idx in range(server.qp_count):
1143 c_send_wr, c_sg, msg = get_send_elements_raw_qp(client, l3, l4, False)
1144 send(client, c_send_wr, ibv_wr_opcode.IBV_WR_SEND, False, qp_idx)
1145 poll(client.cq)
1146
1147
1148 def raw_traffic(client, server, iters, l3=PacketConsts.IP_V4,
1149 l4=PacketConsts.UDP_PROTO, with_vlan=False, expected_packet=None,
1150 skip_idxs=None, packet_to_send=None):
1151 """
1152 Runs raw ethernet traffic between two sides
1153 :param client: client side, clients base class is BaseTraffic
1154 :param server: server side, servers base class is BaseTraffic
1155 :param iters: number of traffic iterations
1156 :param l3: Packet layer 3 type: 4 for IPv4 or 6 for IPv6
1157 :param l4: Packet layer 4 type: 'tcp' or 'udp'
1158 :param with_vlan: if True add VLAN header to the packet
1159 :param expected_packet: Expected packet for validation (when different from
1160 the originally sent).
1161 :param skip_idxs: indexes to skip during packet validation
1162 :param packet_to_send: If passed, the other packet related parameters would
1163 be ignored, and this will be the packet to send.
1164 """
1165 skip_idxs = [] if skip_idxs is None else skip_idxs
1166 s_recv_wr = get_recv_wr(server)
1167 c_recv_wr = get_recv_wr(client)
1168 for qp_idx in range(server.qp_count):
1169 # prepare the receive queue with RecvWR
1170 post_recv(client, c_recv_wr, qp_idx=qp_idx)
1171 post_recv(server, s_recv_wr, qp_idx=qp_idx)
1172 read_offset = 0
1173 poll = poll_cq_ex if isinstance(client.cq, CQEX) else poll_cq
1174 for _ in range(iters):
1175 for qp_idx in range(server.qp_count):
1176 c_send_wr, c_sg, msg = get_send_elements_raw_qp(client, l3, l4, with_vlan,
1177 packet_to_send=packet_to_send)
1178 send(client, c_send_wr, ibv_wr_opcode.IBV_WR_SEND, False, qp_idx)
1179 poll(client.cq)
1180 poll(server.cq)
1181 post_recv(server, s_recv_wr, qp_idx=qp_idx)
1182 msg_received = server.mr.read(server.msg_size, read_offset)
1183 # Validate received packet
1184 validate_raw(msg_received,
1185 expected_packet if expected_packet else msg, skip_idxs)
1186
1187
1188 def raw_rss_traffic(client, server, iters, l3=PacketConsts.IP_V4,
1189 l4=PacketConsts.UDP_PROTO, with_vlan=False, num_packets=1):
1190 """
1191 Runs raw ethernet rss traffic between two sides.
1192 :param client: client side, clients base class is BaseTraffic
1193 :param server: server side, servers base class is BaseTraffic
1194 :param iters: number of traffic iterations
1195 :param l3: Packet layer 3 type: 4 for IPv4 or 6 for IPv6
1196 :param l4: Packet layer 4 type: 'tcp' or 'udp'
1197 :param with_vlan: if True add VLAN header to the packet
1198 :param num_packets: Number of packets to send with different ipv4 src
1199 address in each iteration.
1200 :return: None
1201 """
1202 s_recv_wr = get_recv_wr(server)
1203 for qp_idx in range(server.qp_count):
1204 # prepare the receive queue with RecvWR
1205 post_recv(server, s_recv_wr, qp_idx=qp_idx, num_wqes=num_packets)
1206 for _ in range(iters):
1207 for qp_idx in range(server.qp_count):
1208 for i in range(num_packets):
1209 c_send_wr, c_sg, msg = get_send_elements_raw_qp(
1210 client, l3, l4, with_vlan,
1211 src_ipv4='.'.join([str(num) for num in range(i, i + 4)]))
1212 send(client, c_send_wr, ibv_wr_opcode.IBV_WR_SEND, False, qp_idx)
1213 poll_cq(client.cq)
1214 completions = 0
1215 start_poll_t = time.perf_counter()
1216 while completions < num_packets and \
1217 (time.perf_counter() - start_poll_t < POLL_CQ_TIMEOUT):
1218 for cq in server.cqs:
1219 n, wcs = cq.poll()
1220 if n > 0:
1221 if wcs[0].status != ibv_wc_status.IBV_WC_SUCCESS:
1222 raise PyverbsRDMAError(
1223 f'Completion status is {wc_status_to_str(wcs[0].status)}',
1224 wcs[0].status)
1225 completions += 1
1226 if completions >= num_packets:
1227 break
1228 if completions < num_packets:
1229 raise PyverbsError(f'Expected {num_packets} completions - got {completions}')
1230 post_recv(server, s_recv_wr, qp_idx=qp_idx, num_wqes=num_packets)
1231
1232
1233 def flush_traffic(client, server, iters, gid_idx, port, new_send=False,
1234 send_op=None):
1235 """
1236 Runs basic RDMA FLUSH traffic that client requests a FLUSH to server.
1237 Simply, run RDMA WRITE and then follow up by a RDMA FLUSH.
1238 No receive WQEs are posted.
1239 :param client: client side, clients base class is BaseTraffic
1240 :param server: server side, servers base class is BaseTraffic
1241 :param iters: number of traffic iterations
1242 :param gid_idx: local gid index
1243 :param port: IB port
1244 :param new_send: If True use new post send API.
1245 :param send_op: The send_wr opcode.
1246 :return:
1247 """
1248 rdma_traffic(client, server, iters, gid_idx, port, new_send, ibv_wr_opcode.IBV_WR_RDMA_WRITE)
1249 for i in range(iters):
1250 if client.level == ibv_selectivity_level.IBV_FLUSH_MR:
1251 client.msg_size = 0 if i == 0 else random.randint(0, 12345678)
1252 send(client, None, send_op, new_send)
1253 wcs = _poll_cq(client.cq)
1254 if (wcs[0].status != ibv_wc_status.IBV_WC_SUCCESS):
1255 break
1256 return wcs
1257
1258
1259 def prepare_validate_data(client=None, server=None):
1260 if server:
1261 server.mem_write('s' * server.msg_size, server.msg_size)
1262 if client:
1263 client.mem_write('c' * client.msg_size, client.msg_size)
1264
1265
1266 def rdma_traffic(client, server, iters, gid_idx, port, new_send=False,
1267 send_op=None, force_page_faults=False):
1268 """
1269 Runs basic RDMA traffic between two sides. No receive WQEs are posted. For
1270 RDMA send with immediate, use traffic().
1271 :param client: client side, clients base class is BaseTraffic
1272 :param server: server side, servers base class is BaseTraffic
1273 :param iters: number of traffic iterations
1274 :param gid_idx: local gid index
1275 :param port: IB port
1276 :param new_send: If True use new post send API.
1277 :param send_op: The send_wr opcode.
1278 :param force_page_faults: If True, use madvise to hint that we don't need the MR's buffer to
1279 force page faults (useful for ODP testing).
1280 :return:
1281 """
1282 # Using the new post send API, we need the SGE, not the SendWR
1283 if isinstance(client, Mlx5DcResources) or \
1284 isinstance(client, SRDResources):
1285 ah_client = get_global_ah(client, gid_idx, port)
1286 ah_server = get_global_ah(server, gid_idx, port)
1287 else:
1288 ah_client = None
1289 ah_server = None
1290 send_element_idx = 1 if new_send else 0
1291 same_side_check = send_op in [ibv_wr_opcode.IBV_WR_RDMA_READ,
1292 ibv_wr_opcode.IBV_WR_ATOMIC_CMP_AND_SWP,
1293 ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD]
1294 for _ in range(iters):
1295 if force_page_faults:
1296 madvise(client.mr.buf, client.msg_size)
1297 madvise(server.mr.buf, server.msg_size)
1298 prepare_validate_data(client=client, server=server)
1299 c_send_wr = get_send_elements(client, False, send_op)[send_element_idx]
1300 send(client, c_send_wr, send_op, new_send, ah=ah_client)
1301 poll_cq(client.cq)
1302 if same_side_check:
1303 msg_received = client.mem_read(client.msg_size)
1304 else:
1305 msg_received = server.mem_read(server.msg_size)
1306 validate(msg_received, False if same_side_check else True,
1307 server.msg_size)
1308 s_send_wr = get_send_elements(server, True, send_op)[send_element_idx]
1309 prepare_validate_data(client=client, server=server)
1310 send(server, s_send_wr, send_op, new_send, ah=ah_server)
1311 poll_cq(server.cq)
1312 if same_side_check:
1313 msg_received = server.mem_read(client.msg_size)
1314 else:
1315 msg_received = client.mem_read(server.msg_size)
1316 validate(msg_received, True if same_side_check else False,
1317 client.msg_size)
1318
1319
1320 def atomic_traffic(client, server, iters, gid_idx, port, new_send=False,
1321 send_op=None, receiver_val=1, sender_val=2, swap=0,
1322 client_wr=1, server_wr=1, **kwargs):
1323 """
1324 Runs atomic traffic between two sides.
1325 :param client: Client side, clients base class is BaseTraffic
1326 :param server: Server side, servers base class is BaseTraffic
1327 :param iters: Number of traffic iterations
1328 :param gid_idx: Local gid index
1329 :param port: IB port
1330 :param new_send: If True use new post send API.
1331 :param send_op: The send_wr opcode.
1332 :param receiver_val: The requested value on the reciver MR.
1333 :param sender_val: The requested value on the sender SendWR.
1334 :param client_wr: Number of WR the client will post before polling all of them
1335 :param server_wr: Number of WR the server will post before polling all of them
1336 :param kwargs: General arguments (shared with other traffic functions).
1337 """
1338 send_element_idx = 1 if new_send else 0
1339 if is_datagram_qp(client):
1340 ah_client = get_global_ah(client, gid_idx, port)
1341 ah_server = get_global_ah(server, gid_idx, port)
1342 else:
1343 ah_client = None
1344 ah_server = None
1345
1346 for _ in range(iters):
1347 client.mr.write(int.to_bytes(sender_val, 1, byteorder='big') * 8, 8)
1348 server.mr.write(int.to_bytes(receiver_val, 1, byteorder='big') * 8, 8)
1349 for _ in range(client_wr):
1350 c_send_wr = get_atomic_send_elements(client,
1351 send_op,
1352 cmp_add=sender_val,
1353 swap=swap)[send_element_idx]
1354 if isinstance(server, XRCResources):
1355 c_send_wr.set_qp_type_xrc(server.srq.get_srq_num())
1356 send(client, c_send_wr, send_op, new_send, ah=ah_client,
1357 cmp_add=sender_val, swap=swap)
1358 poll_cq(client.cq, count=client_wr)
1359 validate_atomic(send_op, server, client,
1360 receiver_val=receiver_val + sender_val * (client_wr - 1),
1361 send_cmp_add=sender_val, send_swp=swap)
1362 server.mr.write(int.to_bytes(sender_val, 1, byteorder='big') * 8, 8)
1363 client.mr.write(int.to_bytes(receiver_val, 1, byteorder='big') * 8, 8)
1364 for _ in range(server_wr):
1365 s_send_wr = get_atomic_send_elements(server,
1366 send_op,
1367 cmp_add=sender_val,
1368 swap=swap)[send_element_idx]
1369 if isinstance(client, XRCResources):
1370 s_send_wr.set_qp_type_xrc(client.srq.get_srq_num())
1371 send(server, s_send_wr, send_op, new_send, ah=ah_server,
1372 cmp_add=sender_val, swap=swap)
1373 poll_cq(server.cq, count=server_wr)
1374 validate_atomic(send_op, client, server,
1375 receiver_val=receiver_val + sender_val * (server_wr - 1),
1376 send_cmp_add=sender_val, send_swp=swap)
1377
1378
1379 def validate_atomic(opcode, recv_player, send_player, receiver_val,
1380 send_cmp_add, send_swp):
1381 """
1382 Validate the data after atomic operations. The expected data in each side of
1383 traffic depends on the atomic type and the sender SendWR values.
1384 :param opcode: The atomic opcode.
1385 :param recv_player: The receiver player.
1386 :param send_player: The sender player.
1387 :param receiver_val: The value on the receiver MR before the atomic action.
1388 :param send_cmp_add: The send WR compare/add value depende on the atomic
1389 type.
1390 :param send_swp: The send WR swap value, used only in atomic compare and
1391 swap.
1392 """
1393 send_expected = receiver_val
1394 if opcode in [ibv_wr_opcode.IBV_WR_ATOMIC_CMP_AND_SWP,
1395 ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP]:
1396 recv_expected = send_swp if receiver_val == send_cmp_add \
1397 else receiver_val
1398 if opcode in [ibv_wr_opcode.IBV_WR_ATOMIC_FETCH_AND_ADD,
1399 ibv_qp_create_send_ops_flags.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD]:
1400 recv_expected = receiver_val + send_cmp_add
1401 send_actual = int.from_bytes(send_player.mr.read(length=8, offset=0),
1402 byteorder='big')
1403 recv_actual = int.from_bytes(recv_player.mr.read(length=8, offset=0),
1404 byteorder='big')
1405 if send_actual != int8b_from_int(send_expected):
1406 raise PyverbsError(
1407 'Atomic sender data validation failed: expected {exp}, received {rcv}'.
1408 format(exp=int8b_from_int(send_expected), rcv=send_actual))
1409 if recv_actual != int8b_from_int(recv_expected):
1410 raise PyverbsError(
1411 'Atomic reciver data validation failed: expected {exp}, received {rcv}'.
1412 format(exp=int8b_from_int(recv_expected), rcv=recv_actual))
1413
1414
1415 def int8b_from_int(num):
1416 """
1417 Duplicate one-byte value int to 8 bytes.
1418 e.g. 1 => b'\x01\x01\x01\x01\x01\x01\x01\x01' == 72340172838076673
1419 :param num: One byte int number (0 <= num < 256).
1420 :return: The new number in int format.
1421 """
1422 num_multi_8_str = int.to_bytes(num, 1, byteorder='big') * 8
1423 return int.from_bytes(num_multi_8_str, byteorder='big')
1424
1425
1426 def get_atomic_send_elements(agr_obj, opcode, cmp_add=0, swap=0):
1427 """
1428 Creates a single SGE and a single Send WR for atomic operations.
1429 :param agr_obj: Aggregation object which contains all resources necessary
1430 :param opcode: The send opcode
1431 :param cmp_add: The compare or add value (depends on the opcode).
1432 :param swap: The swap value.
1433 :return: Send WR and its SGE
1434 """
1435 sge = SGE(agr_obj.mr.buf, 8, agr_obj.mr_lkey)
1436 send_wr = SendWR(opcode=opcode, num_sge=1, sg=[sge])
1437 send_wr.set_wr_atomic(rkey=int(agr_obj.rkey), addr=int(agr_obj.raddr),
1438 compare_add=int8b_from_int(cmp_add),
1439 swap=int8b_from_int(swap))
1440 return send_wr, sge
1441
1442
1443 def xrc_traffic(client, server, is_cq_ex=False, send_op=None, force_page_faults=False):
1444 """
1445 Runs basic xrc traffic, this function assumes that number of QPs, which
1446 server and client have are equal, server.send_qp[i] is connected to
1447 client.recv_qp[i], each time server.send_qp[i] sends a message, it is
1448 redirected to client.srq because client.recv_qp[i] and client.srq are
1449 under the same xrcd. The traffic flow in the opposite direction is the same.
1450 :param client: Aggregation object of the active side, should be an instance
1451 of XRCResources class
1452 :param server: Aggregation object of the passive side, should be an instance
1453 of XRCResources class
1454 :param is_cq_ex: If True, use poll_cq_ex() rather than poll_cq()
1455 :param send_op: If not None, new post send API is assumed.
1456 :param force_page_faults: If True, use madvise to hint that we don't need the MR's buffer to
1457 force page faults (useful for ODP testing).
1458 :return: None
1459 """
1460 poll = poll_cq_ex if is_cq_ex else poll_cq
1461 server.remote_srqn = client.srq.get_srq_num()
1462 client.remote_srqn = server.srq.get_srq_num()
1463 s_recv_wr = get_recv_wr(server)
1464 c_recv_wr = get_recv_wr(client)
1465 post_recv(client, c_recv_wr, num_wqes=client.qp_count*client.num_msgs)
1466 post_recv(server, s_recv_wr, num_wqes=server.qp_count*server.num_msgs)
1467 # Using the new post send API, we need the SGE, not the SendWR
1468 send_element_idx = 1 if send_op else 0
1469 for _ in range(client.num_msgs):
1470 for i in range(server.qp_count):
1471 if force_page_faults:
1472 madvise(client.mr.buf, client.msg_size)
1473 madvise(server.mr.buf, server.msg_size)
1474 c_send_wr = get_send_elements(client, False)[send_element_idx]
1475 if send_op is None:
1476 c_send_wr.set_qp_type_xrc(client.remote_srqn)
1477 xrc_post_send(client, i, c_send_wr, send_op)
1478 poll(client.cq)
1479 poll(server.cq)
1480 msg_received = server.mr.read(server.msg_size, 0)
1481 validate(msg_received, True, server.msg_size)
1482 s_send_wr = get_send_elements(server, True)[send_element_idx]
1483 if send_op is None:
1484 s_send_wr.set_qp_type_xrc(server.remote_srqn)
1485 xrc_post_send(server, i, s_send_wr, send_op)
1486 poll(server.cq)
1487 poll(client.cq)
1488 msg_received = client.mr.read(client.msg_size, 0)
1489 validate(msg_received, False, client.msg_size)
1490
1491
1492 # Decorators
1493 def requires_odp(qp_type, required_odp_caps):
1494 def outer(func):
1495 def inner(instance):
1496 ctx = getattr(instance, 'ctx', d.Context(name=instance.dev_name))
1497 odp_supported(ctx, qp_type, required_odp_caps)
1498 if getattr(instance, 'is_implicit', False):
1499 odp_implicit_supported(instance.ctx)
1500 return func(instance)
1501 return inner
1502 return outer
1503
1504
1505 def requires_root():
1506 def outer(func):
1507 def inner(instance):
1508 if not is_root():
1509 raise unittest.SkipTest('Must be run by root')
1510 return func(instance)
1511 return inner
1512 return outer
1513
1514
1515 def requires_no_sriov():
1516 def outer(func):
1517 def inner(instance):
1518 physfn_path = f'/sys/class/infiniband/{instance.dev_name}/device/physfn'
1519 if os.path.isdir(physfn_path):
1520 sriov_path = f'{physfn_path}/sriov_totalvfs'
1521 else:
1522 sriov_path = f'/sys/class/infiniband/{instance.dev_name}/device/sriov_totalvfs'
1523 if os.path.isfile(sriov_path):
1524 raise unittest.SkipTest('Virtual Functions are present on the device')
1525 return func(instance)
1526 return inner
1527 return outer
1528
1529
1530 def requires_eth(port_num=1):
1531 def outer(func):
1532 def inner(instance):
1533 if not is_eth(instance.ctx, port_num):
1534 raise unittest.SkipTest('Must be run on Ethernet link layer')
1535 return func(instance)
1536 return inner
1537 return outer
1538
1539
1540 def requires_mcast_support():
1541 """
1542 Check if the device support multicast
1543 return: True if multicast is supported
1544 """
1545 def outer(func):
1546 def inner(instance):
1547 ctx = d.Context(name=instance.dev_name)
1548 if ctx.query_device().max_mcast_grp == 0:
1549 raise unittest.SkipTest('Multicast is not supported on this device')
1550 return func(instance)
1551 return inner
1552 return outer
1553
1554
1555 def odp_supported(ctx, qp_type, required_odp_caps):
1556 """
1557 Check device ODP capabilities
1558 :param ctx: Device Context
1559 :param qp_type: QP type ('rc', 'ud' or 'uc')
1560 :param required_odp_caps: ODP Capability mask of specified device
1561 :return: None
1562 """
1563 odp_caps = ctx.query_device_ex().odp_caps
1564 if odp_caps.general_caps == 0:
1565 raise unittest.SkipTest('ODP is not supported - No ODP caps')
1566 qp_odp_caps = getattr(odp_caps, '{}_odp_caps'.format(qp_type))
1567 if required_odp_caps & qp_odp_caps != required_odp_caps:
1568 raise unittest.SkipTest('ODP is unavailable - Operation not supported on this device')
1569
1570
1571 def odp_implicit_supported(ctx):
1572 """
1573 Check device ODP implicit capability.
1574 :param ctx: Device Context
1575 :return: None
1576 """
1577 odp_caps = ctx.query_device_ex().odp_caps
1578 has_odp_implicit = odp_caps.general_caps & ibv_odp_general_caps.IBV_ODP_SUPPORT_IMPLICIT
1579 if has_odp_implicit == 0:
1580 raise unittest.SkipTest('ODP implicit is not supported')
1581
1582
1583 def odp_v2_supported(ctx):
1584 """
1585 ODPv2 check
1586 :return: True/False if ODPv2 supported
1587 """
1588 from tests.mlx5_prm_structs import QueryHcaCapIn, QueryOdpCapOut, DevxOps, QueryHcaCapMod
1589 query_cap_in = QueryHcaCapIn(op_mod=DevxOps.MLX5_CMD_OP_QUERY_ODP_CAP << 1 | \
1590 QueryHcaCapMod.CURRENT)
1591 cmd_res = ctx.devx_general_cmd(query_cap_in, len(QueryOdpCapOut()))
1592 query_cap_out = QueryOdpCapOut(cmd_res)
1593 if query_cap_out.status:
1594 raise PyverbsRDMAError(f'QUERY_HCA_CAP has failed with status ({query_cap_out.status}) '
1595 f'and syndrome ({query_cap_out.syndrome})')
1596 return query_cap_out.capability.mem_page_fault == 1
1597
1598
1599 def requires_odpv2(func):
1600 def inner(instance):
1601 if not odp_v2_supported(instance.ctx):
1602 raise unittest.SkipTest('ODPv2 is not supported')
1603 return func(instance)
1604 return inner
1605
1606
1607 def get_pci_name(dev_name):
1608 pci_name = glob.glob(f'/sys/bus/pci/devices/*/infiniband/{dev_name}')
1609 if not pci_name:
1610 raise unittest.SkipTest(f'Could not find the PCI device of {dev_name}')
1611 return pci_name[0].split('/')[5]
1612
1613
1614 def requires_eswitch_on(func):
1615 def inner(instance):
1616 if not (is_eth(d.Context(name=instance.dev_name), instance.ib_port)
1617 and eswitch_mode_check(instance.dev_name)):
1618 raise unittest.SkipTest('Must be run on Ethernet link layer with Eswitch on')
1619 return func(instance)
1620 return inner
1621
1622
1623 def eswitch_mode_check(dev_name):
1624 pci_name = get_pci_name(dev_name)
1625 eswicth_off_msg = f'Device {dev_name} must be in switchdev mode'
1626 try:
1627 cmd_out = subprocess.check_output(['devlink', 'dev', 'eswitch', 'show', f'pci/{pci_name}'],
1628 stderr=subprocess.DEVNULL)
1629 if 'switchdev' not in str(cmd_out):
1630 raise unittest.SkipTest(eswicth_off_msg)
1631 except subprocess.CalledProcessError:
1632 raise unittest.SkipTest(eswicth_off_msg)
1633 return True
1634
1635
1636 def requires_roce_disabled(func):
1637 def inner(instance):
1638 if is_roce_enabled(instance.dev_name):
1639 raise unittest.SkipTest('ROCE must be disabled')
1640 return func(instance)
1641 return inner
1642
1643
1644 def is_roce_enabled(dev_name):
1645 pci_name = get_pci_name(dev_name)
1646 cmd_out = subprocess.check_output(['devlink', 'dev', 'param', 'show', f'pci/{pci_name}',
1647 'name', 'enable_roce'],
1648 stderr=subprocess.DEVNULL)
1649 if 'value true' in str(cmd_out):
1650 return True
1651 return False
1652
1653
1654 def requires_encap_disabled_if_eswitch_on(func):
1655 def inner(instance):
1656 if not (is_eth(d.Context(name=instance.dev_name), instance.ib_port)
1657 and encap_mode_check(instance.dev_name)):
1658 raise unittest.SkipTest('Encap must be disabled when Eswitch on')
1659 return func(instance)
1660 return inner
1661
1662
1663 def encap_mode_check(dev_name):
1664 pci_name = get_pci_name(dev_name)
1665 encap_enable_msg = f'Device {dev_name}: Encap must be disabled over switchdev mode'
1666 try:
1667 cmd_out = subprocess.check_output(['devlink', 'dev', 'eswitch', 'show', f'pci/{pci_name}'],
1668 stderr=subprocess.DEVNULL)
1669 if 'switchdev' in str(cmd_out):
1670 if any([i for i in ['encap enable', 'encap-mode basic'] if i in str(cmd_out)]):
1671 raise unittest.SkipTest(encap_enable_msg)
1672 except subprocess.CalledProcessError:
1673 raise unittest.SkipTest(encap_enable_msg)
1674 return True
1675
1676 def requires_huge_pages():
1677 def outer(func):
1678 def inner(instance):
1679 huge_pages_supported()
1680 return func(instance)
1681 return inner
1682 return outer
1683
1684
1685 def skip_unsupported(func):
1686 def func_wrapper(*args, **kwargs):
1687 try:
1688 return func(*args, **kwargs)
1689 except PyverbsRDMAError as ex:
1690 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
1691 raise unittest.SkipTest(f'Operation not supported ({str(ex)})')
1692 raise ex
1693 return func_wrapper
1694
1695
1696 def huge_pages_supported():
1697 """
1698 Check if huge pages are supported in the kernel.
1699 :return: None
1700 """
1701 huge_path = '/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages'
1702 if not os.path.isfile(huge_path):
1703 raise unittest.SkipTest('Huge pages of size 2M is not supported in this platform')
1704 with open(huge_path, 'r') as f:
1705 if not int(f.read()):
1706 raise unittest.SkipTest('There are no huge pages of size 2M allocated')
1707
1708
1709 @skip_unsupported
1710 def query_nic_flow_table_caps(instance):
1711 from tests.mlx5_prm_structs import QueryHcaCapIn, QueryQosCapOut, QueryHcaCapOp, \
1712 QueryHcaCapMod, QueryCmdHcaNicFlowTableCapOut
1713 try:
1714 ctx = Mlx5Context(Mlx5DVContextAttr(), instance.dev_name)
1715 except PyverbsUserError as ex:
1716 raise unittest.SkipTest(f'Could not open mlx5 context ({ex})')
1717 except PyverbsRDMAError:
1718 raise unittest.SkipTest('Opening mlx5 context is not supported')
1719 # Query NIC Flow Table capabilities
1720 query_cap_in = QueryHcaCapIn(op_mod=(QueryHcaCapOp.HCA_NIC_FLOW_TABLE_CAP << 0x1) | \
1721 QueryHcaCapMod.CURRENT)
1722 cmd_res = ctx.devx_general_cmd(query_cap_in, len(QueryQosCapOut()))
1723 query_cap_out = QueryCmdHcaNicFlowTableCapOut(cmd_res)
1724 if query_cap_out.status:
1725 raise PyverbsRDMAError(f'QUERY_HCA_CAP has failed with status ({query_cap_out.status}) '
1726 f'and syndrome ({query_cap_out.syndrome})')
1727 return query_cap_out.capability
1728
1729
1730 def prefetch_mrs(agr_obj, sg_list, advice=_IBV_ADVISE_MR_ADVICE_PREFETCH_WRITE,
1731 flags=_IBV_ADVISE_MR_FLAG_FLUSH):
1732 """
1733 Pre-fetch a range of an on-demand paging MR.
1734 :param agr_obj: Aggregation object which contains all resources necessary
1735 :param sg_list: SGE list
1736 :param advice: The requested advice value
1737 :param flags: Describes the properties of the advice operation
1738 :return: None
1739 """
1740 try:
1741 agr_obj.pd.advise_mr(advice, flags, sg_list)
1742 except PyverbsRDMAError as ex:
1743 if ex.error_code == errno.EOPNOTSUPP:
1744 raise unittest.SkipTest(f'Advise MR with flags ({flags}) and advice ({advice}) is not supported')
1745 raise ex
1746
1747
1748 def is_eth(ctx, port_num):
1749 """
1750 Querires the device's context's <port_num> port for its link layer.
1751 :param ctx: The Context to query
1752 :param port_num: Which Context's port to query
1753 :return: True if the port's link layer is Ethernet, else False
1754 """
1755 return ctx.query_port(port_num).link_layer == IBV_LINK_LAYER_ETHERNET
1756
1757
1758 def is_datagram_qp(agr_obj):
1759 if agr_obj.qp.qp_type == ibv_qp_type.IBV_QPT_UD or \
1760 isinstance(agr_obj, SRDResources) or \
1761 isinstance(agr_obj, Mlx5DcResources):
1762 return True
1763 return False
1764
1765
1766 def is_root():
1767 return os.geteuid() == 0
1768
1769
1770 def has_cap_net_raw():
1771 """
1772 Check if the current process has CAP_NET_RAW capability.
1773 """
1774 try:
1775 with open('/proc/self/status', 'r') as f:
1776 for line in f:
1777 if line.startswith('CapEff:'):
1778 # Parse the effective capabilities (hex format)
1779 cap_eff = int(line.split()[1], 16)
1780 return bool(cap_eff & 1 << 13)
1781 except (IOError, ValueError, IndexError):
1782 pass
1783 return False
1784
1785
1786 def requires_cap_net_raw():
1787 def outer(func):
1788 def inner(instance):
1789 if not has_cap_net_raw():
1790 raise unittest.SkipTest('The process must have CAP_NET_RAW set')
1791 return func(instance)
1792 return inner
1793 return outer
1794
1795
1796 def post_rq_state_bad_flow(test_obj):
1797 """
1798 Check post_recive on rq while qp is in invalid state.
1799 - Change qp's state to IBV_QPS_RESET
1800 - Verify post receive on qp fails
1801 :param test_obj: An instance of RDMATestCase
1802 :return: None.
1803 """
1804 qp_attr = QPAttr(qp_state=ibv_qp_state.IBV_QPS_RESET, cur_qp_state=ibv_qp_state.IBV_QPS_RTS)
1805 test_obj.server.qps[0].modify(qp_attr, ibv_qp_attr_mask.IBV_QP_STATE)
1806 recv_wr = get_recv_wr(test_obj.server)
1807 with test_obj.assertRaises(PyverbsRDMAError) as ex:
1808 post_recv(test_obj.server, recv_wr, qp_idx=0)
1809 test_obj.assertEqual(ex.exception.error_code, errno.EINVAL)
1810
1811
1812 def post_sq_state_bad_flow(test_obj):
1813 """
1814 Check post_send on sq while qp is in invalid state.
1815 - Change qp's state to IBV_QPS_RESET
1816 - Verify post send on qp fails
1817 :param test_obj: An instance of RDMATestCase
1818 :return: None.
1819 """
1820 qp_idx = 0
1821 qp_attr = QPAttr(qp_state=ibv_qp_state.IBV_QPS_RESET, cur_qp_state=ibv_qp_state.IBV_QPS_RTS)
1822 test_obj.client.qps[qp_idx].modify(qp_attr, ibv_qp_attr_mask.IBV_QP_STATE)
1823 ah = get_global_ah(test_obj.client, test_obj.gid_index, test_obj.ib_port)
1824 _, sg = get_send_elements(test_obj.client, False)
1825 with test_obj.assertRaises(PyverbsRDMAError) as ex:
1826 send(test_obj.client, sg, ibv_wr_opcode.IBV_WR_SEND, new_send=True,
1827 qp_idx=qp_idx, ah=ah)
1828 test_obj.assertEqual(ex.exception.error_code, errno.EINVAL)
1829
1830
1831 def full_rq_bad_flow(test_obj):
1832 """
1833 Check post_recive while qp's rq is full.
1834 - Find qp's rq length.
1835 - Fill the qp with work requests until overflow.
1836 :param test_obj: An instance of RDMATestCase
1837 :return: None.
1838 """
1839 qp_attr, _ = test_obj.server.qps[0].query(ibv_qp_attr_mask.IBV_QP_CAP)
1840 max_recv_wr = qp_attr.cap.max_recv_wr
1841 with test_obj.assertRaises(PyverbsRDMAError) as ex:
1842 for _ in range (max_recv_wr + 1):
1843 s_recv_wr = get_recv_wr(test_obj.server)
1844 post_recv(test_obj.server, s_recv_wr, qp_idx=0)
1845 test_obj.assertEqual(ex.exception.error_code, errno.ENOMEM)
1846
1847
1848 def create_rq_with_larger_sgl_bad_flow(test_obj):
1849 """
1850 Check post_receive on qp while wr sgl is bigger than
1851 max sge allowed for the qp
1852 - Find max sge allowed for the qp
1853 - Create wr with sgl bigger than the max
1854 - Verify post receive on qp fails
1855 :param test_obj: An instance of RDMATestCase
1856 :return: None.
1857 """
1858 qp_idx = 0
1859 server_mr = test_obj.server.mr
1860 server_mr_buf = server_mr.buf
1861 qp_attr, _ = test_obj.server.qps[qp_idx].query(ibv_qp_attr_mask.IBV_QP_CAP)
1862 max_recv_sge = qp_attr.cap.max_recv_sge
1863 length = test_obj.server.msg_size // (max_recv_sge + 1)
1864 sgl = []
1865 offset = 0
1866 for _ in range(max_recv_sge + 1):
1867 sgl.append(SGE(server_mr_buf + offset, length, server_mr.lkey))
1868 offset = offset + length
1869 s_recv_wr = RecvWR(sg=sgl, num_sge=max_recv_sge + 1)
1870 with test_obj.assertRaises(PyverbsRDMAError) as ex:
1871 post_recv(test_obj.server, s_recv_wr, qp_idx=qp_idx)
1872 test_obj.assertEqual(ex.exception.error_code, errno.EINVAL)
1873
1874
1875 def high_rate_send(agr_obj, packet, rate_limit, timeout=2):
1876 """
1877 Sends packet at high rate for 'timeout' seconds.
1878 :param agr_obj: Aggregation object which contains all resources necessary
1879 :param packet: Packet to send
1880 :param rate_limit: Minimal rate limit in MBps
1881 :param timeout: Seconds to send the packets
1882 """
1883 send_sg = SGE(agr_obj.mr.buf, len(packet), agr_obj.mr.lkey)
1884 agr_obj.mr.write(packet, len(packet))
1885 send_wr = SendWR(num_sge=1, sg=[send_sg])
1886 poll = poll_cq_ex if isinstance(agr_obj.cq, CQEX) else poll_cq
1887 iterations = 0
1888 start_send_t = time.perf_counter()
1889 while (time.perf_counter() - start_send_t) < timeout:
1890 agr_obj.qp.post_send(send_wr)
1891 poll(agr_obj.cq)
1892 iterations += 1
1893 # Calculate the rate
1894 rate = agr_obj.msg_size * iterations / timeout / 1000000
1895 assert rate > rate_limit, 'Traffic rate is smaller than minimal rate for the test'
1896
1897
1898 def get_pkey_from_kernel(device, port=1, index=0):
1899 path = f'/sys/class/infiniband/{device}/ports/{port}/pkeys/{index}'
1900 output = subprocess.check_output(['cat', path], universal_newlines=True)
1901 pkey_hex = output.strip()
1902 pkey_decimal = int(pkey_hex, 16)
1903 return pkey_decimal