Debian Codegraph @

tests/test_device.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2018 Mellanox Technologies, Inc. All rights reserved. See COPYING file
3 # Copyright 2020 Amazon.com, Inc. or its affiliates. All rights reserved.
4 """
5 Test module for pyverbs' device module.
6 """
7 from multiprocessing import Process, Queue
8 import unittest
9 import resource
10 import random
11 import errno
12 import os
13
14 from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError
15 from tests.base import PyverbsAPITestCase
16 from pyverbs.device import Context, DM
17 import tests.utils as u
18 import pyverbs.device as d
19 from pyverbs.libibverbs_enums import ibv_node_type
20
21 PAGE_SIZE = resource.getpagesize()
22
23
24 class DeviceTest(PyverbsAPITestCase):
25 """
26 Test various functionalities of the Device class.
27 """
28
29 def get_device_list(self):
30 lst = d.get_device_list()
31 if len(lst) == 0:
32 raise unittest.SkipTest('No IB device found')
33 dev_name = self.config['dev']
34 if dev_name:
35 for dev in lst:
36 if dev.name.decode() == dev_name:
37 lst = [dev]
38 break
39 if len(lst) == 0:
40 raise PyverbsRDMAError(f'No IB device with name {dev_name} found')
41 return lst
42
43 def test_dev_list(self):
44 """
45 Verify that it's possible to get IB devices list.
46 """
47 self.get_device_list()
48
49 def test_open_dev(self):
50 """
51 Test ibv_open_device()
52 """
53 for dev in self.get_device_list():
54 d.Context(name=dev.name.decode())
55
56 def test_query_device(self):
57 """
58 Test ibv_query_device()
59 """
60 for dev in self.get_device_list():
61 with d.Context(name=dev.name.decode()) as ctx:
62 attr = ctx.query_device()
63 self.verify_device_attr(attr, dev)
64
65 def test_query_pkey(self):
66 """
67 Test ibv_query_pkey()
68 """
69 for dev in self.get_device_list():
70 with d.Context(name=dev.name.decode()) as ctx:
71 if dev.node_type == ibv_node_type.IBV_NODE_CA:
72 ctx.query_pkey(port_num=self.ib_port, index=0)
73
74 def test_get_pkey_index(self):
75 """
76 Test ibv_get_pkey_index()
77 """
78 source_pkey_index = 0
79 for dev in self.get_device_list():
80 with d.Context(name=dev.name.decode()) as ctx:
81 if dev.node_type == ibv_node_type.IBV_NODE_CA:
82 pkey = u.get_pkey_from_kernel(device=dev.name.decode(),
83 port=self.ib_port,
84 index=source_pkey_index)
85 queried_pkey_idx = ctx.get_pkey_index(port_num=self.ib_port,
86 pkey=pkey)
87 self.assertEqual(queried_pkey_idx, source_pkey_index,
88 f'Got index={queried_pkey_idx}\nExpected index={source_pkey_index}')
89
90 def test_query_gid(self):
91 """
92 Test ibv_query_gid()
93 """
94 for dev in self.get_device_list():
95 with d.Context(name=dev.name.decode()) as ctx:
96 gid_tbl_len = ctx.query_port(self.ib_port).gid_tbl_len
97 if gid_tbl_len > 0:
98 ctx.query_gid(port_num=self.ib_port, index=0)
99
100 def test_query_gid_table(self):
101 """
102 Test ibv_query_gid_table()
103 """
104 for dev in self.get_device_list():
105 with d.Context(name=dev.name.decode()) as ctx:
106 device_attr = ctx.query_device()
107 max_entries = 0
108 for port_num in range(1, device_attr.phys_port_cnt + 1):
109 port_attr = ctx.query_port(port_num)
110 max_entries += port_attr.gid_tbl_len
111 try:
112 if max_entries > 0:
113 ctx.query_gid_table(max_entries)
114 except PyverbsRDMAError as ex:
115 if ex.error_code in [-errno.EOPNOTSUPP, -errno.EPROTONOSUPPORT]:
116 raise unittest.SkipTest('ibv_query_gid_table is not'\
117 ' supported on this device')
118 raise ex
119
120 def test_query_gid_table_bad_flow(self):
121 """
122 Test ibv_query_gid_table() with too small a buffer
123 """
124 try:
125 self.ctx.query_gid_table(0)
126 except PyverbsRDMAError as ex:
127 if ex.error_code in [-errno.EOPNOTSUPP, -errno.EPROTONOSUPPORT]:
128 raise unittest.SkipTest('ibv_query_gid_table is not'
129 ' supported on this device')
130 self.assertEqual(ex.error_code, -errno.EINVAL,
131 f'Got -{os.strerror(-ex.error_code)} but '
132 f'Expected -{os.strerror(errno.EINVAL)} ')
133 else:
134 raise PyverbsRDMAError('Successfully queried '
135 'gid_table with an insufficient buffer')
136
137 def test_query_gid_ex(self):
138 """
139 Test ibv_query_gid_ex()
140 """
141 for dev in self.get_device_list():
142 with d.Context(name=dev.name.decode()) as ctx:
143 try:
144 gid_tbl_len = ctx.query_port(self.ib_port).gid_tbl_len
145 if gid_tbl_len > 0:
146 ctx.query_gid_ex(port_num=self.ib_port, gid_index=0)
147 except PyverbsRDMAError as ex:
148 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
149 raise unittest.SkipTest('ibv_query_gid_ex is not'\
150 ' supported on this device')
151 raise ex
152
153 def test_query_gid_ex_bad_flow(self):
154 """
155 Test ibv_query_gid_ex() with an empty index
156 """
157 try:
158 port_attr = self.ctx.query_port(self.ib_port)
159 max_entries = 0
160 for port_num in range(1, self.attr.phys_port_cnt + 1):
161 attr = self.ctx.query_port(port_num)
162 max_entries += attr.gid_tbl_len
163 if max_entries > 0:
164 gid_indices = {gid_entry.gid_index for gid_entry in
165 self.ctx.query_gid_table(max_entries) if gid_entry.port_num == self.ib_port}
166 else:
167 gid_indices = {}
168
169 possible_indices = set(range(port_attr.gid_tbl_len)) if port_attr.gid_tbl_len > 1 else set()
170 try:
171 no_gid_index = possible_indices.difference(gid_indices).pop()
172 except KeyError:
173 # all indices are populated by GIDs
174 raise unittest.SkipTest('All gid indices populated,'
175 ' cannot check bad flow')
176
177 self.ctx.query_gid_ex(port_num=self.ib_port, gid_index=no_gid_index)
178 except PyverbsRDMAError as ex:
179 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
180 raise unittest.SkipTest('ibv_query_gid_ex is not'
181 ' supported on this device')
182 self.assertEqual(ex.error_code, errno.ENODATA,
183 f'Got {os.strerror(ex.error_code)} but '
184 f'Expected {os.strerror(errno.ENODATA)}')
185 else:
186 raise PyverbsRDMAError('Successfully queried '
187 f'non-existent gid index {no_gid_index}')
188
189 @staticmethod
190 def verify_device_attr(attr, device):
191 """
192 Helper method that verifies correctness of some members of DeviceAttr
193 object.
194 :param attr: A DeviceAttr object
195 :param device: A Device object
196 :return: None
197 """
198 if device.node_type != ibv_node_type.IBV_NODE_UNSPECIFIED and device.node_type != ibv_node_type.IBV_NODE_UNKNOWN:
199 assert attr.node_guid != 0
200 assert attr.sys_image_guid != 0
201 assert attr.max_mr_size > PAGE_SIZE
202 assert attr.page_size_cap >= PAGE_SIZE
203 assert attr.vendor_id != 0
204 assert attr.max_qp > 0
205 assert attr.max_qp_wr > 0
206 assert attr.max_sge > 0
207 assert attr.max_sge_rd >= 0
208 assert attr.max_cq > 0
209 assert attr.max_cqe > 0
210 assert attr.max_mr > 0
211 assert attr.max_pd > 0
212 if device.node_type == ibv_node_type.IBV_NODE_CA:
213 assert attr.max_pkeys > 0
214
215 def test_query_device_ex(self):
216 """
217 Test ibv_query_device_ex()
218 """
219 for dev in self.get_device_list():
220 with d.Context(name=dev.name.decode()) as ctx:
221 attr_ex = ctx.query_device_ex()
222 self.verify_device_attr(attr_ex.orig_attr, dev)
223
224 def test_phys_port_cnt_ex(self):
225 """
226 Test phys_port_cnt_ex
227 """
228 for dev in self.get_device_list():
229 with d.Context(name=dev.name.decode()) as ctx:
230 attr_ex = ctx.query_device_ex()
231 phys_port_cnt = attr_ex.orig_attr.phys_port_cnt
232 phys_port_cnt_ex = attr_ex.phys_port_cnt_ex
233 if phys_port_cnt_ex > 255:
234 self.assertEqual(phys_port_cnt, 255,
235 f'phys_port_cnt should be 255 if ' +
236 f'phys_port_cnt_ex is bigger than 255')
237 else:
238 self.assertEqual(phys_port_cnt, phys_port_cnt_ex,
239 f'phys_port_cnt_ex and phys_port_cnt ' +
240 f'should be equal if number of ports is ' +
241 f'less than 256')
242
243 @staticmethod
244 def verify_port_attr(attr):
245 """
246 Helper method that verifies correctness of some members of PortAttr
247 object.
248 :param attr: A PortAttr object
249 :return: None
250 """
251 assert 'Invalid' not in d.phys_state_to_str(attr.state)
252 assert 'Invalid' not in d.translate_mtu(attr.max_mtu)
253 assert 'Invalid' not in d.translate_mtu(attr.active_mtu)
254 assert 'Invalid' not in d.width_to_str(attr.active_width)
255 assert 'Invalid' not in d.speed_to_str(attr.active_speed,
256 attr.active_speed_ex)
257 assert 'Invalid' not in d.translate_link_layer(attr.link_layer)
258 assert attr.max_msg_sz > 0x1000
259
260 def test_query_port(self):
261 """
262 Test ibv_query_port
263 """
264 for dev in self.get_device_list():
265 with d.Context(name=dev.name.decode()) as ctx:
266 port_attr = ctx.query_port(self.ib_port)
267 self.verify_port_attr(port_attr)
268
269 def test_query_port_bad_flow(self):
270 """
271 Verify that querying non-existing ports fails as expected
272 """
273 for dev in self.get_device_list():
274 with d.Context(name=dev.name.decode()) as ctx:
275 num_ports = ctx.query_device().phys_port_cnt
276 try:
277 port = num_ports + random.randint(1, 10)
278 ctx.query_port(port)
279 except PyverbsRDMAError as e:
280 assert 'Failed to query port' in e.args[0]
281 assert 'Invalid argument' in e.args[0]
282 else:
283 raise PyverbsRDMAError(
284 'Successfully queried non-existing port {p}'. \
285 format(p=port))
286
287 def test_query_port_speed(self):
288 """
289 Test ibv_query_port_speed
290 """
291 for dev in self.get_device_list():
292 with d.Context(name=dev.name.decode()) as ctx:
293 try:
294 port_speed = ctx.query_port_speed(self.ib_port)
295 except PyverbsRDMAError as ex:
296 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
297 raise unittest.SkipTest('ibv_query_port_speed is not'
298 ' supported on this device')
299 raise ex
300 self.assertGreaterEqual(port_speed, 0, 'Port speed should be positive, got'
301 f' {port_speed} Mbps')
302
303
304 class DMTest(PyverbsAPITestCase):
305 """
306 Test various functionalities of the DM class.
307 """
308 def setUp(self):
309 super().setUp()
310 if self.attr_ex.max_dm_size == 0:
311 raise unittest.SkipTest('Device memory is not supported')
312
313 def test_create_dm(self):
314 """
315 test ibv_alloc_dm()
316 """
317 dm_len = random.randrange(u.MIN_DM_SIZE, int(self.attr_ex.max_dm_size/2),
318 u.DM_ALIGNMENT)
319 dm_attrs = u.get_dm_attrs(dm_len)
320 with d.DM(self.ctx, dm_attrs):
321 pass
322
323 def test_destroy_dm(self):
324 """
325 test ibv_free_dm()
326 """
327 dm_len = random.randrange(u.MIN_DM_SIZE, int(self.attr_ex.max_dm_size/2),
328 u.DM_ALIGNMENT)
329 dm_attrs = u.get_dm_attrs(dm_len)
330 dm = d.DM(self.ctx, dm_attrs)
331 dm.close()
332
333 def test_create_dm_bad_flow(self):
334 """
335 test ibv_alloc_dm() with an illegal size and comp mask
336 """
337 dm_len = self.attr_ex.max_dm_size + 1
338 dm_attrs = u.get_dm_attrs(dm_len)
339 try:
340 d.DM(self.ctx, dm_attrs)
341 except PyverbsRDMAError as e:
342 assert 'Failed to allocate device memory of size' in \
343 e.args[0]
344 assert 'Max available size' in e.args[0]
345 else:
346 raise PyverbsError(
347 'Created a DM with size larger than max reported')
348 dm_attrs.comp_mask = random.randint(1, 100)
349 try:
350 d.DM(self.ctx, dm_attrs)
351 except PyverbsRDMAError as e:
352 assert 'Failed to allocate device memory of size' in \
353 e.args[0]
354 else:
355 raise PyverbsError(
356 'Created a DM with illegal comp mask {c}'. \
357 format(c=dm_attrs.comp_mask))
358
359 def test_destroy_dm_bad_flow(self):
360 """
361 Test calling ibv_free_dm() twice
362 """
363 dm_len = random.randrange(u.MIN_DM_SIZE, int(self.attr_ex.max_dm_size/2),
364 u.DM_ALIGNMENT)
365 dm_attrs = u.get_dm_attrs(dm_len)
366 dm = d.DM(self.ctx, dm_attrs)
367 dm.close()
368 dm.close()
369
370 def test_dm_write(self):
371 """
372 Test writing to the device memory
373 """
374 dm_len = random.randrange(u.MIN_DM_SIZE, int(self.attr_ex.max_dm_size/2),
375 u.DM_ALIGNMENT)
376 dm_attrs = u.get_dm_attrs(dm_len)
377 with d.DM(self.ctx, dm_attrs) as dm:
378 data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT)
379 data_offset = random.randrange(0, dm_len - data_length,
380 u.DM_ALIGNMENT)
381 data = 'a' * data_length
382 dm.copy_to_dm(data_offset, data.encode(), data_length)
383
384 def test_dm_write_bad_flow(self):
385 """
386 Test writing to the device memory with bad offset and length
387 """
388 dm_len = random.randrange(u.MIN_DM_SIZE, int(self.attr_ex.max_dm_size/2),
389 u.DM_ALIGNMENT)
390 dm_attrs = u.get_dm_attrs(dm_len)
391 with d.DM(self.ctx, dm_attrs) as dm:
392 data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT)
393 data_offset = random.randrange(0, dm_len - data_length,
394 u.DM_ALIGNMENT)
395 data_offset += 1 # offset needs to be a multiple of 4
396 data = 'a' * data_length
397 try:
398 dm.copy_to_dm(data_offset, data.encode(), data_length)
399 except PyverbsRDMAError as e:
400 assert 'Failed to copy to dm' in e.args[0]
401 else:
402 raise PyverbsError(
403 'Wrote to device memory with a bad offset')
404
405 def test_dm_read(self):
406 """
407 Test reading from the device memory
408 """
409 dm_len = random.randrange(u.MIN_DM_SIZE, int(self.attr_ex.max_dm_size/2),
410 u.DM_ALIGNMENT)
411 dm_attrs = u.get_dm_attrs(dm_len)
412 with d.DM(self.ctx, dm_attrs) as dm:
413 data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT)
414 data_offset = random.randrange(0, dm_len - data_length,
415 u.DM_ALIGNMENT)
416 data = 'a' * data_length
417 dm.copy_to_dm(data_offset, data.encode(), data_length)
418 read_str = dm.copy_from_dm(data_offset, data_length)
419 assert read_str.decode() == data
420
421 def alloc_dm(self, res_queue, size):
422 """
423 Alloc device memory. Used by multiple processes that allocate DMs in
424 parallel.
425 :param res_queue: Result Queue to return the result to the parent
426 process.
427 :param size: The DM allocation size.
428 :return: None
429 """
430 try:
431 d.DM(self.ctx, d.AllocDmAttr(length=size))
432 except PyverbsError as err:
433 res_queue.put(err.error_code)
434 res_queue.put(0)
435
436 def test_multi_process_alloc_dm(self):
437 """
438 Several processes try to allocate device memory simultaneously.
439 """
440 res_queue = Queue()
441 processes = []
442 processes_num = 5
443 # Dividing the max dm size by 2 since we're not
444 # guaranteed to have the max size free for us.
445 total_size = self.attr_ex.max_dm_size / 2 / processes_num
446 for i in range(processes_num):
447 processes.append(Process(target=self.alloc_dm,
448 args=(res_queue, total_size)))
449 for i in range(processes_num):
450 processes[i].start()
451 for i in range(processes_num):
452 processes[i].join()
453 rc = res_queue.get()
454 self.assertEqual(rc, 0, f'Parallel device memory allocation failed with errno: {rc}')
455
456
457 class SharedDMTest(PyverbsAPITestCase):
458 """
459 Tests shared device memory by importing DMs
460 """
461 def setUp(self):
462 super().setUp()
463 if self.attr_ex.max_dm_size == 0:
464 raise unittest.SkipTest('Device memory is not supported')
465 self.dm_size = int(self.attr_ex.max_dm_size / 2)
466
467 def test_import_dm(self):
468 """
469 Creates a DM and imports it from a different (duplicated) Context.
470 Then writes some data to the original DM, reads it from the imported DM
471 and verifies that the read data is as expected.
472 """
473 with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm:
474 cmd_fd_dup = os.dup(self.ctx.cmd_fd)
475 try:
476 imported_ctx = Context(cmd_fd=cmd_fd_dup)
477 imported_dm = DM(imported_ctx, handle=dm.handle)
478 except PyverbsRDMAError as ex:
479 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
480 raise unittest.SkipTest('Some object imports are not supported')
481 raise ex
482 original_data = b'\xab' * self.dm_size
483 dm.copy_to_dm(0, original_data, self.dm_size)
484 read_data = imported_dm.copy_from_dm(0, self.dm_size)
485 self.assertEqual(original_data, read_data)
486 imported_dm.unimport()