Debian Codegraph @

tests/test_mlx5_dm_ops.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2020 NVIDIA Corporation . All rights reserved. See COPYING file
3
4 from threading import Thread
5 from queue import Queue
6 import unittest
7 import struct
8 import errno
9
10 from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \
11 Mlx5DmOpAddr
12 from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError
13 from tests.mlx5_base import Mlx5PyverbsAPITestCase
14 from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_context_attr_flags
15 import pyverbs.device as d
16
17
18 MEMIC_ATOMIC_INCREMENT = 0x0
19 MEMIC_ATOMIC_TEST_AND_SET = 0x1
20
21 MLX5_CMD_OP_QUERY_HCA_CAP = 0x100
22 MLX5_CMD_MOD_DEVICE_MEMORY_CAP = 0xF
23 MLX5_CMD_OP_QUERY_HCA_CAP_OUT_LEN = 0x1010
24
25
26 def requires_memic_atomic_support(func):
27 def wrapper(instance):
28 cmd_in = struct.pack('!HIH8s', MLX5_CMD_OP_QUERY_HCA_CAP, 0,
29 MLX5_CMD_MOD_DEVICE_MEMORY_CAP << 1 | 0x1,
30 bytes(8))
31 cmd_out = Mlx5Context.devx_general_cmd(instance.ctx, cmd_in,
32 MLX5_CMD_OP_QUERY_HCA_CAP_OUT_LEN)
33 cmd_view = memoryview(cmd_out)
34 status = cmd_view[0]
35 if status:
36 raise PyverbsRDMAError('Query Device Memory CAPs failed with status'
37 f' ({status})')
38
39 memic_op_support = int.from_bytes(cmd_view[80:84], 'big')
40 increment_size_sup = cmd_view[20]
41 test_and_set_size_sup = cmd_view[22]
42 # Verify that MEMIC atomic operations (both increment and test_and_set)
43 # are supported with write/read size of 1 Byte.
44 if memic_op_support & 0x3 != 0x3:
45 raise unittest.SkipTest('MEMIC atomic operations are not supported')
46 if not increment_size_sup & test_and_set_size_sup & 0x1:
47 raise unittest.SkipTest(
48 'MEMIC atomic operations are not supported with 1 Bytes read/write sizes')
49 return func(instance)
50 return wrapper
51
52
53 class Mlx5DmOpAddresses(Mlx5PyverbsAPITestCase):
54 def setUp(self):
55 super().setUp()
56 self.dm_size = int(self.attr_ex.max_dm_size / 2)
57
58 def create_context(self):
59 try:
60 attr = Mlx5DVContextAttr(mlx5dv_context_attr_flags.MLX5DV_CONTEXT_FLAGS_DEVX)
61 self.ctx = Mlx5Context(attr, self.dev_name)
62 except PyverbsUserError as ex:
63 raise unittest.SkipTest(f'Could not open mlx5 context ({ex})')
64 except PyverbsRDMAError:
65 raise unittest.SkipTest('Opening mlx5 DevX context is not supported')
66
67 def _write_to_op_addr(self):
68 try:
69 inc_addr = Mlx5DmOpAddr(self.dm, MEMIC_ATOMIC_INCREMENT)
70 except PyverbsRDMAError as ex:
71 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
72 self.skip_queue.put(unittest.SkipTest(
73 'MEMIC_ATOMIC_INCREMENT op is not supported'))
74 return
75 raise ex
76 inc_addr.write(b'\x01')
77 inc_addr.unmap(self.dm_size)
78
79 def _read_from_op_addr(self):
80 try:
81 test_and_set_addr = Mlx5DmOpAddr(self.dm, MEMIC_ATOMIC_TEST_AND_SET)
82 except PyverbsRDMAError as ex:
83 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
84 self.skip_queue.put(unittest.SkipTest(
85 'MEMIC_ATOMIC_TEST_AND_SET op is not supported'))
86 return
87 raise ex
88 val = test_and_set_addr.read(1)
89 test_and_set_addr.unmap(self.dm_size)
90 return val
91
92 @requires_memic_atomic_support
93 def test_dm_atomic_ops(self):
94 """
95 Tests "increment" and "test_and_set" MEMIC atomic operations.
96 The test does two increments to the same buffer data and verifies the
97 values using test_and_set.
98 Then verifies that the latter op sets the buffer as expected.
99 """
100 with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm:
101 # Set DM buffer to 0
102 dm.copy_to_dm(0, bytes(self.dm_size), self.dm_size)
103 try:
104 inc_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_INCREMENT)
105 test_and_set_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_TEST_AND_SET)
106 except PyverbsRDMAError as ex:
107 if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
108 raise unittest.SkipTest('MEMIC atomic operations are not supported')
109 raise ex
110 inc_addr.write(b'\x01')
111 inc_addr.write(b'\x01')
112 # Now we should read 0x02 and the memory set to 0x1
113 val = int.from_bytes(test_and_set_addr.read(1), 'big')
114 self.assertEqual(val, 2)
115 # Verify that TEST_AND_SET set the memory to 0x1
116 val = int.from_bytes(test_and_set_addr.read(1), 'big')
117 self.assertEqual(val, 1)
118 inc_addr.unmap(self.dm_size)
119 test_and_set_addr.unmap(self.dm_size)
120
121 @requires_memic_atomic_support
122 def test_parallel_dm_atomic_ops(self):
123 """
124 Runs multiple threads that do test_and_set operation, followed by
125 multiple threads that do increments of +1, to the same DM buffer.
126 Then verifies that the buffer data was incremented as expected.
127 """
128 threads = []
129 num_threads = 10
130 self.skip_queue = Queue()
131 with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as self.dm:
132 for _ in range(num_threads):
133 threads.append(Thread(target=self._read_from_op_addr))
134 threads[-1].start()
135 for thread in threads:
136 thread.join()
137
138 threads = []
139 for _ in range(num_threads):
140 threads.append(Thread(target=self._write_to_op_addr))
141 threads[-1].start()
142 for thread in threads:
143 thread.join()
144
145 if not self.skip_queue.empty():
146 raise self.skip_queue.get()
147
148 val = int.from_bytes(self._read_from_op_addr(), 'big')
149 self.assertEqual(val, num_threads + 1,
150 f'Read value is ({val}) is different than expected ({num_threads+1})' )