Debian Codegraph @

tests/test_parent_domain.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
3 """
4 Test module for Pyverbs' ParentDomain.
5 """
6 from pyverbs.pd import ParentDomainInitAttr, ParentDomain, ParentDomainContext
7 from tests.base import RCResources, UDResources, RDMATestCase
8 from pyverbs.pyverbs_error import PyverbsRDMAError
9 from pyverbs.cq import CqInitAttrEx, CQEX
10 import pyverbs.mem_alloc as mem
11 from pyverbs.libibverbs_enums import _IBV_ALLOCATOR_USE_DEFAULT, IBV_WC_STANDARD_FLAGS, ibv_cq_init_attr_mask
12 import tests.utils as u
13
14 import unittest
15 import errno
16
17
18 HUGE_PAGE_SIZE = 0x200000
19
20
21 def default_allocator(pd, context, size, alignment, resource_type):
22 return _IBV_ALLOCATOR_USE_DEFAULT
23
24
25 def default_free(pd, context, ptr, resource_type):
26 return _IBV_ALLOCATOR_USE_DEFAULT
27
28
29 def mem_align_allocator(pd, context, size, alignment, resource_type):
30 p = mem.posix_memalign(size, alignment)
31 return p
32
33
34 def free_func(pd, context, ptr, resource_type):
35 mem.free(ptr)
36
37
38 def huge_page_alloc(pd, context, size, alignment, resource_type):
39 ptr = context.user_data
40 remainder = ptr % alignment
41 ptr += 0 if remainder == 0 else (alignment - remainder)
42 context.user_data += size
43 return ptr
44
45
46 def huge_page_free(pd, context, ptr, resource_type):
47 """
48 No need to free memory, since this allocator assumes the huge page was
49 externally mapped (and will be externally un-mapped).
50 """
51 pass
52
53
54 def create_parent_domain_with_allocators(res):
55 """
56 Creates parent domain for res instance. The allocators themselves are taken
57 from res.allocator_func and res.free_func.
58 :param res: The resources instance to work on (an instance of BaseResources)
59 """
60 if res.allocator_func and res.free_func:
61 res.pd_ctx = ParentDomainContext(res.pd, res.allocator_func,
62 res.free_func, res.user_data)
63 pd_attr = ParentDomainInitAttr(pd=res.pd, pd_context=res.pd_ctx)
64 try:
65 res.pd = ParentDomain(res.ctx, attr=pd_attr)
66 except PyverbsRDMAError as ex:
67 if ex.error_code == errno.EOPNOTSUPP:
68 raise unittest.SkipTest('Parent Domain is not supported on this device')
69 raise ex
70
71
72 def parent_domain_res_cls(base_class):
73 """
74 This is a factory function which creates a class that inherits base_class of
75 any BaseResources type. Its purpose is to behave exactly as base_class does,
76 except for creating a parent domain with custom allocators.
77 Hence the returned class must be initialized with (alloc_func, free_func,
78 user_data, **kwargs), while kwargs are the arguments needed (if any) for
79 base_class.
80 :param base_class: The base resources class to inherit from
81 :return: ParentDomainRes(alloc_func=None, free_func=None, **kwargs) class
82 """
83 class ParentDomainRes(base_class):
84 def __init__(self, alloc_func=None, free_func=None, user_data=None, **kwargs):
85 self.pd_ctx = None
86 self.protection_domain = None
87 self.allocator_func = alloc_func
88 self.free_func = free_func
89 self.user_data = user_data
90 super().__init__(**kwargs)
91
92 def create_pd(self):
93 super().create_pd()
94 self.protection_domain = self.pd
95 create_parent_domain_with_allocators(self)
96
97 return ParentDomainRes
98
99
100 class ParentDomainHugePageRcRes(parent_domain_res_cls(RCResources)):
101 def __init__(self, alloc_func=None, free_func=None, **kwargs):
102 user_data = mem.mmap(length=HUGE_PAGE_SIZE,
103 flags=mem.MAP_ANONYMOUS_ | mem.MAP_PRIVATE_ | mem.MAP_HUGETLB_)
104 super().__init__(alloc_func=alloc_func, free_func=free_func,
105 user_data=user_data, **kwargs)
106
107 def __del__(self):
108 mem.munmap(self.user_data, HUGE_PAGE_SIZE)
109
110
111 class ParentDomainCqExSrqRes(parent_domain_res_cls(RCResources)):
112 """
113 Parent domain resources. Based on RCResources.
114 This includes a parent domain created with the given allocators, in addition
115 it creates an extended CQ and a SRQ for RC traffic.
116 :param dev_name: Device name to be used
117 :param ib_port: IB port of the device to use
118 :param gid_index: Which GID index to use
119 :param alloc_func: Custom allocator function
120 :param free_func: Custom free function
121 """
122 def __init__(self, dev_name, ib_port=None, gid_index=None, alloc_func=None,
123 free_func=None):
124 super().__init__(dev_name=dev_name, ib_port=ib_port,
125 gid_index=gid_index, alloc_func=alloc_func,
126 free_func=free_func, with_srq=True)
127
128 def create_cq(self):
129 wc_flags = IBV_WC_STANDARD_FLAGS
130 cia = CqInitAttrEx(cqe=2000, wc_flags=wc_flags, parent_domain=self.pd,
131 comp_mask=ibv_cq_init_attr_mask.IBV_CQ_INIT_ATTR_MASK_FLAGS |
132 ibv_cq_init_attr_mask.IBV_CQ_INIT_ATTR_MASK_PD)
133 try:
134 self.cq = CQEX(self.ctx, cia)
135 except PyverbsRDMAError as ex:
136 if ex.error_code == errno.EOPNOTSUPP:
137 raise unittest.SkipTest('Extended CQ with Parent Domain is not supported')
138 raise ex
139
140
141 class ParentDomainTrafficTest(RDMATestCase):
142 def setUp(self):
143 super().setUp()
144 self.iters = 10
145 self.server = None
146 self.client = None
147
148 def test_without_allocators_rc_traffic(self):
149 parent_domain_rc_res = parent_domain_res_cls(RCResources)
150 self.create_players(parent_domain_rc_res)
151 u.traffic(**self.traffic_args)
152
153 def test_default_allocators_rc_traffic(self):
154 parent_domain_rc_res = parent_domain_res_cls(RCResources)
155 self.create_players(parent_domain_rc_res, alloc_func=default_allocator,
156 free_func=default_free)
157 u.traffic(**self.traffic_args)
158
159 def test_mem_align_rc_traffic(self):
160 parent_domain_rc_res = parent_domain_res_cls(RCResources)
161 self.create_players(parent_domain_rc_res,
162 alloc_func=mem_align_allocator, free_func=free_func)
163 u.traffic(**self.traffic_args)
164
165 def test_mem_align_ud_traffic(self):
166 parent_domain_ud_res = parent_domain_res_cls(UDResources)
167 self.create_players(parent_domain_ud_res,
168 alloc_func=mem_align_allocator, free_func=free_func)
169 u.traffic(**self.traffic_args)
170
171 def test_mem_align_srq_excq_rc_traffic(self):
172 self.create_players(ParentDomainCqExSrqRes,
173 alloc_func=mem_align_allocator, free_func=free_func)
174 u.traffic(**self.traffic_args, is_cq_ex=True)
175
176 @u.requires_huge_pages()
177 def test_huge_page_traffic(self):
178 self.create_players(ParentDomainHugePageRcRes,
179 alloc_func=huge_page_alloc, free_func=huge_page_free)
180 u.traffic(**self.traffic_args)