Debian Codegraph @

tests/base_rdmacm.py

raw
1 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
2 # Copyright (c) 2019 Mellanox Technologies, Inc . All rights reserved. See COPYING file
3
4 import abc
5
6 from pyverbs.cmid import CMID, AddrInfo, CMEventChannel, ConnParam, UDParam
7 from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP
8 from pyverbs.pyverbs_error import PyverbsUserError
9 from pyverbs.librdmacm_enums import rdma_port_space, RAI_PASSIVE
10 from pyverbs.libibverbs_enums import ibv_qp_type, ibv_qp_state
11 from pyverbs.cq import CQ
12
13
14 GRH_SIZE = 40
15 qp_type_per_ps = {rdma_port_space.RDMA_PS_TCP: ibv_qp_type.IBV_QPT_RC, rdma_port_space.RDMA_PS_UDP: ibv_qp_type.IBV_QPT_UD,
16 rdma_port_space.RDMA_PS_IPOIB : ibv_qp_type.IBV_QPT_UD}
17
18
19 class CMResources(abc.ABC):
20 """
21 CMResources class is an abstract base class which contains basic resources
22 for RDMA CM communication.
23 """
24 def __init__(self, passive=None, **kwargs):
25 """
26 :param addr: Local address to bind to.
27 :param passive: Indicate if this CM is the passive CM.
28 :param kwargs: Arguments:
29 * *port* (str)
30 Port number of the address
31 * *with_ext_qp* (bool)
32 If set, an external RC QP will be created and used by RDMACM
33 * *port_space* (str)
34 If set, indicates the CMIDs port space
35 """
36 self.qp_init_attr = None
37 self.passive = passive
38 self.with_ext_qp = kwargs.get('with_ext_qp', False)
39 self.port = kwargs.get('port') if kwargs.get('port') else '7471'
40 self.ib_port = int(kwargs.get('ib_port', '1'))
41 self.port_space = kwargs.get('port_space', rdma_port_space.RDMA_PS_TCP)
42 self.remote_operation = kwargs.get('remote_op')
43 self.qp_type = qp_type_per_ps[self.port_space]
44 self.qp_init_attr = QPInitAttr(qp_type=self.qp_type, cap=QPCap())
45 self.connected = False
46 # When passive side (server) listens to incoming connection requests,
47 # for each new request it creates a new cmid which is used to establish
48 # the connection with the remote side
49 self.msg_size = 1024
50 self.num_msgs = 10
51 self.channel = None
52 self.cq = None
53 self.qps = {}
54 self.mr = None
55 self.remote_qpn = None
56 self.ud_params = None
57 self.child_ids = {}
58 self.cmids = {}
59 self.ai = AddrInfo()
60
61 @property
62 def child_id(self):
63 if self.child_ids:
64 return self.child_ids[0]
65
66 @property
67 def cmid(self):
68 if self.cmids:
69 return self.cmids[0]
70
71 @property
72 def qp(self):
73 if self.qps:
74 return self.qps[0]
75
76 def create_mr(self):
77 cmid = self.child_id if self.passive else self.cmid
78 mr_remote_function = {None: cmid.reg_msgs, 'read': cmid.reg_read,
79 'write': cmid.reg_write}
80 self.mr = mr_remote_function[self.remote_operation](self.msg_size + GRH_SIZE)
81
82 def create_event_channel(self):
83 self.channel = CMEventChannel()
84
85 def create_qp_init_attr(self, rcq=None, scq=None):
86 return QPInitAttr(qp_type=self.qp_type, rcq=rcq, scq=scq,
87 cap=QPCap(max_recv_wr=1))
88
89 def create_conn_param(self, qp_num=0, conn_idx=0):
90 if self.with_ext_qp:
91 qp_num = self.qp.qp_num
92 return ConnParam(qp_num=qp_num)
93
94 def set_ud_params(self, cm_event):
95 if self.port_space in [rdma_port_space.RDMA_PS_UDP, rdma_port_space.RDMA_PS_IPOIB]:
96 self.ud_params = UDParam(cm_event)
97
98 def my_qp_number(self):
99 if self.with_ext_qp:
100 return self.qp.qp_num
101 else:
102 cm = self.child_id if self.passive else self.cmid
103 return cm.qpn
104
105 def create_qp(self, conn_idx=0):
106 """
107 Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and
108 QP will be created. In case that CQ is already created, it is used
109 for the newly created QP.
110 :param conn_idx: The connection index.
111 """
112 cmid = self.child_id if self.passive else self.cmid
113 if not self.with_ext_qp:
114 cmid.create_qp(self.create_qp_init_attr())
115 else:
116 self.create_cq(cmid)
117 init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq)
118 self.qps[conn_idx] = QP(cmid.pd, init_attr, QPAttr())
119
120 def create_cq(self, cmid):
121 if not self.cq:
122 self.cq = CQ(cmid.context, self.num_msgs, None, None, 0)
123
124 def modify_ext_qp_to_rts(self, conn_idx=0):
125 cmid = self.child_id if self.passive else self.cmid
126 attr, mask = cmid.init_qp_attr(ibv_qp_state.IBV_QPS_INIT)
127 self.qps[conn_idx].modify(attr, mask)
128 attr, mask = cmid.init_qp_attr(ibv_qp_state.IBV_QPS_RTR)
129 self.qps[conn_idx].modify(attr, mask)
130 attr, mask = cmid.init_qp_attr(ibv_qp_state.IBV_QPS_RTS)
131 self.qps[conn_idx].modify(attr, mask)
132
133 def mem_write(self, data, size, offset=0):
134 self.mr.write(data, size, offset)
135
136 def mem_read(self, size=None, offset=0):
137 size_ = self.msg_size if size is None else size
138 return self.mr.read(size_, offset)
139
140 @abc.abstractmethod
141 def create_child_id(self, cm_event=None):
142 pass
143
144 @property
145 def mr_lkey(self):
146 return self.mr.lkey
147
148
149 class AsyncCMResources(CMResources):
150 """
151 AsyncCMResources class contains resources for RDMA CM asynchronous
152 communication.
153 :param passive: Indicate if this CM is the passive CM.
154 """
155 def __init__(self, passive=None, **kwargs):
156 super(AsyncCMResources, self).__init__(passive=passive,
157 **kwargs)
158 self.create_event_channel()
159
160 def create_cmid(self, idx=0):
161 self.cmids[idx] = CMID(creator=self.channel, port_space=self.port_space)
162
163 def create_child_id(self, cm_event=None):
164 if not self.passive:
165 raise PyverbsUserError('create_child_id can be used only in passive side')
166 new_child_idx = len(self.child_ids)
167 self.child_ids[new_child_idx] = CMID(creator=cm_event, listen_id=self.cmid)
168
169
170 class SyncCMResources(CMResources):
171 """
172 SyncCMResources class contains resources for RDMA CM synchronous
173 communication.
174 :param passive: Indicate if this CM is the passive CM.
175 """
176 def __init__(self, passive=None, **kwargs):
177 super(SyncCMResources, self).__init__(passive=passive,
178 **kwargs)
179
180 def create_cmid(self, idx=0):
181 self.cmids[idx] = CMID(creator=self.ai, qp_init_attr=self.qp_init_attr)
182
183 def create_child_id(self, cm_event=None):
184 if not self.passive:
185 raise PyverbsUserError('create_child_id can be used only in passive side')
186 new_child_idx = len(self.child_ids)
187 self.child_ids[new_child_idx] = self.cmid.get_request()