| 1 |
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| 2 |
# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file |
| 3 |
""" |
| 4 |
Provide some useful helper function for pyverbs rdmacm' tests. |
| 5 |
""" |
| 6 |
import sys |
| 7 |
from tests.utils import validate, poll_cq, get_send_elements, get_recv_wr |
| 8 |
from tests.base_rdmacm import AsyncCMResources, SyncCMResources |
| 9 |
from pyverbs.cmid import CMEvent, AddrInfo, JoinMCAttrEx |
| 10 |
from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError |
| 11 |
from pyverbs.librdmacm_enums import rdma_port_space, rdma_cm_event_type, rdma_cm_mc_join_flags, \ |
| 12 |
rdma_cm_join_mc_attr_mask, RDMA_OPTION_ID, RDMA_OPTION_ID_ACK_TIMEOUT, RAI_PASSIVE |
| 13 |
from pyverbs.addr import AH |
| 14 |
from pyverbs.libibverbs_enums import ibv_send_flags, ibv_qp_type, ibv_qp_attr_mask |
| 15 |
import abc |
| 16 |
import errno |
| 17 |
|
| 18 |
GRH_SIZE = 40 |
| 19 |
MULTICAST_QPN = 0xffffff |
| 20 |
REJECT_MSG = 'connection rejected' |
| 21 |
|
| 22 |
|
| 23 |
class CMConnection(abc.ABC): |
| 24 |
""" |
| 25 |
RDMA CM base abstract connection class. The class contains the rdmacm |
| 26 |
resources and other methods to easily establish a connection and run |
| 27 |
traffic using the rdmacm resources. Each type of connection or traffic |
| 28 |
should inherit from this class and implement the necessary methods such as |
| 29 |
connection establishment and traffic. |
| 30 |
""" |
| 31 |
def __init__(self, syncer=None, notifier=None): |
| 32 |
""" |
| 33 |
Initializes a connection object. |
| 34 |
:param syncer: Barrier object to sync between all the test processes. |
| 35 |
:param notifier: Queue object to pass objects between the connection |
| 36 |
sides. |
| 37 |
""" |
| 38 |
self.syncer = syncer |
| 39 |
self.notifier = notifier |
| 40 |
self.cm_res = None |
| 41 |
|
| 42 |
def rdmacm_traffic(self, server=None, multicast=False): |
| 43 |
""" |
| 44 |
Run rdmacm traffic. This method runs the compatible traffic flow |
| 45 |
depending on the CMResources. If self.with_ext_qp is set the traffic |
| 46 |
will go through the external QP. |
| 47 |
:param server: Run as server. |
| 48 |
:param multicast: Run multicast traffic. |
| 49 |
""" |
| 50 |
server = server if server is not None else self.cm_res.passive |
| 51 |
if self.cm_res.with_ext_qp: |
| 52 |
if server: |
| 53 |
self._ext_qp_server_traffic() |
| 54 |
else: |
| 55 |
self._ext_qp_client_traffic() |
| 56 |
elif multicast: |
| 57 |
if server: |
| 58 |
self._cmid_server_multicast_traffic() |
| 59 |
else: |
| 60 |
self._cmid_client_multicast_traffic() |
| 61 |
else: |
| 62 |
if server: |
| 63 |
self._cmid_server_traffic() |
| 64 |
else: |
| 65 |
self._cmid_client_traffic() |
| 66 |
|
| 67 |
def remote_traffic(self, passive, remote_op='write'): |
| 68 |
""" |
| 69 |
Run rdmacm remote traffic. This method runs RDMA remote traffic from |
| 70 |
the active to the passive. |
| 71 |
:param passive: If True, run as server. |
| 72 |
:param remote_op: 'write'/'read', The type of the RDMA remote operation. |
| 73 |
""" |
| 74 |
msg_size = self.cm_res.msg_size |
| 75 |
if passive: |
| 76 |
self.cm_res.mr.write((msg_size) * 's', msg_size) |
| 77 |
mr_details = (self.cm_res.mr.rkey, self.cm_res.mr.buf) |
| 78 |
self.notifier.put(mr_details) |
| 79 |
self.syncer.wait() |
| 80 |
self.syncer.wait() |
| 81 |
if remote_op == 'write': |
| 82 |
msg_received = self.cm_res.mr.read(msg_size, 0) |
| 83 |
validate(msg_received, True, msg_size) |
| 84 |
else: |
| 85 |
self.cm_res.mr.write((msg_size) * 'c', msg_size) |
| 86 |
self.syncer.wait() |
| 87 |
rkey, remote_addr = self.notifier.get() |
| 88 |
cmid = self.cm_res.cmid |
| 89 |
post_func = cmid.post_write if remote_op == 'write' else \ |
| 90 |
cmid.post_read |
| 91 |
for _ in range(self.cm_res.num_msgs): |
| 92 |
post_func(self.cm_res.mr, msg_size, remote_addr, rkey, |
| 93 |
flags=ibv_send_flags.IBV_SEND_SIGNALED) |
| 94 |
cmid.get_send_comp() |
| 95 |
self.syncer.wait() |
| 96 |
if remote_op == 'read': |
| 97 |
msg_received = self.cm_res.mr.read(msg_size, 0) |
| 98 |
validate(msg_received, False, msg_size) |
| 99 |
|
| 100 |
def _ext_qp_server_traffic(self): |
| 101 |
""" |
| 102 |
RDMACM server side traffic function which sends and receives a message, |
| 103 |
and then validates the received message. This traffic method uses the CM |
| 104 |
external QP and CQ for send, recv and get_completion. |
| 105 |
:return: None |
| 106 |
""" |
| 107 |
recv_wr = get_recv_wr(self.cm_res) |
| 108 |
self.cm_res.qp.post_recv(recv_wr) |
| 109 |
self.syncer.wait() |
| 110 |
for _ in range(self.cm_res.num_msgs): |
| 111 |
poll_cq(self.cm_res.cq) |
| 112 |
self.cm_res.qp.post_recv(recv_wr) |
| 113 |
msg_received = self.cm_res.mr.read(self.cm_res.msg_size, 0) |
| 114 |
validate(msg_received, self.cm_res.passive, self.cm_res.msg_size) |
| 115 |
send_wr = get_send_elements(self.cm_res, self.cm_res.passive)[0] |
| 116 |
self.cm_res.qp.post_send(send_wr) |
| 117 |
poll_cq(self.cm_res.cq) |
| 118 |
|
| 119 |
def _ext_qp_client_traffic(self): |
| 120 |
""" |
| 121 |
RDMACM client side traffic function which sends and receives a message, |
| 122 |
and then validates the received message. This traffic method uses the CM |
| 123 |
external QP and CQ for send, recv and get_completion. |
| 124 |
:return: None |
| 125 |
""" |
| 126 |
recv_wr = get_recv_wr(self.cm_res) |
| 127 |
self.syncer.wait() |
| 128 |
for _ in range(self.cm_res.num_msgs): |
| 129 |
send_wr = get_send_elements(self.cm_res, self.cm_res.passive)[0] |
| 130 |
self.cm_res.qp.post_send(send_wr) |
| 131 |
poll_cq(self.cm_res.cq) |
| 132 |
self.cm_res.qp.post_recv(recv_wr) |
| 133 |
poll_cq(self.cm_res.cq) |
| 134 |
msg_received = self.cm_res.mr.read(self.cm_res.msg_size, 0) |
| 135 |
validate(msg_received, self.cm_res.passive, self.cm_res.msg_size) |
| 136 |
|
| 137 |
def _cmid_server_traffic(self): |
| 138 |
""" |
| 139 |
RDMACM server side traffic function which sends and receives a message, |
| 140 |
and then validates the received message. This traffic method uses the |
| 141 |
RDMACM API for send, recv and get_completion. |
| 142 |
:return: None |
| 143 |
""" |
| 144 |
grh_offset = GRH_SIZE if self.cm_res.qp_type == ibv_qp_type.IBV_QPT_UD else 0 |
| 145 |
send_msg = (self.cm_res.msg_size + grh_offset) * 's' |
| 146 |
cmid = self.cm_res.child_id |
| 147 |
for _ in range(self.cm_res.num_msgs): |
| 148 |
cmid.post_recv(self.cm_res.mr) |
| 149 |
self.syncer.wait() |
| 150 |
self.syncer.wait() |
| 151 |
wc = cmid.get_recv_comp() |
| 152 |
msg_received = self.cm_res.mr.read(self.cm_res.msg_size, grh_offset) |
| 153 |
validate(msg_received, True, self.cm_res.msg_size) |
| 154 |
if self.cm_res.port_space == rdma_port_space.RDMA_PS_TCP: |
| 155 |
self.cm_res.mr.write(send_msg, self.cm_res.msg_size) |
| 156 |
cmid.post_send(self.cm_res.mr) |
| 157 |
else: |
| 158 |
ah = AH(cmid.pd, wc=wc, port_num=self.cm_res.ib_port, grh=self.cm_res.mr.buf) |
| 159 |
rqpn = self.cm_res.remote_qpn |
| 160 |
self.cm_res.mr.write(send_msg, self.cm_res.msg_size + GRH_SIZE) |
| 161 |
cmid.post_ud_send(self.cm_res.mr, ah, rqpn=rqpn, |
| 162 |
length=self.cm_res.msg_size) |
| 163 |
cmid.get_send_comp() |
| 164 |
self.syncer.wait() |
| 165 |
|
| 166 |
def _cmid_client_traffic(self): |
| 167 |
""" |
| 168 |
RDMACM client side traffic function which sends and receives a message, |
| 169 |
and then validates the received message. This traffic method uses the |
| 170 |
RDMACM API for send, recv and get_completion. |
| 171 |
:return: None |
| 172 |
""" |
| 173 |
grh_offset = GRH_SIZE if self.cm_res.qp_type == ibv_qp_type.IBV_QPT_UD else 0 |
| 174 |
send_msg = (self.cm_res.msg_size + grh_offset) * 'c' |
| 175 |
cmid = self.cm_res.cmid |
| 176 |
for _ in range(self.cm_res.num_msgs): |
| 177 |
self.cm_res.mr.write(send_msg, self.cm_res.msg_size + grh_offset) |
| 178 |
self.syncer.wait() |
| 179 |
if self.cm_res.port_space == rdma_port_space.RDMA_PS_TCP: |
| 180 |
cmid.post_send(self.cm_res.mr) |
| 181 |
else: |
| 182 |
ah = AH(cmid.pd, attr=self.cm_res.ud_params.ah_attr) |
| 183 |
cmid.post_ud_send(self.cm_res.mr, ah, rqpn=self.cm_res.ud_params.qp_num, |
| 184 |
length=self.cm_res.msg_size) |
| 185 |
cmid.get_send_comp() |
| 186 |
cmid.post_recv(self.cm_res.mr) |
| 187 |
self.syncer.wait() |
| 188 |
self.syncer.wait() |
| 189 |
cmid.get_recv_comp() |
| 190 |
msg_received = self.cm_res.mr.read(self.cm_res.msg_size, grh_offset) |
| 191 |
validate(msg_received, False, self.cm_res.msg_size) |
| 192 |
|
| 193 |
def _cmid_server_multicast_traffic(self): |
| 194 |
""" |
| 195 |
RDMACM server side multicast traffic function which receives a message, |
| 196 |
and then validates its data. |
| 197 |
""" |
| 198 |
for _ in range(self.cm_res.num_msgs): |
| 199 |
self.cm_res.cmid.post_recv(self.cm_res.mr) |
| 200 |
self.syncer.wait() |
| 201 |
self.syncer.wait() |
| 202 |
self.cm_res.cmid.get_recv_comp() |
| 203 |
msg_received = self.cm_res.mr.read(self.cm_res.msg_size, GRH_SIZE) |
| 204 |
validate(msg_received, True, self.cm_res.msg_size) |
| 205 |
|
| 206 |
def _cmid_client_multicast_traffic(self): |
| 207 |
""" |
| 208 |
RDMACM client side multicast traffic function which sends a message to |
| 209 |
the multicast group. |
| 210 |
""" |
| 211 |
send_msg = (self.cm_res.msg_size + GRH_SIZE) * 'c' |
| 212 |
for _ in range(self.cm_res.num_msgs): |
| 213 |
self.cm_res.mr.write(send_msg, self.cm_res.msg_size + GRH_SIZE) |
| 214 |
self.syncer.wait() |
| 215 |
ah = AH(self.cm_res.cmid.pd, attr=self.cm_res.ud_params.ah_attr) |
| 216 |
self.cm_res.cmid.post_ud_send(self.cm_res.mr, ah, rqpn=MULTICAST_QPN, |
| 217 |
length=self.cm_res.msg_size) |
| 218 |
self.cm_res.cmid.get_send_comp() |
| 219 |
self.syncer.wait() |
| 220 |
|
| 221 |
def event_handler(self, expected_event=None): |
| 222 |
""" |
| 223 |
Handle and execute corresponding API for RDMACM events of asynchronous |
| 224 |
communication. |
| 225 |
:param expected_event: The user expected event. |
| 226 |
:return: None |
| 227 |
""" |
| 228 |
cm_event = CMEvent(self.cm_res.cmid.event_channel) |
| 229 |
if cm_event.event_type == rdma_cm_event_type.RDMA_CM_EVENT_CONNECT_REQUEST: |
| 230 |
self.cm_res.create_child_id(cm_event) |
| 231 |
elif cm_event.event_type in [rdma_cm_event_type.RDMA_CM_EVENT_ESTABLISHED, |
| 232 |
rdma_cm_event_type.RDMA_CM_EVENT_MULTICAST_JOIN]: |
| 233 |
self.cm_res.set_ud_params(cm_event) |
| 234 |
if expected_event and expected_event != cm_event.event_type: |
| 235 |
raise PyverbsError('Expected this event: {}, got this event: {}'. |
| 236 |
format(expected_event, cm_event.event_str())) |
| 237 |
if expected_event == rdma_cm_event_type.RDMA_CM_EVENT_REJECTED: |
| 238 |
assert cm_event.private_data[:len(REJECT_MSG)].decode() == REJECT_MSG, \ |
| 239 |
f'CM event data ({cm_event.private_data}) is different than the expected ({REJECT_MSG})' |
| 240 |
cm_event.ack_cm_event() |
| 241 |
|
| 242 |
@abc.abstractmethod |
| 243 |
def establish_connection(self): |
| 244 |
pass |
| 245 |
|
| 246 |
@abc.abstractmethod |
| 247 |
def disconnect(self): |
| 248 |
pass |
| 249 |
|
| 250 |
def get_and_update_addrinfo(self, passive, ip_addr): |
| 251 |
""" |
| 252 |
Get and set address information for RDMA connection. |
| 253 |
:param passive: If True, get address info for passive side |
| 254 |
:param ip_addr: IP address to use for resolution |
| 255 |
""" |
| 256 |
if passive: |
| 257 |
self.cm_res.ai.get_addrinfo(src=ip_addr, src_service=self.cm_res.port, |
| 258 |
flags=RAI_PASSIVE, |
| 259 |
port_space=self.cm_res.port_space) |
| 260 |
else: |
| 261 |
self.cm_res.ai.get_addrinfo(src=ip_addr, dst=ip_addr, dst_service=self.cm_res.port, |
| 262 |
port_space=self.cm_res.port_space) |
| 263 |
|
| 264 |
|
| 265 |
class CMAsyncConnection(CMConnection): |
| 266 |
""" |
| 267 |
Implement RDMACM connection management for asynchronous CMIDs. It includes |
| 268 |
connection establishment, disconnection and other methods such as traffic. |
| 269 |
""" |
| 270 |
def __init__(self, ip_addr, syncer=None, notifier=None, passive=False, |
| 271 |
num_conns=1, qp_timeout=-1, reject_conn=False, **kwargs): |
| 272 |
""" |
| 273 |
Init the CMConnection and then init the AsyncCMResources. |
| 274 |
:param ip_addr: IP address to use. |
| 275 |
:param syncer: Barrier object to sync between all the test processes. |
| 276 |
:param notifier: Queue object to pass objects between the connection |
| 277 |
sides. |
| 278 |
:param passive: Indicate if it's a passive side. |
| 279 |
:param num_conns: Number of connections. |
| 280 |
:param qp_timeout: Value of the QP timeout. |
| 281 |
:param reject_conn: True if the server will reject the connection. |
| 282 |
:param kwargs: Arguments used to initialize the CM resources. For more |
| 283 |
info please check CMResources. |
| 284 |
""" |
| 285 |
super(CMAsyncConnection, self).__init__(syncer=syncer, notifier=notifier) |
| 286 |
self.num_conns = num_conns |
| 287 |
self.create_cm_res(ip_addr, passive=passive, **kwargs) |
| 288 |
self.qp_timeout = qp_timeout |
| 289 |
self.reject_conn = reject_conn |
| 290 |
|
| 291 |
def create_cm_res(self, ip_addr, passive, **kwargs): |
| 292 |
self.cm_res = AsyncCMResources(passive=passive, **kwargs) |
| 293 |
if passive: |
| 294 |
self.cm_res.create_cmid() |
| 295 |
else: |
| 296 |
for i in range(self.num_conns): |
| 297 |
self.cm_res.create_cmid(i) |
| 298 |
|
| 299 |
if kwargs.get('use_resolve_addrinfo'): |
| 300 |
self.resolve_and_update_addrinfo(passive, ip_addr) |
| 301 |
else: |
| 302 |
self.get_and_update_addrinfo(passive, ip_addr) |
| 303 |
|
| 304 |
def join_to_multicast(self, mc_addr=None, src_addr=None, extended=False): |
| 305 |
""" |
| 306 |
Join the CMID to multicast group. |
| 307 |
:param mc_addr: The multicast IP address. |
| 308 |
:param src_addr: The CMIDs source address. |
| 309 |
:param extended: Use the join_multicast_ex API. |
| 310 |
""" |
| 311 |
self.cm_res.cmid.bind_addr(self.cm_res.ai) |
| 312 |
resolve_addr_info = AddrInfo(src=src_addr, dst=mc_addr) |
| 313 |
self.cm_res.cmid.resolve_addr(resolve_addr_info) |
| 314 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_ADDR_RESOLVED) |
| 315 |
self.cm_res.create_qp() |
| 316 |
mc_addr_info = AddrInfo(src=mc_addr) |
| 317 |
if not extended: |
| 318 |
self.cm_res.cmid.join_multicast(addr=mc_addr_info) |
| 319 |
else: |
| 320 |
flags = rdma_cm_mc_join_flags.RDMA_MC_JOIN_FLAG_FULLMEMBER |
| 321 |
comp_mask = rdma_cm_join_mc_attr_mask.RDMA_CM_JOIN_MC_ATTR_ADDRESS | \ |
| 322 |
rdma_cm_join_mc_attr_mask.RDMA_CM_JOIN_MC_ATTR_JOIN_FLAGS |
| 323 |
mcattr = JoinMCAttrEx(addr=mc_addr_info, comp_mask=comp_mask, |
| 324 |
join_flags=flags) |
| 325 |
self.cm_res.cmid.join_multicast(mc_attr=mcattr) |
| 326 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_MULTICAST_JOIN) |
| 327 |
self.cm_res.create_mr() |
| 328 |
|
| 329 |
def leave_multicast(self, mc_addr=None): |
| 330 |
""" |
| 331 |
Leave multicast group. |
| 332 |
:param mc_addr: The multicast IP address. |
| 333 |
""" |
| 334 |
mc_addr_info = AddrInfo(src=mc_addr) |
| 335 |
self.cm_res.cmid.leave_multicast(mc_addr_info) |
| 336 |
|
| 337 |
def establish_connection(self): |
| 338 |
""" |
| 339 |
Establish RDMACM connection between two Async CMIDs. |
| 340 |
""" |
| 341 |
if self.cm_res.passive: |
| 342 |
self.cm_res.cmid.bind_addr(self.cm_res.ai) |
| 343 |
self.cm_res.cmid.listen() |
| 344 |
for conn_idx in range(self.num_conns): |
| 345 |
if self.cm_res.passive: |
| 346 |
self.syncer.wait() |
| 347 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_CONNECT_REQUEST) |
| 348 |
self.cm_res.create_qp(conn_idx=conn_idx) |
| 349 |
if self.qp_timeout >= 0: |
| 350 |
self.set_qp_timeout(self.cm_res.child_ids[conn_idx], self.qp_timeout) |
| 351 |
if self.cm_res.with_ext_qp: |
| 352 |
self.set_cmids_qp_ece(self.cm_res.passive) |
| 353 |
self.cm_res.modify_ext_qp_to_rts(conn_idx=conn_idx) |
| 354 |
self.set_cmid_ece(self.cm_res.passive) |
| 355 |
child_id = self.cm_res.child_ids[conn_idx] |
| 356 |
if self.reject_conn: |
| 357 |
child_id.reject(REJECT_MSG.encode()) |
| 358 |
return |
| 359 |
child_id.accept(self.cm_res.create_conn_param(conn_idx=conn_idx)) |
| 360 |
if self.qp_timeout >= 0: |
| 361 |
attr, _ = child_id.query_qp(ibv_qp_attr_mask.IBV_QP_TIMEOUT) |
| 362 |
assert self.qp_timeout == attr.timeout |
| 363 |
if self.cm_res.port_space == rdma_port_space.RDMA_PS_TCP: |
| 364 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_ESTABLISHED) |
| 365 |
else: |
| 366 |
cmid = self.cm_res.cmids[conn_idx] |
| 367 |
cmid.resolve_addr(self.cm_res.ai) |
| 368 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_ADDR_RESOLVED) |
| 369 |
self.syncer.wait() |
| 370 |
cmid.resolve_route() |
| 371 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_ROUTE_RESOLVED) |
| 372 |
self.cm_res.create_qp(conn_idx=conn_idx) |
| 373 |
if self.qp_timeout >= 0: |
| 374 |
self.set_qp_timeout(self.cm_res.cmid, self.qp_timeout) |
| 375 |
if self.cm_res.with_ext_qp: |
| 376 |
self.set_cmid_ece(self.cm_res.passive) |
| 377 |
cmid.connect(self.cm_res.create_conn_param(conn_idx=conn_idx)) |
| 378 |
if self.cm_res.with_ext_qp: |
| 379 |
self.event_handler(expected_event=\ |
| 380 |
rdma_cm_event_type.RDMA_CM_EVENT_CONNECT_RESPONSE) |
| 381 |
self.set_cmids_qp_ece(self.cm_res.passive) |
| 382 |
self.cm_res.modify_ext_qp_to_rts(conn_idx=conn_idx) |
| 383 |
cmid.establish() |
| 384 |
else: |
| 385 |
if self.reject_conn: |
| 386 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_REJECTED) |
| 387 |
return |
| 388 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_ESTABLISHED) |
| 389 |
if self.qp_timeout >= 0: |
| 390 |
attr, _ = self.cm_res.cmid.query_qp(ibv_qp_attr_mask.IBV_QP_TIMEOUT) |
| 391 |
assert self.qp_timeout == attr.timeout |
| 392 |
self.cm_res.create_mr() |
| 393 |
self.sync_qp_numbers() |
| 394 |
|
| 395 |
def set_qp_timeout(self, cm_id, ack_timeout): |
| 396 |
cm_id.set_option(RDMA_OPTION_ID, RDMA_OPTION_ID_ACK_TIMEOUT, ack_timeout, 1) |
| 397 |
|
| 398 |
def sync_qp_numbers(self): |
| 399 |
""" |
| 400 |
Sync the QP numbers of the connections sides. |
| 401 |
""" |
| 402 |
if self.cm_res.passive: |
| 403 |
self.syncer.wait() |
| 404 |
self.notifier.put(self.cm_res.my_qp_number()) |
| 405 |
self.syncer.wait() |
| 406 |
self.cm_res.remote_qpn = self.notifier.get() |
| 407 |
else: |
| 408 |
self.syncer.wait() |
| 409 |
self.cm_res.remote_qpn = self.notifier.get() |
| 410 |
self.notifier.put(self.cm_res.my_qp_number()) |
| 411 |
self.syncer.wait() |
| 412 |
|
| 413 |
def disconnect(self): |
| 414 |
""" |
| 415 |
Disconnect the connection. |
| 416 |
""" |
| 417 |
if self.cm_res.port_space == rdma_port_space.RDMA_PS_TCP: |
| 418 |
if self.cm_res.passive: |
| 419 |
for child_id in self.cm_res.child_ids.values(): |
| 420 |
child_id.disconnect() |
| 421 |
else: |
| 422 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_DISCONNECTED) |
| 423 |
for cmid in self.cm_res.cmids.values(): |
| 424 |
cmid.disconnect() |
| 425 |
|
| 426 |
def set_cmid_ece(self, passive): |
| 427 |
""" |
| 428 |
Set the local CMIDs ECE. The ECE is taken from the CMIDs QP ECE. |
| 429 |
:param passive: Indicates if this CMID is participate as passive in |
| 430 |
this connection. |
| 431 |
""" |
| 432 |
cmid = self.cm_res.child_id if passive else self.cm_res.cmid |
| 433 |
try: |
| 434 |
ece = self.cm_res.qp.query_ece() |
| 435 |
cmid.set_local_ece(ece) |
| 436 |
except PyverbsRDMAError as ex: |
| 437 |
if ex.error_code != errno.EOPNOTSUPP: |
| 438 |
raise ex |
| 439 |
|
| 440 |
def set_cmids_qp_ece(self, passive): |
| 441 |
""" |
| 442 |
Set the CMIDs QP ECE. |
| 443 |
:param passive: Indicates if this CMID is participate as passive in |
| 444 |
this connection. |
| 445 |
""" |
| 446 |
cmid = self.cm_res.child_id if passive else self.cm_res.cmid |
| 447 |
try: |
| 448 |
ece = cmid.get_remote_ece() |
| 449 |
self.cm_res.qp.set_ece(ece) |
| 450 |
except PyverbsRDMAError as ex: |
| 451 |
if ex.error_code != errno.EOPNOTSUPP: |
| 452 |
raise ex |
| 453 |
|
| 454 |
def resolve_and_update_addrinfo(self, passive, ip_addr): |
| 455 |
""" |
| 456 |
Resolve and query and update the address information for the CMID. |
| 457 |
:param passive: If True, resolve as passive side |
| 458 |
:param ip_addr: IP address to resolve |
| 459 |
""" |
| 460 |
if passive: |
| 461 |
self.cm_res.cmid.resolve_addrinfo( |
| 462 |
node=ip_addr, service=self.cm_res.port, |
| 463 |
port_space=self.cm_res.port_space, |
| 464 |
flags=RAI_PASSIVE) |
| 465 |
else: |
| 466 |
self.cm_res.cmid.resolve_addrinfo( |
| 467 |
node=ip_addr, service=self.cm_res.port, |
| 468 |
port_space=self.cm_res.port_space) |
| 469 |
self.event_handler(expected_event=rdma_cm_event_type.RDMA_CM_EVENT_ADDRINFO_RESOLVED) |
| 470 |
self.cm_res.ai = self.cm_res.cmid.query_addrinfo() |
| 471 |
|
| 472 |
|
| 473 |
class CMSyncConnection(CMConnection): |
| 474 |
""" |
| 475 |
Implement RDMACM connection management for synchronous CMIDs. It includes |
| 476 |
connection establishment, disconnection and other methods such as traffic. |
| 477 |
""" |
| 478 |
def __init__(self, ip_addr, syncer=None, notifier=None, passive=False, **kwargs): |
| 479 |
""" |
| 480 |
Init the CMConnection and then init the SyncCMResources. |
| 481 |
:param ip_addr: IP address to use. |
| 482 |
:param syncer: Barrier object to sync between all the test processes. |
| 483 |
:param notifier: Queue object to pass objects between the connection |
| 484 |
sides. |
| 485 |
:param passive: Indicate if it's a passive side. |
| 486 |
:param kwargs: Arguments used to initialize the CM resources. For more |
| 487 |
info please check CMResources. |
| 488 |
""" |
| 489 |
super(CMSyncConnection, self).__init__(syncer=syncer, notifier=notifier) |
| 490 |
self.create_cm_res(ip_addr, passive=passive, **kwargs) |
| 491 |
|
| 492 |
def create_cm_res(self, ip_addr, passive, **kwargs): |
| 493 |
self.cm_res = SyncCMResources(passive=passive, **kwargs) |
| 494 |
if self.cm_res.passive: |
| 495 |
self.cm_res.ai = AddrInfo(src=ip_addr, src_service=self.cm_res.port, |
| 496 |
port_space=self.cm_res.port_space, flags=RAI_PASSIVE) |
| 497 |
else: |
| 498 |
self.cm_res.ai = AddrInfo(src=ip_addr, dst=ip_addr, dst_service=self.cm_res.port, |
| 499 |
port_space=self.cm_res.port_space) |
| 500 |
self.cm_res.create_cmid() |
| 501 |
|
| 502 |
def establish_connection(self): |
| 503 |
""" |
| 504 |
Establish RDMACM connection between two Sync CMIDs. |
| 505 |
""" |
| 506 |
if self.cm_res.passive: |
| 507 |
self.cm_res.cmid.listen() |
| 508 |
self.syncer.wait() |
| 509 |
self.cm_res.create_child_id() |
| 510 |
self.cm_res.child_id.accept() |
| 511 |
self.cm_res.create_mr() |
| 512 |
else: |
| 513 |
self.syncer.wait() |
| 514 |
self.cm_res.cmid.connect() |
| 515 |
self.cm_res.create_mr() |
| 516 |
|
| 517 |
def disconnect(self): |
| 518 |
""" |
| 519 |
Disconnect the connection. |
| 520 |
""" |
| 521 |
if self.cm_res.port_space == rdma_port_space.RDMA_PS_TCP: |
| 522 |
if self.cm_res.passive: |
| 523 |
self.cm_res.child_id.disconnect() |
| 524 |
else: |
| 525 |
self.cm_res.cmid.disconnect() |