Debian Codegraph @

prov/verbs/src/windows/verbs_nd_ov.c

raw
1 /*
2 * Copyright (c) 2022 Canon Medical Research USA, Inc. All rights reserved.
3 * Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <assert.h>
35 #include "ndspi.h"
36 #include "verbs_nd.h"
37
38 void CALLBACK nd_io_cb(DWORD error, DWORD bytes, LPOVERLAPPED ov)
39 {
40 struct nd_event_base *base;
41
42 assert(ov);
43 base = container_of(ov, struct nd_event_base, ov);
44 ofi_mutex_lock(&base->lock);
45
46 VRB_DBG(FI_LOG_EP_CTRL, "IO callback: error: %s, bytes: %d, ov: %p\n",
47 nd_error_str(error), bytes, ov);
48
49 if (error) {
50 assert(base->error_cb);
51 base->error_cb(base, bytes, error);
52 } else {
53 assert(base->event_cb);
54 base->event_cb(base, bytes);
55 }
56
57 assert(base->cb_pending);
58 --base->cb_pending;
59 ofi_mutex_unlock(&base->lock);
60 pthread_cond_broadcast(&base->cond);
61 }
62
63 HRESULT nd_cancel_pending(struct nd_event_base *event, IND2Overlapped *ov)
64 {
65 HRESULT hr;
66
67 ofi_mutex_lock(&event->lock);
68 if (event->cb_pending) {
69 hr = ov->lpVtbl->CancelOverlappedRequests(ov);
70 if (FAILED(hr)) {
71 errno = hresult2fi(hr);
72 ofi_mutex_unlock(&event->lock);
73 return errno;
74 }
75
76 while (event->cb_pending) {
77 ofi_pthread_wait_cond(&event->cond, &event->lock,
78 INFINITE);
79 }
80 }
81 ofi_mutex_unlock(&event->lock);
82
83 return 0;
84 }
85
86 struct nd_cm_event *nd_allocate_cm_event(struct rdma_cm_id *id,
87 enum rdma_cm_event_type type)
88 {
89 struct nd_cm_event *cm_event;
90
91 // TODO: Find an alternative to memory allocation in
92 // asynchronous completion routines
93 cm_event = calloc(1, sizeof(*cm_event));
94 assert(cm_event);
95 cm_event->event.id = id;
96 cm_event->event.event = type;
97
98 return cm_event;
99 }
100
101 void nd_get_read_limits(IND2Connector *connector, struct nd_cm_event *event)
102 {
103 ULONG in = 0;
104 ULONG out = 0;
105 HRESULT hr;
106
107 hr = connector->lpVtbl->GetReadLimits(connector, &in, &out);
108 FI_LOG(&vrb_prov, FAILED(hr) ? FI_LOG_WARN : FI_LOG_DEBUG,
109 FI_LOG_EP_CTRL, "IND2Connector::GetReadLimits: hr=0x%08lx\n",
110 hr);
111
112 assert(in <= 255);
113 event->event.param.conn.responder_resources = (uint8_t)in;
114 assert(out <= 255);
115 event->event.param.conn.initiator_depth = (uint8_t)out;
116 }
117
118 void nd_get_connection_data(IND2Connector *connector, struct nd_cm_event *event)
119 {
120 ULONG len = 0;
121 HRESULT hr;
122
123 hr = connector->lpVtbl->GetPrivateData(connector, NULL, &len);
124 FI_LOG(&vrb_prov,
125 (FAILED(hr) && (hr != ND_BUFFER_OVERFLOW)) ? FI_LOG_WARN :
126 FI_LOG_DEBUG,
127 FI_LOG_EP_CTRL, "IND2Connector::GetPrivateData: hr=0x%08lx\n",
128 hr);
129
130 if ((SUCCEEDED(hr) || hr == ND_BUFFER_OVERFLOW) && len) {
131 event->event.param.conn.private_data_len = (uint8_t)len;
132 event->event.param.conn.private_data = malloc(len);
133 if (event->event.param.conn.private_data) {
134 hr = connector->lpVtbl->GetPrivateData(
135 connector,
136 (void *)event->event.param.conn.private_data,
137 &len);
138 FI_LOG(&vrb_prov,
139 FAILED(hr) ? FI_LOG_WARN : FI_LOG_DEBUG,
140 FI_LOG_EP_CTRL,
141 "IND2Connector::GetPrivateData: hr=0x%08lx\n",
142 hr);
143 if (FAILED(hr)) {
144 free(event->event.param.conn.private_data);
145 event->event.param.conn.private_data = NULL;
146 event->event.param.conn.private_data_len = 0;
147 }
148 } else {
149 event->event.param.conn.private_data_len = 0;
150 VRB_WARN(
151 FI_LOG_EP_CTRL,
152 "Failed to allocate memory for connection data.\n");
153 }
154 } else {
155 event->event.param.conn.private_data_len = 0;
156 event->event.param.conn.private_data = NULL;
157 }
158 }
159
160 static void nd_insert_cm_event(struct rdma_event_channel *channel,
161 struct nd_cm_event *event)
162 {
163 struct nd_event_channel *ch_nd;
164
165 ch_nd = container_of(channel, struct nd_event_channel, channel);
166 VRB_DBG(FI_LOG_EQ, "EQ SET event:%p %p %d\n", event->event.id,
167 event->event.listen_id, event->event.event);
168 dlistfd_insert_tail(&event->entry, &ch_nd->q);
169 }
170
171 // Used in conjunction with IND2Listener::GetConnectionRequest()
172 void nd_cm_connreq_event(struct nd_event_base *base, DWORD bytes)
173 {
174 struct nd_cm_event *cm_event;
175 struct nd_cm_listen_event *event;
176 IND2Connector *connector;
177 struct rdma_cm_id *id;
178 int ret;
179 struct nd_cm_id *id_nd;
180 struct nd_cm_id *listen_id_nd;
181 ULONG len;
182 HRESULT hr;
183
184 VRB_TRACE(FI_LOG_FABRIC, "\n");
185
186 event = container_of(base, struct nd_cm_listen_event, base);
187 connector = event->connector;
188
189 ret = rdma_create_id(event->listen_id->channel, &id,
190 event->listen_id->context, RDMA_PS_TCP);
191 assert(!ret);
192 id->verbs = event->listen_id->verbs;
193 id_nd = container_of(id, struct nd_cm_id, id);
194 listen_id_nd = container_of(event->listen_id, struct nd_cm_id, id);
195 id_nd->connector = listen_id_nd->connector;
196 id_nd->connect_event.connector = listen_id_nd->connect_event.connector;
197 id_nd->peer_event.connector = listen_id_nd->peer_event.connector;
198 id_nd->listen_event.connector = listen_id_nd->listen_event.connector;
199
200 cm_event = nd_allocate_cm_event(id, RDMA_CM_EVENT_CONNECT_REQUEST);
201 cm_event->event.listen_id = event->listen_id;
202
203 nd_get_read_limits(connector, cm_event);
204 nd_get_connection_data(connector, cm_event);
205
206 len = sizeof(id->route.addr.src_addr);
207 hr = connector->lpVtbl->GetLocalAddress(connector,
208 &id->route.addr.src_addr, &len);
209 FI_LOG(&vrb_prov, FAILED(hr) ? FI_LOG_WARN : FI_LOG_DEBUG,
210 FI_LOG_EP_CTRL, "IND2Connector::GetLocalAddress: hr=0x%08lx\n",
211 hr);
212
213 len = sizeof(id->route.addr.dst_addr);
214 hr = connector->lpVtbl->GetPeerAddress(connector,
215 &id->route.addr.dst_addr, &len);
216 FI_LOG(&vrb_prov, FAILED(hr) ? FI_LOG_WARN : FI_LOG_DEBUG,
217 FI_LOG_EP_CTRL, "IND2Connector::GetPeerAddress: hr=0x%08lx\n",
218 hr);
219
220 nd_insert_cm_event(id->channel, cm_event);
221
222 // Need to obtain a new connector for the listening id.
223 ret = rdma_bind_addr(&listen_id_nd->id,
224 &listen_id_nd->id.route.addr.src_addr);
225 assert(!ret);
226
227 ofi_mutex_lock(&listen_id_nd->listen_event.base.lock);
228 ++listen_id_nd->listen_event.base.cb_pending;
229 hr = listen_id_nd->listener->lpVtbl->GetConnectionRequest(
230 listen_id_nd->listener, (IUnknown *)listen_id_nd->connector,
231 &listen_id_nd->listen_event.base.ov);
232 FI_LOG(&vrb_prov, FAILED(hr) ? FI_LOG_WARN : FI_LOG_DEBUG,
233 FI_LOG_EP_CTRL,
234 "IND2Listener::GetConnectionRequest: hr=0x%08lx; ov=%p\n", hr,
235 &listen_id_nd->listen_event.base.ov);
236 if (FAILED(hr))
237 --listen_id_nd->listen_event.base.cb_pending;
238 ofi_mutex_unlock(&listen_id_nd->listen_event.base.lock);
239 }
240
241 void nd_cm_connreq_error(struct nd_event_base *base, DWORD bytes, DWORD error)
242 {
243 struct nd_cm_listen_event *event;
244 struct nd_cm_event *cm_event;
245
246 VRB_TRACE(FI_LOG_FABRIC, "\n");
247
248 if (error == ND_CANCELED)
249 return;
250
251 event = container_of(base, struct nd_cm_listen_event, base);
252 cm_event = nd_allocate_cm_event(event->listen_id,
253 RDMA_CM_EVENT_CONNECT_ERROR);
254 cm_event->event.listen_id = event->listen_id;
255 cm_event->event.status = error;
256
257 nd_insert_cm_event(event->listen_id->channel, cm_event);
258 }
259
260 // Used in conjunction with
261 // IND2Connector::Connect()
262 // IND2Connector::Accept()
263 // IND2Connector::CompleteConnect()
264 // IND2Connector::NotifyDisconnect()
265 // IND2Connector::Disconnect()
266 static void nd_handle_cm_connect(IND2Connector *connector,
267 struct rdma_cm_id *id, IND2QueuePair *qp)
268 {
269 HRESULT hr;
270 struct nd_cm_id *id_nd;
271 struct nd_cm_event tmp_event = { 0 };
272
273 id_nd = container_of(id, struct nd_cm_id, id);
274 id_nd->connect_event.type = ND_CM_COMPLETE;
275
276 nd_get_read_limits(connector, &tmp_event);
277 id_nd->connect_event.param.responder_resources =
278 tmp_event.event.param.conn.responder_resources;
279 id_nd->connect_event.param.initiator_depth =
280 tmp_event.event.param.conn.initiator_depth;
281
282 nd_get_connection_data(connector, &tmp_event);
283 id_nd->connect_event.param.private_data_len =
284 tmp_event.event.param.conn.private_data_len;
285 id_nd->connect_event.param.private_data =
286 tmp_event.event.param.conn.private_data;
287
288 ofi_mutex_lock(&id_nd->connect_event.base.lock);
289 ++id_nd->connect_event.base.cb_pending;
290 hr = connector->lpVtbl->CompleteConnect(connector,
291 &id_nd->connect_event.base.ov);
292 FI_LOG(&vrb_prov, FAILED(hr) ? FI_LOG_WARN : FI_LOG_DEBUG,
293 FI_LOG_EP_CTRL,
294 "IND2Connector::CompleteConnect: hr=0x%08lx; ov=%p\n", hr,
295 &id_nd->connect_event.base.ov);
296 if (FAILED(hr))
297 --id_nd->connect_event.base.cb_pending;
298 ofi_mutex_unlock(&id_nd->connect_event.base.lock);
299 }
300
301 static void nd_notify_disconnect(struct rdma_cm_id *id,
302 IND2Connector *connector)
303 {
304 HRESULT hr;
305 struct nd_cm_id *id_nd = container_of(id, struct nd_cm_id, id);
306 id_nd->peer_event.type = ND_CM_DISCONNECTED;
307
308 ofi_mutex_lock(&id_nd->peer_event.base.lock);
309 ++id_nd->peer_event.base.cb_pending;
310 hr = connector->lpVtbl->NotifyDisconnect(connector,
311 &id_nd->peer_event.base.ov);
312 FI_LOG(&vrb_prov, FAILED(hr) ? FI_LOG_WARN : FI_LOG_DEBUG,
313 FI_LOG_EP_CTRL,
314 "IND2Connector::NotifyDisconnect: hr=0x%08lx; ov=%p\n", hr,
315 &id_nd->peer_event.base.ov);
316 if (FAILED(hr))
317 --id_nd->peer_event.base.cb_pending;
318 ofi_mutex_unlock(&id_nd->peer_event.base.lock);
319 }
320
321 void nd_cm_connect_ack(struct nd_event_base *base, DWORD bytes)
322 {
323 struct nd_cm_connect_event *event;
324 IND2Connector *connector;
325 struct rdma_cm_id *id;
326 IND2QueuePair *qp;
327 struct nd_cm_event *cm_event = NULL;
328
329 VRB_TRACE(FI_LOG_FABRIC, "\n");
330
331 event = container_of(base, struct nd_cm_connect_event, base);
332 connector = event->connector;
333 id = event->id;
334 qp = event->qp;
335
336 switch (event->type) {
337 case ND_CM_CONNECT:
338 nd_handle_cm_connect(connector, id, qp);
339 return;
340
341 case ND_CM_COMPLETE:
342 cm_event = nd_allocate_cm_event(id, RDMA_CM_EVENT_ESTABLISHED);
343
344 cm_event->event.param.conn.responder_resources =
345 event->param.responder_resources;
346 cm_event->event.param.conn.initiator_depth =
347 event->param.initiator_depth;
348 cm_event->event.param.conn.private_data_len =
349 event->param.private_data_len;
350 cm_event->event.param.conn.private_data =
351 event->param.private_data;
352 nd_insert_cm_event(event->channel, cm_event);
353 nd_notify_disconnect(id, connector);
354 break;
355
356 case ND_CM_ACCEPT:
357 cm_event = nd_allocate_cm_event(id, RDMA_CM_EVENT_ESTABLISHED);
358 nd_insert_cm_event(event->channel, cm_event);
359 nd_notify_disconnect(id, connector);
360 break;
361
362 case ND_CM_DISCONNECT:
363 case ND_CM_DISCONNECTED:
364 cm_event = nd_allocate_cm_event(id, RDMA_CM_EVENT_DISCONNECTED);
365 nd_insert_cm_event(event->channel, cm_event);
366 break;
367
368 default:
369 break;
370 }
371 }
372
373 void nd_cm_connect_nack(struct nd_event_base *base, DWORD bytes, DWORD error)
374 {
375 struct nd_cm_connect_event *event;
376 struct nd_cm_event *cm_event = NULL;
377 HRESULT hr;
378
379 VRB_TRACE(FI_LOG_FABRIC, "\n");
380
381 event = container_of(base, struct nd_cm_connect_event, base);
382
383 if (error == ND_CANCELED)
384 return;
385
386 if ((event->type == ND_CM_DISCONNECTED) && (error == ND_DISCONNECTED)) {
387 nd_cm_connect_ack(base, bytes);
388 return;
389 }
390
391 switch (event->type) {
392 case ND_CM_CONNECT:
393 case ND_CM_ACCEPT:
394 cm_event = nd_allocate_cm_event(event->id,
395 RDMA_CM_EVENT_REJECTED);
396 cm_event->event.status = ECONNREFUSED;
397
398 nd_get_read_limits(event->connector, cm_event);
399 nd_get_connection_data(event->connector, cm_event);
400
401 hr = event->qp->lpVtbl->Flush(event->qp);
402 FI_LOG(&vrb_prov, FAILED(hr) ? FI_LOG_WARN : FI_LOG_DEBUG,
403 FI_LOG_EP_CTRL, "IND2QueuePair::Flush: hr=0x%08lx\n",
404 hr);
405 nd_insert_cm_event(event->channel, cm_event);
406 break;
407
408 case ND_CM_COMPLETE:
409 case ND_CM_DISCONNECT:
410 cm_event = nd_allocate_cm_event(event->id,
411 RDMA_CM_EVENT_CONNECT_ERROR);
412 cm_event->event.status = error;
413 nd_insert_cm_event(event->channel, cm_event);
414 break;
415
416 default:
417 break;
418 }
419 }
420
421 // Used in conjunction with IND2CompletionQueue::Notify
422 void nd_cq_notify_event(struct nd_event_base *base, DWORD bytes)
423 {
424 struct nd_cq *cq_nd;
425 struct nd_comp_channel *ch_nd;
426
427 VRB_TRACE(FI_LOG_FABRIC, "\n");
428
429 cq_nd = container_of(base, struct nd_cq, notification);
430 ch_nd = container_of(cq_nd->cq.channel, struct nd_comp_channel,
431 channel);
432
433 ofi_mutex_lock(&ch_nd->q_lock);
434 dlistfd_insert_tail(&cq_nd->entry, &ch_nd->q);
435 ofi_mutex_unlock(&ch_nd->q_lock);
436 }
437
438 void nd_cq_notify_error(struct nd_event_base *base, DWORD bytes, DWORD error)
439 {
440 VRB_TRACE(FI_LOG_FABRIC, "\n");
441
442 if (error == ND_CANCELED) {
443 nd_cq_notify_event(base, bytes);
444 } else {
445 VRB_WARN(FI_LOG_CQ, "Unknown error: %s, bytes %d, ov: %p\n",
446 nd_error_str(error), bytes, base);
447 }
448 }