| 1 |
/* |
| 2 |
* Copyright (c) 2013. Intel Corporation. All rights reserved. |
| 3 |
* Copyright (c) 2006-2012. QLogic Corporation. All rights reserved. |
| 4 |
* Copyright (c) 2003-2006, PathScale, Inc. All rights reserved. |
| 5 |
* |
| 6 |
* This software is available to you under a choice of one of two |
| 7 |
* licenses. You may choose to be licensed under the terms of the GNU |
| 8 |
* General Public License (GPL) Version 2, available from the file |
| 9 |
* COPYING in the main directory of this source tree, or the |
| 10 |
* OpenIB.org BSD license below: |
| 11 |
* |
| 12 |
* Redistribution and use in source and binary forms, with or |
| 13 |
* without modification, are permitted provided that the following |
| 14 |
* conditions are met: |
| 15 |
* |
| 16 |
* - Redistributions of source code must retain the above |
| 17 |
* copyright notice, this list of conditions and the following |
| 18 |
* disclaimer. |
| 19 |
* |
| 20 |
* - Redistributions in binary form must reproduce the above |
| 21 |
* copyright notice, this list of conditions and the following |
| 22 |
* disclaimer in the documentation and/or other materials |
| 23 |
* provided with the distribution. |
| 24 |
* |
| 25 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 26 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 27 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 28 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 29 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 30 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 31 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 32 |
* SOFTWARE. |
| 33 |
*/ |
| 34 |
|
| 35 |
/* |
| 36 |
* This file implements the PSM PTL for self (loopback) |
| 37 |
*/ |
| 38 |
|
| 39 |
#include "psm_user.h" |
| 40 |
#include "psm_mq_internal.h" |
| 41 |
|
| 42 |
struct ptl { |
| 43 |
psm_ep_t ep; |
| 44 |
psm_epid_t epid; |
| 45 |
psm_epaddr_t epaddr; |
| 46 |
ptl_ctl_t *ctl; |
| 47 |
}; |
| 48 |
|
| 49 |
static |
| 50 |
psm_error_t __fastpath |
| 51 |
ptl_handle_rtsmatch(psm_mq_req_t recv_req, int was_posted) |
| 52 |
{ |
| 53 |
psm_mq_req_t send_req = (psm_mq_req_t) recv_req->ptl_req_ptr; |
| 54 |
|
| 55 |
if (recv_req->recv_msglen > 0) { |
| 56 |
PSM_VALGRIND_DEFINE_MQ_RECV(recv_req->buf, recv_req->buf_len, |
| 57 |
recv_req->recv_msglen); |
| 58 |
VALGRIND_MAKE_MEM_DEFINED(send_req->buf, send_req->buf_len); |
| 59 |
VALGRIND_MAKE_MEM_DEFINED(send_req->buf, recv_req->recv_msglen); |
| 60 |
|
| 61 |
psmi_mq_mtucpy(recv_req->buf, send_req->buf, recv_req->recv_msglen); |
| 62 |
} |
| 63 |
|
| 64 |
psmi_mq_handle_rts_complete(recv_req); |
| 65 |
|
| 66 |
/* If the send is already marked complete, that's because it was internally |
| 67 |
* buffered. */ |
| 68 |
if (send_req->state == MQ_STATE_COMPLETE) { |
| 69 |
psmi_mq_stats_rts_account(send_req); |
| 70 |
if (send_req->buf != NULL && send_req->send_msglen > 0) |
| 71 |
psmi_mq_sysbuf_free(send_req->mq, send_req->buf); |
| 72 |
psmi_mq_req_free(send_req); /* req was left "live" even though the |
| 73 |
* sender was told that the send was done */ |
| 74 |
} |
| 75 |
else |
| 76 |
psmi_mq_handle_rts_complete(send_req); |
| 77 |
|
| 78 |
_IPATH_VDBG("[self][complete][b=%p][sreq=%p][rreq=%p]\n", |
| 79 |
recv_req->buf, send_req, recv_req); |
| 80 |
return PSM_OK; |
| 81 |
} |
| 82 |
|
| 83 |
static |
| 84 |
psm_error_t |
| 85 |
self_mq_send_testwait(psm_mq_req_t *ireq, int istest, psm_mq_status_t *status) |
| 86 |
{ |
| 87 |
uint8_t *ubuf; |
| 88 |
psm_mq_req_t req = *ireq; |
| 89 |
|
| 90 |
PSMI_PLOCK_ASSERT(); |
| 91 |
|
| 92 |
/* We're waiting on a send request, and the matching receive has not been |
| 93 |
* posted yet. This is a deadlock condition in MPI but we accodomate it |
| 94 |
* here in the "self ptl" by using system-allocated memory. |
| 95 |
*/ |
| 96 |
req->testwait_callback = NULL; /* no more calls here */ |
| 97 |
|
| 98 |
ubuf = req->buf; |
| 99 |
if (ubuf != NULL && req->send_msglen > 0) { |
| 100 |
req->buf = psmi_mq_sysbuf_alloc(req->mq, req->send_msglen); |
| 101 |
if (req->buf == NULL) |
| 102 |
return PSM_NO_MEMORY; |
| 103 |
psmi_mq_mtucpy(req->buf, ubuf, req->send_msglen); |
| 104 |
} |
| 105 |
|
| 106 |
/* Mark it complete but don't free the req, it's freed when the receiver |
| 107 |
* does the match */ |
| 108 |
req->state = MQ_STATE_COMPLETE; |
| 109 |
*ireq = PSM_MQ_REQINVALID; |
| 110 |
|
| 111 |
if (status != NULL) |
| 112 |
mq_status_copy(req, status); |
| 113 |
return PSM_OK; |
| 114 |
} |
| 115 |
|
| 116 |
/* Self is different. We do everything as rendezvous. */ |
| 117 |
static |
| 118 |
psm_error_t __fastpath |
| 119 |
self_mq_isend(psm_mq_t mq, psm_epaddr_t epaddr, uint32_t flags, |
| 120 |
uint64_t tag, const void *ubuf, uint32_t len, void *context, |
| 121 |
psm_mq_req_t *req_o) |
| 122 |
{ |
| 123 |
psm_mq_req_t send_req; |
| 124 |
psm_mq_req_t recv_req; |
| 125 |
int rc; |
| 126 |
|
| 127 |
send_req = psmi_mq_req_alloc(mq, MQE_TYPE_SEND); |
| 128 |
if_pf (send_req == NULL) |
| 129 |
return PSM_NO_MEMORY; |
| 130 |
|
| 131 |
rc = psmi_mq_handle_rts(mq, tag, (uintptr_t) ubuf, len, epaddr, |
| 132 |
ptl_handle_rtsmatch, &recv_req); |
| 133 |
send_req->buf = (void *) ubuf; |
| 134 |
send_req->send_msglen = len; |
| 135 |
send_req->context = context; |
| 136 |
recv_req->ptl_req_ptr = (void *) send_req; |
| 137 |
if (rc == MQ_RET_MATCH_OK) |
| 138 |
ptl_handle_rtsmatch(recv_req, 1); |
| 139 |
else |
| 140 |
send_req->testwait_callback = self_mq_send_testwait; |
| 141 |
|
| 142 |
_IPATH_VDBG("[self][b=%p][m=%d][t=%"PRIx64"][match=%s][req=%p]\n", |
| 143 |
ubuf, len, tag, rc == MQ_RET_MATCH_OK ? "YES" : "NO", send_req); |
| 144 |
*req_o = send_req; |
| 145 |
return PSM_OK; |
| 146 |
} |
| 147 |
|
| 148 |
static __fastpath |
| 149 |
psm_error_t |
| 150 |
self_mq_send(psm_mq_t mq, psm_epaddr_t epaddr, uint32_t flags, |
| 151 |
uint64_t tag, const void *ubuf, uint32_t len) |
| 152 |
{ |
| 153 |
psm_error_t err; |
| 154 |
psm_mq_req_t req; |
| 155 |
err = self_mq_isend(mq,epaddr,flags,tag,ubuf,len,NULL,&req); |
| 156 |
psmi_mq_wait_internal(&req); |
| 157 |
return err; |
| 158 |
} |
| 159 |
|
| 160 |
static |
| 161 |
psm_error_t |
| 162 |
self_connect(ptl_t *ptl, |
| 163 |
int numep, |
| 164 |
const psm_epid_t array_of_epid[], |
| 165 |
const int array_of_epid_mask[], |
| 166 |
psm_error_t array_of_errors[], |
| 167 |
psm_epaddr_t array_of_epaddr[], |
| 168 |
uint64_t timeout_ns) |
| 169 |
{ |
| 170 |
psmi_assert_always(ptl->epaddr != NULL); |
| 171 |
psm_epaddr_t epaddr; |
| 172 |
psm_error_t err = PSM_OK; |
| 173 |
int i; |
| 174 |
|
| 175 |
PSMI_PLOCK_ASSERT(); |
| 176 |
|
| 177 |
for (i = 0; i < numep; i++) { |
| 178 |
if (!array_of_epid_mask[i]) |
| 179 |
continue; |
| 180 |
|
| 181 |
if (array_of_epid[i] == ptl->epid) { |
| 182 |
epaddr = psmi_epid_lookup(ptl->ep, ptl->epid); |
| 183 |
psmi_assert_always(epaddr == NULL); |
| 184 |
array_of_epaddr[i] = ptl->epaddr; |
| 185 |
array_of_epaddr[i]->ptl = ptl; |
| 186 |
array_of_epaddr[i]->ptlctl = ptl->ctl; |
| 187 |
array_of_epaddr[i]->epid = ptl->epid; |
| 188 |
array_of_epaddr[i]->ep = ptl->ep; |
| 189 |
if (psmi_epid_set_hostname(psm_epid_nid(ptl->epid), |
| 190 |
psmi_gethostname(), 0)) { |
| 191 |
err = PSM_NO_MEMORY; |
| 192 |
goto fail; |
| 193 |
} |
| 194 |
psmi_epid_add(ptl->ep, ptl->epid, ptl->epaddr); |
| 195 |
array_of_errors[i] = PSM_OK; |
| 196 |
} |
| 197 |
else { |
| 198 |
array_of_epaddr[i] = NULL; |
| 199 |
array_of_errors[i] = PSM_EPID_UNREACHABLE; |
| 200 |
} |
| 201 |
} |
| 202 |
|
| 203 |
fail: |
| 204 |
return err; |
| 205 |
} |
| 206 |
|
| 207 |
#if 0 |
| 208 |
static |
| 209 |
psm_error_t |
| 210 |
self_disconnect(ptl_t *ptl, int numep, |
| 211 |
const psm_epaddr_t array_of_epaddr[], |
| 212 |
int array_of_epaddr_mask[], |
| 213 |
int force, uint64_t timeout_ns) |
| 214 |
{ |
| 215 |
int i; |
| 216 |
for (i = 0; i < numep; i++) { |
| 217 |
if (array_of_epaddr_mask[i] == 0) |
| 218 |
continue; |
| 219 |
|
| 220 |
if (array_of_epaddr[i] == ptl->epaddr) |
| 221 |
array_of_epaddr_mask[i] = 1; |
| 222 |
else |
| 223 |
array_of_epaddr_mask[i] = 0; |
| 224 |
} |
| 225 |
return PSM_OK; |
| 226 |
} |
| 227 |
#endif |
| 228 |
|
| 229 |
static |
| 230 |
size_t |
| 231 |
self_ptl_sizeof(void) |
| 232 |
{ |
| 233 |
return sizeof(ptl_t); |
| 234 |
} |
| 235 |
|
| 236 |
static |
| 237 |
psm_error_t |
| 238 |
self_ptl_init(const psm_ep_t ep, ptl_t *ptl, ptl_ctl_t *ctl) |
| 239 |
{ |
| 240 |
psmi_assert_always(ep != NULL); |
| 241 |
psmi_assert_always(ep->epaddr != NULL); |
| 242 |
psmi_assert_always(ep->epid != 0); |
| 243 |
|
| 244 |
ptl->ep = ep; |
| 245 |
ptl->epid = ep->epid; |
| 246 |
ptl->epaddr = ep->epaddr; |
| 247 |
ptl->ctl = ctl; |
| 248 |
ep->epaddr->mctxt_prev = ep->epaddr; |
| 249 |
ep->epaddr->mctxt_next = ep->epaddr; |
| 250 |
ep->epaddr->mctxt_master = ep->epaddr; |
| 251 |
|
| 252 |
memset(ctl, 0, sizeof(*ctl)); |
| 253 |
/* Fill in the control structure */ |
| 254 |
ctl->ptl = ptl; |
| 255 |
ctl->ep_poll = NULL; |
| 256 |
ctl->ep_connect = self_connect; |
| 257 |
ctl->ep_disconnect = NULL; |
| 258 |
|
| 259 |
ctl->mq_send = self_mq_send; |
| 260 |
ctl->mq_isend = self_mq_isend; |
| 261 |
|
| 262 |
/* No stats in self */ |
| 263 |
ctl->epaddr_stats_num = NULL; |
| 264 |
ctl->epaddr_stats_init = NULL; |
| 265 |
ctl->epaddr_stats_get = NULL; |
| 266 |
|
| 267 |
return PSM_OK; |
| 268 |
} |
| 269 |
|
| 270 |
static |
| 271 |
psm_error_t |
| 272 |
self_ptl_fini(ptl_t *ptl, int force, uint64_t timeout_ns) |
| 273 |
{ |
| 274 |
return PSM_OK; /* nothing to do */ |
| 275 |
} |
| 276 |
|
| 277 |
static |
| 278 |
psm_error_t |
| 279 |
self_ptl_setopt(const void *component_obj, int optname, |
| 280 |
const void *optval, uint64_t optlen) |
| 281 |
{ |
| 282 |
/* No options for SELF PTL at the moment */ |
| 283 |
return psmi_handle_error(NULL, PSM_PARAM_ERR, "Unknown SELF ptl option %u.", optname); |
| 284 |
} |
| 285 |
|
| 286 |
static |
| 287 |
psm_error_t |
| 288 |
self_ptl_getopt(const void *component_obj, int optname, |
| 289 |
void *optval, uint64_t *optlen) |
| 290 |
{ |
| 291 |
/* No options for SELF PTL at the moment */ |
| 292 |
return psmi_handle_error(NULL, PSM_PARAM_ERR, "Unknown SELF ptl option %u.", optname); |
| 293 |
} |
| 294 |
|
| 295 |
/* Only symbol we expose out of here */ |
| 296 |
struct ptl_ctl_init |
| 297 |
psmi_ptl_self = { |
| 298 |
self_ptl_sizeof, self_ptl_init, self_ptl_fini,self_ptl_setopt,self_ptl_getopt |
| 299 |
}; |