| 1 |
/* |
| 2 |
* Copyright (c) 2013. Intel Corporation. All rights reserved. |
| 3 |
* Copyright (c) 2006-2012. QLogic Corporation. All rights reserved. |
| 4 |
* Copyright (c) 2003-2006, PathScale, Inc. All rights reserved. |
| 5 |
* |
| 6 |
* This software is available to you under a choice of one of two |
| 7 |
* licenses. You may choose to be licensed under the terms of the GNU |
| 8 |
* General Public License (GPL) Version 2, available from the file |
| 9 |
* COPYING in the main directory of this source tree, or the |
| 10 |
* OpenIB.org BSD license below: |
| 11 |
* |
| 12 |
* Redistribution and use in source and binary forms, with or |
| 13 |
* without modification, are permitted provided that the following |
| 14 |
* conditions are met: |
| 15 |
* |
| 16 |
* - Redistributions of source code must retain the above |
| 17 |
* copyright notice, this list of conditions and the following |
| 18 |
* disclaimer. |
| 19 |
* |
| 20 |
* - Redistributions in binary form must reproduce the above |
| 21 |
* copyright notice, this list of conditions and the following |
| 22 |
* disclaimer in the documentation and/or other materials |
| 23 |
* provided with the distribution. |
| 24 |
* |
| 25 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 26 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 27 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 28 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 29 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 30 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 31 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 32 |
* SOFTWARE. |
| 33 |
*/ |
| 34 |
|
| 35 |
#include "psm_user.h" |
| 36 |
#include "ipserror.h" |
| 37 |
#include "ips_proto.h" |
| 38 |
#include "ips_proto_internal.h" |
| 39 |
|
| 40 |
#define COMMIDX_MAX 65535 /* Last valid communication idx is 65535 */ |
| 41 |
|
| 42 |
/* Connections are not pairwise but we keep a single 'epaddr' for messages from |
| 43 |
* and messages to a remote 'epaddr'. State transitions for connecting TO and |
| 44 |
* FROM 'epaddrs' are the following: |
| 45 |
* Connect TO: |
| 46 |
* NONE -> WAITING -> ESTABLISHED -> WAITING_DISC -> DISCONNECTED -> NONE |
| 47 |
* |
| 48 |
* Connect FROM (we receive a connect request) |
| 49 |
* NONE -> ESTABLISHED -> NONE |
| 50 |
*/ |
| 51 |
#define CSTATE_ESTABLISHED 1 |
| 52 |
#define CSTATE_NONE 2 |
| 53 |
#define CSTATE_TO_DISCONNECTED 3 |
| 54 |
#define CSTATE_TO_WAITING 4 |
| 55 |
#define CSTATE_TO_WAITING_DISC 5 |
| 56 |
|
| 57 |
#define IPS_CONNECT_VERNO 0x0201 /* major,major,minor,minor */ |
| 58 |
#define BIG_ENDIAN_TEST_WORD 0xA5A5 |
| 59 |
|
| 60 |
/* We can use up to 16-bits of features, we only use 5 of them for now. */ |
| 61 |
#define EP_FEATURES_ENDIAN_BIG 0x0001 |
| 62 |
#define EP_FEATURES_ENDIAN_LITTLE 0x0002 |
| 63 |
#define EP_FEATURES_BITWIDTH_32 0x0004 |
| 64 |
#define EP_FEATURES_BITWIDTH_64 0x0008 |
| 65 |
#define EP_FEATURES_RCVTHREAD 0x8000 |
| 66 |
#define EP_FEATURES_MULTIFLOW 0x4000 |
| 67 |
|
| 68 |
#define EP_FEATURES_NODETYPE 0x0f |
| 69 |
|
| 70 |
struct connect_msghdr { |
| 71 |
uint8_t opcode; |
| 72 |
uint8_t _unused1; |
| 73 |
|
| 74 |
uint16_t connect_verno; /* be */ |
| 75 |
uint16_t psm_verno; /* be */ |
| 76 |
uint16_t phase; /* be connect/disconnect phase (unused now) */ |
| 77 |
|
| 78 |
uint16_t hca_type; /* HCA type of remote endpoint */ |
| 79 |
uint16_t sl; /* Default SL request for remote endpoint*/ |
| 80 |
uint32_t _unused[1]; |
| 81 |
|
| 82 |
psm_uuid_t uuid; |
| 83 |
}; |
| 84 |
#define IPS_CONNECT_MSGHDR_SIZE 32 /* 16 + 16-byte-uuid */ |
| 85 |
|
| 86 |
struct ips_connect_reqrep { |
| 87 |
struct connect_msghdr hdr; |
| 88 |
uint32_t flags; /* unused */ |
| 89 |
uint16_t connect_result; /* be */ |
| 90 |
|
| 91 |
/* Per-job info */ |
| 92 |
uint32_t commidx; /* ignore if 0xffffffff */ |
| 93 |
uint32_t runid_key; /* one-time stamp connect key */ |
| 94 |
uint16_t job_pkey; /* (future use) */ |
| 95 |
uint64_t _unused1[4]; |
| 96 |
|
| 97 |
/* Per-node characteristics */ |
| 98 |
uint32_t features; /* be - endpoint desc (endian + bidwidth) */ |
| 99 |
uint16_t hdrq_msg_size; /* where is the header/eager cutoff */ |
| 100 |
uint16_t mtu; /* receive payload */ |
| 101 |
char hostname[128]; /* always NULL-terminated */ |
| 102 |
uint64_t _unused2[4]; |
| 103 |
|
| 104 |
uint8_t version_1_offset[0]; |
| 105 |
}; |
| 106 |
|
| 107 |
/* Used for sanity checking in processing message arrivals */ |
| 108 |
#define IPS_CONNECT_REQREP_MINIMUM_SIZE \ |
| 109 |
(offsetof(struct ips_connect_reqrep, version_1_offset)) |
| 110 |
#define IPS_MAX_CONNECT_PAYLEN 512 |
| 111 |
|
| 112 |
struct ips_disconnect_reqrep { |
| 113 |
struct connect_msghdr hdr; |
| 114 |
uint32_t flags; /* unused */ |
| 115 |
|
| 116 |
uint16_t mode; |
| 117 |
uint16_t _unused1[3]; |
| 118 |
uint64_t _unused2[4]; |
| 119 |
uint8_t version_1_offset[0]; |
| 120 |
}; |
| 121 |
/* Used for sanity checking in processing message arrivals */ |
| 122 |
#define IPS_DISCONNECT_REQREP_MINIMUM_SIZE \ |
| 123 |
(offsetof(struct ips_disconnect_reqrep, version_1_offset)) |
| 124 |
|
| 125 |
const struct ips_transfer_fn psmi_xfer_fn[PSM_TRANSFER_LAST] = |
| 126 |
{ |
| 127 |
PIO_TRANSFER_FUNCTIONS, |
| 128 |
DMA_TRANSFER_FUNCTIONS |
| 129 |
}; |
| 130 |
|
| 131 |
const struct ips_protocol_fn psmi_protocol_fn[PSM_PROTOCOL_LAST] = |
| 132 |
{ |
| 133 |
GO_BACK_N_PROTOCOL_FUNCTIONS, |
| 134 |
TIDFLOW_PROTOCOL_FUNCTIONS |
| 135 |
}; |
| 136 |
|
| 137 |
/* Startup protocol in PSM/IPS |
| 138 |
* |
| 139 |
* Start timer. |
| 140 |
* |
| 141 |
* For all nodes to connect to: |
| 142 |
* Grab connect lock |
| 143 |
* Look up epid in table |
| 144 |
* MATCH. |
| 145 |
* assert cstate_to != CONNECT_WAITING (no re-entrancy) |
| 146 |
* If cstate_to == CONNECT_DONE |
| 147 |
* return the already connected address. |
| 148 |
* else |
| 149 |
* assert cstate_to == CONNECT_NONE |
| 150 |
* assert cstate_from == CONNECT_DONE |
| 151 |
* cstate_to := CONNECT_WAITING |
| 152 |
* assert commidx_to != UNKNOWN && commidx_from != UNKNOWN |
| 153 |
* req->commidx := epaddr->commidx_from |
| 154 |
* add to list of pending connect. |
| 155 |
* NO MATCH |
| 156 |
* allocate epaddr and put in table |
| 157 |
* cstate_to := CONNECT_WAITING |
| 158 |
* cstate_from := CONNECT_NONE |
| 159 |
* commidx_to := UNKNOWN |
| 160 |
* req->commidx := epaddr->commidx_from := NEW commidx integer |
| 161 |
* add to list of pending connect |
| 162 |
* Release connect lock |
| 163 |
* |
| 164 |
* expected_connect_count = ep->total_connect_count + num_to_connect |
| 165 |
* while (expected_connect_count != ep->total_connect_count) |
| 166 |
* check for timeout |
| 167 |
* progress(); |
| 168 |
* |
| 169 |
* For all connection requests received (within progress loop) |
| 170 |
* If uuid doesn't match, NAK the connect and skip request |
| 171 |
* Grab connect lock |
| 172 |
* Lock up epid in table |
| 173 |
* MATCH |
| 174 |
* if cstate_from == CONNECT_DONE |
| 175 |
* req->commidx := epaddr->commidx_from |
| 176 |
* compose reply and send again (this is a dupe request). |
| 177 |
* else |
| 178 |
* assert cstate_from == CONNECT_NONE |
| 179 |
* assert cstate_to == (CONNECT_WAITING | CONNECT_DONE) |
| 180 |
* cstate_from := CONNECT_DONE |
| 181 |
* epaddr->commidx_to := req->commidx |
| 182 |
* req->commidx := epaddr->commidx_from |
| 183 |
* NO MATCH |
| 184 |
* allocate epaddr and put in table |
| 185 |
* cstate_from := CONNECT_DONE |
| 186 |
* epaddr->commidx_to = req->commidx; |
| 187 |
* rep->commidx := epaddr->commidx_from := NEW commidx integer |
| 188 |
* compose connect reply and send |
| 189 |
* Release connect lock |
| 190 |
* |
| 191 |
* For all connection replies received: |
| 192 |
* If connect_result != 0, process error and skip. |
| 193 |
* assert cstate_to == CONNECT_WAITING |
| 194 |
* if cstate_from == CONNECT_DONE |
| 195 |
* assert rep->commidx == epaddr->commidx_to |
| 196 |
* else |
| 197 |
* epaddr->commidx_to := rep->commidx |
| 198 |
* cstate_to := CONNECT_DONE |
| 199 |
* ep->total_connect_count ++ |
| 200 |
* |
| 201 |
* * Fill in a connection request: |
| 202 |
* 1. Set connect protocol version and PSM versions |
| 203 |
* 2. Set the uuid attached to current endpoint and add the job_pkey |
| 204 |
* the node wishes to communicate post-connect. |
| 205 |
* 3. Set our mtu, bitwidth and endianess to detect inconsistencies |
| 206 |
* |
| 207 |
*/ |
| 208 |
|
| 209 |
/* Due to an oversight in the inital protocol, only 16 of the 32 bits can |
| 210 |
* actually be used because the little-to-big endian conversion was done with |
| 211 |
* 16 bits from the first version in 2.0. */ |
| 212 |
static |
| 213 |
uint32_t |
| 214 |
psmi_ips_node_features(psm_ep_t ep) |
| 215 |
{ |
| 216 |
uint32_t features = 0; |
| 217 |
if (BIG_ENDIAN_TEST_WORD == __cpu_to_be16(BIG_ENDIAN_TEST_WORD)) |
| 218 |
features |= EP_FEATURES_ENDIAN_BIG; |
| 219 |
else |
| 220 |
features |= EP_FEATURES_ENDIAN_LITTLE; |
| 221 |
if (sizeof(uintptr_t) == 8) |
| 222 |
features |= EP_FEATURES_BITWIDTH_64; |
| 223 |
else |
| 224 |
features |= EP_FEATURES_BITWIDTH_32; |
| 225 |
if (ep->context.runtime_flags & PSMI_RUNTIME_RCVTHREAD) |
| 226 |
features |= EP_FEATURES_RCVTHREAD; |
| 227 |
features |= EP_FEATURES_MULTIFLOW; |
| 228 |
|
| 229 |
return features; |
| 230 |
} |
| 231 |
|
| 232 |
static |
| 233 |
int |
| 234 |
node_matches_bitendian(psm_ep_t ep, uint32_t features) |
| 235 |
{ |
| 236 |
if ((features & EP_FEATURES_NODETYPE) == |
| 237 |
(psmi_ips_node_features(ep) & EP_FEATURES_NODETYPE)) |
| 238 |
return 1; |
| 239 |
else |
| 240 |
return 0; |
| 241 |
} |
| 242 |
|
| 243 |
/* |
| 244 |
* Given a connection request, set mtu, communication index and hdr length |
| 245 |
* parameters. |
| 246 |
* |
| 247 |
* The most subtle parameter is the mtu. When set as 'req->mtu', the mtu |
| 248 |
* is our connecting peer's declared mtu (which may not be the same as our |
| 249 |
* mtu). The approach is to take the smaller of both mtus when communicating |
| 250 |
* with that peer. Also, when using pio, the size can be further restricted by |
| 251 |
* the pio send buffer sizes (i.e. 4K IB MTU but only 2K PIO buffers). |
| 252 |
*/ |
| 253 |
static |
| 254 |
psm_error_t |
| 255 |
ips_ipsaddr_set_req_params(struct ips_proto *proto, |
| 256 |
ips_epaddr_t *ipsaddr, |
| 257 |
const struct ips_connect_reqrep *req, |
| 258 |
uint32_t paylen) |
| 259 |
{ |
| 260 |
psmi_assert_always(req->mtu > 0); |
| 261 |
|
| 262 |
uint32_t peer_mtu = min(req->mtu, proto->epinfo.ep_mtu); |
| 263 |
|
| 264 |
ipsaddr->epr.epr_piosize = min(peer_mtu, proto->epinfo.ep_piosize); |
| 265 |
ipsaddr->epr.epr_hca_type= req->hdr.hca_type; |
| 266 |
|
| 267 |
if (ipsaddr->epr.epr_piosize > PSM_CACHE_LINE_BYTES) |
| 268 |
ipsaddr->epr.epr_piosize &= ~(PSM_CACHE_LINE_BYTES - 1); |
| 269 |
|
| 270 |
/* |
| 271 |
* DMA is bounded by the peer's mtu put also our local PIO send size |
| 272 |
*/ |
| 273 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO].frag_size = ipsaddr->epr.epr_piosize; |
| 274 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_DMA].frag_size = peer_mtu; |
| 275 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_AM_REQ].frag_size=ipsaddr->epr.epr_piosize; |
| 276 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_AM_RSP].frag_size=ipsaddr->epr.epr_piosize; |
| 277 |
|
| 278 |
ipsaddr->epr.epr_commidx_to = req->commidx; |
| 279 |
|
| 280 |
/* |
| 281 |
* For static routes i.e. "none" path resolution update all paths to |
| 282 |
* have the same profile (mtu, sl etc.). |
| 283 |
* |
| 284 |
* For path record queries the epr_mtu and epr_sl are setup correctly |
| 285 |
* from the path itself. |
| 286 |
*/ |
| 287 |
if (proto->ep->path_res_type == PSM_PATH_RES_NONE) { |
| 288 |
int ptype, pidx; |
| 289 |
for (ptype = IPS_PATH_LOW_PRIORITY; ptype < IPS_PATH_MAX_PRIORITY;ptype++) |
| 290 |
for (pidx = 0; pidx < ipsaddr->epr.epr_num_paths[ptype]; pidx++) { |
| 291 |
ipsaddr->epr.epr_path[ptype][pidx]->epr_mtu = peer_mtu; |
| 292 |
ipsaddr->epr.epr_path[ptype][pidx]->epr_sl = req->hdr.sl; |
| 293 |
} |
| 294 |
} |
| 295 |
|
| 296 |
if (paylen > sizeof(struct ips_connect_reqrep)) { |
| 297 |
int count; |
| 298 |
char *p = (char *)(req + 1); |
| 299 |
paylen -= sizeof(struct ips_connect_reqrep); |
| 300 |
if (paylen%(sizeof(uint64_t)+sizeof(psm_epid_t))) { |
| 301 |
return PSM_INTERNAL_ERR; |
| 302 |
} |
| 303 |
count = paylen / (sizeof(uint64_t)+sizeof(psm_epid_t)); |
| 304 |
if (count > IPATH_MAX_UNIT) return PSM_INTERNAL_ERR; |
| 305 |
|
| 306 |
memcpy(ipsaddr->epaddr->mctxt_gidhi, p, count*sizeof(uint64_t)); |
| 307 |
p += count*sizeof(uint64_t); |
| 308 |
memcpy(ipsaddr->epaddr->mctxt_epid, p, count*sizeof(psm_epid_t)); |
| 309 |
ipsaddr->epaddr->mctxt_epcount = count; |
| 310 |
} |
| 311 |
|
| 312 |
return psmi_epid_set_hostname(psm_epid_nid(ipsaddr->epaddr->epid), |
| 313 |
(char*) req->hostname, 0); |
| 314 |
} |
| 315 |
|
| 316 |
static psm_error_t __recvpath |
| 317 |
ips_proto_send_ctrl_message_request(struct ips_proto *proto, |
| 318 |
struct ips_flow *flow, uint8_t message_type, |
| 319 |
uint32_t *msg_queue_mask, void *payload, |
| 320 |
uint64_t timeout) |
| 321 |
{ |
| 322 |
psm_error_t err = PSM_OK; |
| 323 |
|
| 324 |
while (get_cycles() < timeout) { |
| 325 |
err = ips_proto_send_ctrl_message(flow, message_type, |
| 326 |
msg_queue_mask, payload); |
| 327 |
if (err == PSM_OK) { |
| 328 |
break; |
| 329 |
} |
| 330 |
if ((err = psmi_err_only(psmi_poll_internal(proto->ep, 1)))) { |
| 331 |
break; |
| 332 |
} |
| 333 |
} |
| 334 |
return err; |
| 335 |
} |
| 336 |
|
| 337 |
static psm_error_t __recvpath |
| 338 |
ips_proto_send_ctrl_message_reply(struct ips_flow *flow, uint8_t message_type, |
| 339 |
uint32_t *msg_queue_mask, void *payload) |
| 340 |
{ |
| 341 |
/* This will try up to 100 times until the message is sent. The code |
| 342 |
* is persistent becausing dropping replies will lead to a lack of |
| 343 |
* overall progress on the connection/disconnection. We do not want |
| 344 |
* to poll from here, and we cannot afford a lengthy timeout, since |
| 345 |
* this is called from the receive path. |
| 346 |
*/ |
| 347 |
psm_error_t err = PSM_OK; |
| 348 |
int i; |
| 349 |
for (i = 0; i < 100; i++) { |
| 350 |
err = ips_proto_send_ctrl_message(flow, message_type, |
| 351 |
msg_queue_mask, payload); |
| 352 |
if (err == PSM_OK) { |
| 353 |
break; |
| 354 |
} |
| 355 |
} |
| 356 |
return err; |
| 357 |
} |
| 358 |
|
| 359 |
int |
| 360 |
ips_proto_build_connect_message(struct ips_proto *proto, |
| 361 |
struct ips_proto_ctrl_message *msg, |
| 362 |
ips_epaddr_t *ipsaddr, uint8_t opcode, |
| 363 |
void *payload) |
| 364 |
{ |
| 365 |
struct connect_msghdr *hdr = (struct connect_msghdr *) payload; |
| 366 |
struct ips_connect_reqrep *req = |
| 367 |
(struct ips_connect_reqrep *) payload; |
| 368 |
uint32_t paylen = sizeof(struct connect_msghdr); |
| 369 |
|
| 370 |
/* Write standard header that goes out on all connect msgs */ |
| 371 |
hdr->connect_verno = __cpu_to_be16(IPS_CONNECT_VERNO); |
| 372 |
hdr->psm_verno = __cpu_to_be16(PSMI_VERNO); |
| 373 |
hdr->opcode = opcode; |
| 374 |
hdr->phase = 0; |
| 375 |
hdr->hca_type = proto->epinfo.ep_hca_type; |
| 376 |
hdr->sl = ipsaddr->epr.epr_path[IPS_PATH_HIGH_PRIORITY][0]->epr_sl; |
| 377 |
|
| 378 |
/* Some times we simply echo disconnect requests since we can get dupe |
| 379 |
* disconnect requests. Unless that's the case, we always send the full |
| 380 |
* uuid */ |
| 381 |
psmi_assert_always(proto != NULL); |
| 382 |
memcpy(&hdr->uuid, &proto->ep->key, sizeof(psm_uuid_t)); |
| 383 |
|
| 384 |
switch (opcode) { |
| 385 |
case OPCODE_CONNECT_REPLY: |
| 386 |
case OPCODE_CONNECT_REQUEST: |
| 387 |
#if 0 |
| 388 |
psmi_assert_always(ipsaddr->cerror_from != PSM_OK || |
| 389 |
!COMMIDX_IS_UNKNOWN(proto, ipsaddr->commidx_from)); |
| 390 |
#endif |
| 391 |
if (opcode == OPCODE_CONNECT_REQUEST) { |
| 392 |
req->connect_result = __cpu_to_be16(PSM_OK); |
| 393 |
req->runid_key = proto->runid_key; |
| 394 |
} |
| 395 |
else { |
| 396 |
req->connect_result = __cpu_to_be16(ipsaddr->cerror_from); |
| 397 |
req->runid_key = ipsaddr->runid_key; |
| 398 |
} |
| 399 |
req->flags = 0; |
| 400 |
req->commidx = (uint32_t) ipsaddr->epr.epr_commidx_from; |
| 401 |
req->job_pkey = ipsaddr->epr.epr_path[IPS_PATH_HIGH_PRIORITY][0]->epr_pkey; |
| 402 |
|
| 403 |
req->features = |
| 404 |
__cpu_to_be16(psmi_ips_node_features(proto->ep)); |
| 405 |
req->hdrq_msg_size = proto->epinfo.ep_hdrq_msg_size; |
| 406 |
req->mtu = ipsaddr->epr.epr_path[IPS_PATH_HIGH_PRIORITY][0]->epr_mtu; |
| 407 |
strncpy(req->hostname, psmi_gethostname(), |
| 408 |
sizeof(req->hostname) - 1); |
| 409 |
req->hostname[sizeof(req->hostname) - 1] = '\0'; |
| 410 |
paylen = sizeof(struct ips_connect_reqrep); |
| 411 |
|
| 412 |
/* Attach all multi-context subnetids and epids. */ |
| 413 |
if (proto->ep->mctxt_master == proto->ep) { |
| 414 |
psm_epid_t *epid; |
| 415 |
psm_ep_t ep = proto->ep->mctxt_next; |
| 416 |
uint64_t *subnetid = (uint64_t *)(req + 1); |
| 417 |
/* first all subnetids */ |
| 418 |
while (ep != proto->ep) { |
| 419 |
*subnetid = ep->gid_hi; |
| 420 |
subnetid++; |
| 421 |
ep = ep->mctxt_next; |
| 422 |
paylen += sizeof(uint64_t); |
| 423 |
} |
| 424 |
ep = proto->ep->mctxt_next; |
| 425 |
epid = (psm_epid_t *)subnetid; |
| 426 |
/* second all epids */ |
| 427 |
while (ep != proto->ep) { |
| 428 |
*epid = ep->epid; |
| 429 |
epid++; |
| 430 |
ep = ep->mctxt_next; |
| 431 |
paylen += sizeof(psm_epid_t); |
| 432 |
} |
| 433 |
} |
| 434 |
psmi_assert_always(paylen <= IPS_MAX_CONNECT_PAYLEN); |
| 435 |
break; |
| 436 |
|
| 437 |
case OPCODE_DISCONNECT_REQUEST: |
| 438 |
case OPCODE_DISCONNECT_REPLY: |
| 439 |
paylen = sizeof(struct ips_disconnect_reqrep); |
| 440 |
break; |
| 441 |
default: |
| 442 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 443 |
"Unexpected/unhandled connection opcode 0x%x\n", |
| 444 |
opcode); |
| 445 |
break; |
| 446 |
} |
| 447 |
return paylen; |
| 448 |
} |
| 449 |
|
| 450 |
void |
| 451 |
ips_flow_init(struct ips_flow *flow, ips_path_rec_t *path, ips_epaddr_t *ipsaddr, psm_transfer_type_t transfer_type, psm_protocol_type_t protocol, ips_path_type_t path_type, uint32_t flow_index) |
| 452 |
{ |
| 453 |
struct ips_proto *proto = ipsaddr->proto; |
| 454 |
|
| 455 |
psmi_assert_always(protocol < IPS_MAX_PROTOCOL); |
| 456 |
psmi_assert_always(flow_index < IPS_MAX_FLOWINDEX); |
| 457 |
|
| 458 |
SLIST_NEXT(flow, next) = NULL; |
| 459 |
flow->fn.xfer = psmi_xfer_fn[transfer_type]; |
| 460 |
flow->fn.protocol = psmi_protocol_fn[protocol]; |
| 461 |
|
| 462 |
/* If path is not specified pick one accordingly */ |
| 463 |
if (!path) |
| 464 |
path = ips_select_path(proto, path_type, ipsaddr); |
| 465 |
|
| 466 |
flow->path = path; |
| 467 |
flow->ipsaddr = ipsaddr; |
| 468 |
flow->epinfo = &proto->epinfo; |
| 469 |
flow->transfer= transfer_type; |
| 470 |
flow->protocol= protocol; |
| 471 |
flow->flowid = IPS_FLOWID_PACK(protocol, flow_index); |
| 472 |
flow->xmit_seq_num.val = 0; |
| 473 |
flow->xmit_ack_num.val = 0; |
| 474 |
flow->xmit_ack_num.pkt--; /* last acked */ |
| 475 |
flow->recv_seq_num.val = 0; |
| 476 |
flow->flags = 0; |
| 477 |
flow->sl = flow->path->epr_sl; |
| 478 |
flow->cca_ooo_pkts = 0; |
| 479 |
flow->credits = flow->cwin = proto->flow_credits; |
| 480 |
flow->ack_interval = max((proto->flow_credits >> 2) - 1, 1); |
| 481 |
flow->scb_num_pending = 0; |
| 482 |
flow->scb_num_unacked = 0; |
| 483 |
|
| 484 |
psmi_timer_entry_init(&(flow->timer_ack), |
| 485 |
ips_proto_timer_ack_callback, flow); |
| 486 |
|
| 487 |
psmi_timer_entry_init(&(flow->timer_send), |
| 488 |
ips_proto_timer_send_callback, flow); |
| 489 |
|
| 490 |
STAILQ_INIT(&flow->scb_unacked); |
| 491 |
SLIST_INIT(&flow->scb_pend); |
| 492 |
return; |
| 493 |
} |
| 494 |
|
| 495 |
static |
| 496 |
size_t |
| 497 |
epaddr_size() |
| 498 |
{ |
| 499 |
return (size_t) (sizeof(struct psm_epaddr) + sizeof(struct ptl_epaddr)); |
| 500 |
} |
| 501 |
|
| 502 |
static |
| 503 |
psm_error_t |
| 504 |
ips_init_ep_qp_and_pkt_context(uint16_t hca_type, uint32_t qp, |
| 505 |
uint32_t context, ips_epaddr_t *ipsaddr) |
| 506 |
{ |
| 507 |
psm_error_t err = PSM_OK; |
| 508 |
switch(hca_type) { |
| 509 |
case PSMI_HCA_TYPE_QLE73XX: |
| 510 |
/* Bit 5 of the context is inserted into bit 0 of QP */ |
| 511 |
ipsaddr->epr.epr_qp = (qp & ~0x1) | (context >> 4); |
| 512 |
ipsaddr->epr.epr_pkt_context = context & 0xf; |
| 513 |
break; |
| 514 |
case PSMI_HCA_TYPE_QLE72XX: |
| 515 |
if (context == 16) { |
| 516 |
/* For context 16, the bottom bit of qp is toggled */ |
| 517 |
ipsaddr->epr.epr_qp = qp ^ 1; |
| 518 |
ipsaddr->epr.epr_pkt_context = 15; |
| 519 |
} |
| 520 |
else { |
| 521 |
ipsaddr->epr.epr_qp = qp; |
| 522 |
ipsaddr->epr.epr_pkt_context = context; |
| 523 |
} |
| 524 |
break; |
| 525 |
case PSMI_HCA_TYPE_QLE71XX: |
| 526 |
ipsaddr->epr.epr_qp = qp; |
| 527 |
ipsaddr->epr.epr_pkt_context = context; |
| 528 |
break; |
| 529 |
default: |
| 530 |
err = PSM_PARAM_ERR; |
| 531 |
break; |
| 532 |
} |
| 533 |
return err; |
| 534 |
} |
| 535 |
|
| 536 |
static |
| 537 |
psm_epaddr_t |
| 538 |
ips_alloc_epaddr(struct ips_proto *proto, psm_epid_t epid, |
| 539 |
const char *hostname, unsigned long timeout) |
| 540 |
{ |
| 541 |
psm_error_t err = PSM_OK; |
| 542 |
psm_epaddr_t epaddr; |
| 543 |
ips_epaddr_t *ipsaddr; |
| 544 |
uint64_t lid, context, subcontext; |
| 545 |
uint16_t hca_type, path_dlid; |
| 546 |
uint16_t lmc_mask = ~((1 << proto->epinfo.ep_lmc) - 1); |
| 547 |
int i; |
| 548 |
ips_path_type_t prio; |
| 549 |
|
| 550 |
/* The PSM/PTL-level and ips-level epaddr structures are colocated in |
| 551 |
* memory for performance reasons -- this is why ips allocates memory for |
| 552 |
* both the PSM/PTL-level and ips-level epaddr structure. |
| 553 |
* |
| 554 |
* The PSM/PTL structure data is filled in upon successfuly ep connect in |
| 555 |
* ips_ptl_connect(). |
| 556 |
*/ |
| 557 |
epaddr = (psm_epaddr_t) psmi_calloc(proto->ep, PER_PEER_ENDPOINT, |
| 558 |
1, epaddr_size()); |
| 559 |
if (epaddr == NULL) |
| 560 |
return NULL; |
| 561 |
|
| 562 |
epaddr->ptl = proto->ptl; |
| 563 |
epaddr->ptlctl = proto->ptl->ctl; |
| 564 |
epaddr->ep = proto->ep; |
| 565 |
STAILQ_INIT(&epaddr->egrlong); |
| 566 |
STAILQ_INIT(&epaddr->egrdata); |
| 567 |
epaddr->xmit_egrlong.egr_data = 0; |
| 568 |
epaddr->outoforder_q.first = NULL; |
| 569 |
epaddr->outoforder_q.lastp = &epaddr->outoforder_q.first; |
| 570 |
epaddr->mctxt_master = epaddr; |
| 571 |
epaddr->mctxt_current = epaddr; |
| 572 |
epaddr->mctxt_prev = epaddr->mctxt_next = epaddr; |
| 573 |
|
| 574 |
/* IPS-level epaddr */ |
| 575 |
ipsaddr = (ips_epaddr_t *)(epaddr+1); |
| 576 |
epaddr->ptladdr = ipsaddr; |
| 577 |
|
| 578 |
ipsaddr->ptl = proto->ptl; |
| 579 |
ipsaddr->mq = proto->mq; |
| 580 |
ipsaddr->epaddr = epaddr; |
| 581 |
ipsaddr->proto = proto; |
| 582 |
|
| 583 |
/* Setup base fields for remote epid before doing path record lookup: |
| 584 |
*/ |
| 585 |
lid = PSMI_EPID_GET_LID(epid); |
| 586 |
context = PSMI_EPID_GET_CONTEXT(epid); |
| 587 |
subcontext = PSMI_EPID_GET_SUBCONTEXT(epid); |
| 588 |
hca_type = PSMI_EPID_GET_HCATYPE(epid); |
| 589 |
/* Actual context of peer */ |
| 590 |
ipsaddr->epr.epr_context = context; |
| 591 |
|
| 592 |
/* Setup remote endpoint <context,sucontext> */ |
| 593 |
err = ips_init_ep_qp_and_pkt_context(hca_type, proto->epinfo.ep_baseqp, |
| 594 |
context, ipsaddr); |
| 595 |
if (err != PSM_OK) { |
| 596 |
_IPATH_ERROR("Connect: Warning! unknown HCA type %d. Assuming remote HCA is same as local.\n", hca_type); |
| 597 |
ips_init_ep_qp_and_pkt_context(hca_type, proto->epinfo.ep_baseqp, |
| 598 |
PSMI_EPID_GET_CONTEXT(proto->ep->epid), ipsaddr); |
| 599 |
} |
| 600 |
|
| 601 |
/* Subcontext */ |
| 602 |
ipsaddr->epr.epr_subcontext = subcontext; |
| 603 |
|
| 604 |
/* Get path record for <service, slid, dlid> tuple */ |
| 605 |
err = proto->ibta.get_path_rec(proto, proto->epinfo.ep_base_lid, |
| 606 |
__cpu_to_be16(lid), hca_type, timeout, |
| 607 |
ipsaddr); |
| 608 |
if (err != PSM_OK) { |
| 609 |
psmi_free(epaddr); |
| 610 |
return NULL; |
| 611 |
} |
| 612 |
|
| 613 |
/* Determine base lid across all paths */ |
| 614 |
ipsaddr->epr.epr_base_lid = |
| 615 |
__be16_to_cpu(ipsaddr->epr.epr_path[IPS_PATH_HIGH_PRIORITY][0]->epr_dlid); |
| 616 |
|
| 617 |
for (prio = IPS_PATH_LOW_PRIORITY; prio < IPS_PATH_MAX_PRIORITY; prio++) |
| 618 |
for (i = 0; i < ipsaddr->epr.epr_num_paths[prio]; i++) { |
| 619 |
path_dlid = __be16_to_cpu(ipsaddr->epr.epr_path[prio][i]->epr_dlid); |
| 620 |
if (path_dlid < ipsaddr->epr.epr_base_lid) |
| 621 |
ipsaddr->epr.epr_base_lid = path_dlid; |
| 622 |
} |
| 623 |
|
| 624 |
|
| 625 |
/* Finally construct the resolved epaddr->epid for this peer (For torus |
| 626 |
* SL and even the lid may be different!) |
| 627 |
*/ |
| 628 |
path_dlid = ipsaddr->epr.epr_base_lid & lmc_mask; |
| 629 |
|
| 630 |
epaddr->epid = |
| 631 |
PSMI_EPID_PACK_EXT(path_dlid, |
| 632 |
context, subcontext, |
| 633 |
hca_type, |
| 634 |
ipsaddr->epr.epr_path[IPS_PATH_HIGH_PRIORITY][0]->epr_sl); |
| 635 |
|
| 636 |
/* Add this epid as a known hostname to our epid hostname db */ |
| 637 |
if (psmi_epid_set_hostname(psm_epid_nid(epid), hostname, 0)) |
| 638 |
return NULL; |
| 639 |
|
| 640 |
ipsaddr->flags = 0; |
| 641 |
|
| 642 |
/* All flows are over BULK path. Only control messages use the high |
| 643 |
* priority CONTROL path. |
| 644 |
*/ |
| 645 |
ips_flow_init(&ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO], NULL, |
| 646 |
ipsaddr, PSM_TRANSFER_PIO, PSM_PROTOCOL_GO_BACK_N, |
| 647 |
IPS_PATH_NORMAL_PRIORITY, EP_FLOW_GO_BACK_N_PIO); |
| 648 |
|
| 649 |
/* DMA flow uses the same path as PIO flow due to multi MTU sized |
| 650 |
* eager messages. If we use separate paths we are more likely to have |
| 651 |
* payload arrive out of order with respect to envelope leading to |
| 652 |
* un-necessary NAKs. |
| 653 |
*/ |
| 654 |
ips_flow_init(&ipsaddr->flows[EP_FLOW_GO_BACK_N_DMA], |
| 655 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO].path, |
| 656 |
ipsaddr, PSM_TRANSFER_DMA, PSM_PROTOCOL_GO_BACK_N, |
| 657 |
IPS_PATH_NORMAL_PRIORITY, EP_FLOW_GO_BACK_N_DMA); |
| 658 |
|
| 659 |
/* AM Request messages also use the same path as the PIO flow as they |
| 660 |
* also require order with respect to the MPI request messages. |
| 661 |
*/ |
| 662 |
ips_flow_init(&ipsaddr->flows[EP_FLOW_GO_BACK_N_AM_REQ], |
| 663 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO].path, |
| 664 |
ipsaddr, PSM_TRANSFER_PIO, PSM_PROTOCOL_GO_BACK_N, |
| 665 |
IPS_PATH_NORMAL_PRIORITY, EP_FLOW_GO_BACK_N_AM_REQ); |
| 666 |
|
| 667 |
ips_flow_init(&ipsaddr->flows[EP_FLOW_GO_BACK_N_AM_RSP], NULL, |
| 668 |
ipsaddr, PSM_TRANSFER_PIO, PSM_PROTOCOL_GO_BACK_N, |
| 669 |
IPS_PATH_NORMAL_PRIORITY, EP_FLOW_GO_BACK_N_AM_RSP); |
| 670 |
|
| 671 |
/* tidflow for tid get request */ |
| 672 |
ips_flow_init(&ipsaddr->tidgr_flow, NULL, ipsaddr, |
| 673 |
PSM_TRANSFER_DMA, PSM_PROTOCOL_TIDFLOW, |
| 674 |
IPS_PATH_LOW_PRIORITY, 0); |
| 675 |
|
| 676 |
ipsaddr->cstate_to = CSTATE_NONE; |
| 677 |
ipsaddr->cstate_from = CSTATE_NONE; |
| 678 |
|
| 679 |
/* For now, set these to our PSM versions and connect versions. They will |
| 680 |
* be overwritten to the peer's versions in handling connection reqs |
| 681 |
*/ |
| 682 |
ipsaddr->psm_verno = PSMI_VERNO; |
| 683 |
ipsaddr->connect_verno = IPS_CONNECT_VERNO; |
| 684 |
|
| 685 |
/* Add epaddr to PSM's epid table */ |
| 686 |
psmi_epid_add(proto->ep, epaddr->epid, epaddr); |
| 687 |
psmi_assert_always(psmi_epid_lookup(proto->ep, epaddr->epid) == epaddr); |
| 688 |
|
| 689 |
return epaddr; |
| 690 |
} |
| 691 |
|
| 692 |
static |
| 693 |
void |
| 694 |
ips_free_epaddr(ips_epaddr_t *ipsaddr) |
| 695 |
{ |
| 696 |
psm_epaddr_t epaddr = ipsaddr->epaddr; |
| 697 |
_IPATH_VDBG("epaddr=%p,ipsaddr=%p,commidx_from=%d\n", epaddr, ipsaddr, |
| 698 |
ipsaddr->epr.epr_commidx_from); |
| 699 |
psmi_epid_remove(ipsaddr->proto->ep, epaddr->epid); |
| 700 |
ips_epstate_del(ipsaddr->proto->epstate, ipsaddr->epr.epr_commidx_from); |
| 701 |
psmi_free(epaddr); |
| 702 |
return; |
| 703 |
} |
| 704 |
|
| 705 |
static psm_error_t ips_get_addr_from_epid(struct ips_proto *proto, |
| 706 |
psm_epid_t epid, |
| 707 |
unsigned long timeout, |
| 708 |
psm_epaddr_t *epaddr) |
| 709 |
{ |
| 710 |
psm_error_t err; |
| 711 |
uint64_t lid, context, subcontext; |
| 712 |
uint16_t hca_type, path_dlid; |
| 713 |
psm_epid_t path_epid; |
| 714 |
psm_epaddr_t ep_address = NULL; |
| 715 |
uint16_t lmc_mask = ~((1 << proto->epinfo.ep_lmc) - 1); |
| 716 |
ips_epaddr_t ipsaddr; |
| 717 |
|
| 718 |
/* First unpack to get slid/dlid. */ |
| 719 |
lid = PSMI_EPID_GET_LID(epid); |
| 720 |
context = PSMI_EPID_GET_CONTEXT(epid); |
| 721 |
subcontext = PSMI_EPID_GET_SUBCONTEXT(epid); |
| 722 |
hca_type = PSMI_EPID_GET_HCATYPE(epid); |
| 723 |
|
| 724 |
/* Get path record for <service, slid, dlid> tuple */ |
| 725 |
err = proto->ibta.get_path_rec(proto, proto->epinfo.ep_base_lid, |
| 726 |
__cpu_to_be16(lid), hca_type, |
| 727 |
timeout, &ipsaddr); |
| 728 |
if (err != PSM_OK) |
| 729 |
goto fail; |
| 730 |
|
| 731 |
/* Generate path epid to do lookup on - uses the SL from the path record. |
| 732 |
*/ |
| 733 |
path_dlid = (__be16_to_cpu(ipsaddr.epr.epr_path[IPS_PATH_HIGH_PRIORITY][0]->epr_dlid)) & lmc_mask; |
| 734 |
|
| 735 |
path_epid = |
| 736 |
PSMI_EPID_PACK_EXT(path_dlid, |
| 737 |
context, subcontext, hca_type, |
| 738 |
ipsaddr.epr.epr_path[IPS_PATH_HIGH_PRIORITY][0]->epr_sl); |
| 739 |
ep_address = psmi_epid_lookup(proto->ep, path_epid); |
| 740 |
|
| 741 |
fail: |
| 742 |
*epaddr = ep_address; |
| 743 |
return err; |
| 744 |
} |
| 745 |
|
| 746 |
static |
| 747 |
psm_error_t |
| 748 |
ptl_handle_connect_req(struct ips_proto *proto, psm_epid_t epid, |
| 749 |
psm_epaddr_t epaddr, struct ips_connect_reqrep *req, |
| 750 |
uint32_t paylen, int uuid_valid); |
| 751 |
|
| 752 |
psm_error_t |
| 753 |
ips_proto_process_connect(struct ips_proto *proto, psm_epid_t epid, |
| 754 |
uint8_t opcode, struct ips_message_header *p_hdr, |
| 755 |
void *payload, uint32_t paylen) |
| 756 |
{ |
| 757 |
psm_epaddr_t epaddr; |
| 758 |
ips_epaddr_t *ipsaddr; |
| 759 |
struct connect_msghdr *hdr; |
| 760 |
uint16_t connect_result; |
| 761 |
psm_ep_t ep = proto->ep; |
| 762 |
int uuid_valid; |
| 763 |
int uwords = (proto->epinfo.ep_hdrq_msg_size>>2) - |
| 764 |
IPS_HEADER_QUEUE_IWORDS - IPS_HEADER_QUEUE_HWORDS; |
| 765 |
int hdrq_extra; |
| 766 |
uint32_t lid, context, subcontext; |
| 767 |
uint16_t lmc_mask = ~((1 << proto->epinfo.ep_lmc) - 1); |
| 768 |
|
| 769 |
PSMI_PLOCK_ASSERT(); |
| 770 |
|
| 771 |
struct ips_connect_reqrep *req; |
| 772 |
psm_error_t err = PSM_OK; |
| 773 |
|
| 774 |
/* If the sender doesn't have the same header/eager cutoff, we need to make |
| 775 |
* sure we copy the connect data into a contiguous buffer */ |
| 776 |
char buf[IPS_MAX_CONNECT_PAYLEN] PSMI_CACHEALIGN; |
| 777 |
|
| 778 |
hdrq_extra = uwords - p_hdr->hdr_dlen; |
| 779 |
if (hdrq_extra != 0) { |
| 780 |
uint32_t *bufp = (uint32_t *) buf; |
| 781 |
uint32_t *payp = (uint32_t *) payload; |
| 782 |
_IPATH_VDBG("hdrq_extra is %d, uwords=%d, inwords=%d\n", |
| 783 |
hdrq_extra, uwords, p_hdr->hdr_dlen); |
| 784 |
int hdrq_extra = uwords - p_hdr->hdr_dlen; |
| 785 |
if (hdrq_extra > 0) { /* some of it went into our hdrq */ |
| 786 |
psmi_mq_mtucpy(bufp, &p_hdr->data[0].u32w0 + p_hdr->hdr_dlen, |
| 787 |
hdrq_extra<<2); |
| 788 |
psmi_mq_mtucpy(bufp+hdrq_extra, payload, paylen); |
| 789 |
paylen += (hdrq_extra<<2); |
| 790 |
} |
| 791 |
else { /* we got some useless padding in eager */ |
| 792 |
hdrq_extra = -hdrq_extra; |
| 793 |
paylen -= (hdrq_extra<<2); |
| 794 |
psmi_mq_mtucpy(bufp, payp + hdrq_extra, paylen); |
| 795 |
} |
| 796 |
payload = buf; |
| 797 |
} |
| 798 |
|
| 799 |
hdr = (struct connect_msghdr *) payload; |
| 800 |
if (paylen < sizeof(struct connect_msghdr)) { /* drop */ |
| 801 |
_IPATH_PRDBG("dropping unknown connect message of length %d\n", paylen); |
| 802 |
return PSM_OK; |
| 803 |
} |
| 804 |
|
| 805 |
/* Obtain HCA type and SL from request and regenerate epid */ |
| 806 |
lid = PSMI_EPID_GET_LID(epid); |
| 807 |
context = PSMI_EPID_GET_CONTEXT(epid); |
| 808 |
subcontext = PSMI_EPID_GET_SUBCONTEXT(epid); |
| 809 |
epid = PSMI_EPID_PACK_EXT(lid & lmc_mask, context, subcontext, hdr->hca_type, hdr->sl); |
| 810 |
|
| 811 |
/* Don't need to call ips_get_addr_from_epid as the epid cache is keyed |
| 812 |
* of the IPS_PATH_HIGH_PRIORITY dlid and the SL which we already have from |
| 813 |
* the connect request (as all control messages uses the CONTROL path). |
| 814 |
*/ |
| 815 |
epaddr = psmi_epid_lookup(proto->ep, epid); |
| 816 |
ipsaddr = epaddr ? epaddr->ptladdr : NULL; |
| 817 |
|
| 818 |
uuid_valid = (psmi_uuid_compare(ep->key, hdr->uuid) == 0); |
| 819 |
|
| 820 |
if ((opcode == OPCODE_CONNECT_REQUEST || opcode == OPCODE_CONNECT_REPLY) && |
| 821 |
paylen < IPS_CONNECT_REQREP_MINIMUM_SIZE) |
| 822 |
{ |
| 823 |
uint64_t lid, context, subcontext; |
| 824 |
char *type = opcode == OPCODE_CONNECT_REQUEST ? "request" : "reply"; |
| 825 |
lid = PSMI_EPID_GET_LID(epid); |
| 826 |
context = PSMI_EPID_GET_CONTEXT(epid); |
| 827 |
subcontext = PSMI_EPID_GET_SUBCONTEXT(epid); |
| 828 |
psmi_syslog(proto->ep, 1, LOG_INFO, |
| 829 |
"Unrecognized connect %s (size is %d instead of %d) " |
| 830 |
"from epid %ld:%ld:%ld\n", type, paylen, |
| 831 |
(int) IPS_CONNECT_REQREP_MINIMUM_SIZE, |
| 832 |
(long) lid, (long) context, (long) subcontext); |
| 833 |
goto fail; /* Not fatal, just drop the packet */ |
| 834 |
} |
| 835 |
|
| 836 |
switch (opcode) { |
| 837 |
case OPCODE_CONNECT_REQUEST: |
| 838 |
err = ptl_handle_connect_req(proto, epid, epaddr, |
| 839 |
(struct ips_connect_reqrep *) payload, paylen, uuid_valid); |
| 840 |
break; |
| 841 |
|
| 842 |
case OPCODE_CONNECT_REPLY: |
| 843 |
req = (struct ips_connect_reqrep *) payload; |
| 844 |
if (!ipsaddr || req->runid_key != proto->runid_key) { |
| 845 |
uint64_t lid, context, subcontext; |
| 846 |
|
| 847 |
lid = PSMI_EPID_GET_LID(epid); |
| 848 |
context = PSMI_EPID_GET_CONTEXT(epid); |
| 849 |
subcontext = PSMI_EPID_GET_SUBCONTEXT(epid); |
| 850 |
_IPATH_PRDBG("Unknown connectrep (ipsaddr=%p, %d,%d) " |
| 851 |
"from epid %ld:%ld:%ld bad_uuid=%s\n", |
| 852 |
ipsaddr, req->runid_key, proto->runid_key, |
| 853 |
(long) lid, (long) context, (long) subcontext, |
| 854 |
uuid_valid ? "NO" : "YES"); |
| 855 |
break; |
| 856 |
} |
| 857 |
if (ipsaddr->cstate_to != CSTATE_TO_WAITING) { |
| 858 |
/* possible dupe */ |
| 859 |
_IPATH_VDBG("connect dupe, expected %d got %d\n", |
| 860 |
CSTATE_TO_WAITING, ipsaddr->cstate_to); |
| 861 |
break; |
| 862 |
} |
| 863 |
connect_result = __be16_to_cpu(req->connect_result); |
| 864 |
|
| 865 |
/* Reply to our request for connection (i.e. outgoing connection) */ |
| 866 |
if (ipsaddr->cstate_from != CSTATE_ESTABLISHED) { |
| 867 |
err = ips_ipsaddr_set_req_params(proto, ipsaddr, req, paylen); |
| 868 |
if (err) goto fail; |
| 869 |
} |
| 870 |
ipsaddr->cstate_to = CSTATE_ESTABLISHED; |
| 871 |
ipsaddr->cerror_to = connect_result; |
| 872 |
|
| 873 |
break; |
| 874 |
|
| 875 |
case OPCODE_DISCONNECT_REQUEST: |
| 876 |
{ |
| 877 |
ips_epaddr_t ipsaddr_f; /* fake a ptl addr */ |
| 878 |
int ipsaddr_do_free = 0; |
| 879 |
psmi_assert_always(paylen >= IPS_DISCONNECT_REQREP_MINIMUM_SIZE); |
| 880 |
_IPATH_VDBG("Got a disconnect from %s\n", psmi_epaddr_get_name(epid)); |
| 881 |
proto->num_disconnect_requests++; |
| 882 |
/* It's possible to get a disconnection request on a ipsaddr that |
| 883 |
* we've since removed if the request is a dupe. Instead of |
| 884 |
* silently dropping the packet, we "echo" the request in the |
| 885 |
* reply. */ |
| 886 |
if (ipsaddr == NULL) { |
| 887 |
uint16_t src_context = IPS_HEADER_SRCCONTEXT_GET(p_hdr); |
| 888 |
uint32_t qp; |
| 889 |
|
| 890 |
ipsaddr = &ipsaddr_f; |
| 891 |
memset(&ipsaddr_f, 0, sizeof(ips_epaddr_t)); |
| 892 |
ipsaddr_f.epr.epr_context = src_context; |
| 893 |
ipsaddr_f.epr.epr_subcontext = p_hdr->src_subcontext; |
| 894 |
|
| 895 |
/* QLE72XX is special for context 16 */ |
| 896 |
if ((hdr->hca_type == PSMI_HCA_TYPE_QLE72XX) && |
| 897 |
(src_context == 16)) |
| 898 |
ipsaddr_f.epr.epr_pkt_context = 15; |
| 899 |
|
| 900 |
/* Get path record for peer */ |
| 901 |
err = proto->ibta.get_path_rec(proto, |
| 902 |
proto->epinfo.ep_base_lid, |
| 903 |
__cpu_to_be16(lid), |
| 904 |
hdr->hca_type, |
| 905 |
3000, &ipsaddr_f); |
| 906 |
if (err != PSM_OK) |
| 907 |
goto fail; |
| 908 |
|
| 909 |
qp = proto->epinfo.ep_baseqp; |
| 910 |
err = ips_init_ep_qp_and_pkt_context(hdr->hca_type, qp, |
| 911 |
src_context, &ipsaddr_f); |
| 912 |
if (err != PSM_OK) { |
| 913 |
_IPATH_ERROR("Disconnect: Warning! unknown HCA type %d.\n", hdr->hca_type); |
| 914 |
goto fail; |
| 915 |
} |
| 916 |
|
| 917 |
ipsaddr_f.proto = proto; |
| 918 |
ipsaddr_f.ptl = (ptl_t *) -1; |
| 919 |
/* If the send fails because of pio_busy, don't let ips queue |
| 920 |
* the request on an invalid ipsaddr, just drop the reply */ |
| 921 |
ipsaddr_f.ctrl_msg_queued = ~0; |
| 922 |
ips_flow_init(&ipsaddr_f.flows[EP_FLOW_GO_BACK_N_PIO], NULL, |
| 923 |
&ipsaddr_f, PSM_TRANSFER_PIO, |
| 924 |
PSM_PROTOCOL_GO_BACK_N, IPS_PATH_LOW_PRIORITY, |
| 925 |
EP_FLOW_GO_BACK_N_PIO); |
| 926 |
_IPATH_VDBG("Disconnect on unknown epaddr, just echo request\n"); |
| 927 |
} |
| 928 |
else if (ipsaddr->cstate_from != CSTATE_NONE) { |
| 929 |
ipsaddr->cstate_from = CSTATE_NONE; |
| 930 |
proto->num_connected_from--; |
| 931 |
if (ipsaddr->cstate_to == CSTATE_NONE) { |
| 932 |
ipsaddr_do_free = 1; |
| 933 |
} |
| 934 |
if (!uuid_valid) { |
| 935 |
uint64_t lid, context, subcontext; |
| 936 |
|
| 937 |
lid = PSMI_EPID_GET_LID(epid); |
| 938 |
context = PSMI_EPID_GET_CONTEXT(epid); |
| 939 |
subcontext = PSMI_EPID_GET_SUBCONTEXT(epid); |
| 940 |
_IPATH_VDBG("Unknown disconnect request from epid %d:%d.%d " |
| 941 |
"bad_uuid=%s\n", (int) lid, |
| 942 |
(int) context, (int) subcontext, uuid_valid ? "NO" : "YES"); |
| 943 |
} |
| 944 |
} |
| 945 |
|
| 946 |
memset(buf, 0, sizeof buf); |
| 947 |
ips_proto_send_ctrl_message_reply(&ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO], |
| 948 |
OPCODE_DISCONNECT_REPLY, |
| 949 |
&ipsaddr->ctrl_msg_queued, buf); |
| 950 |
/* We can safely free the ipsaddr if required since disconnect |
| 951 |
* messages are never enqueued so no reference to ipsaddr is kept */ |
| 952 |
if (ipsaddr_do_free) |
| 953 |
ips_free_epaddr(ipsaddr); |
| 954 |
} |
| 955 |
break; |
| 956 |
|
| 957 |
case OPCODE_DISCONNECT_REPLY: |
| 958 |
if (!ipsaddr || !uuid_valid) { |
| 959 |
uint64_t lid, context, subcontext; |
| 960 |
lid = PSMI_EPID_GET_LID(epid); |
| 961 |
context = PSMI_EPID_GET_CONTEXT(epid); |
| 962 |
subcontext = PSMI_EPID_GET_SUBCONTEXT(epid); |
| 963 |
_IPATH_VDBG("Unknown disconnect reply from epid %d:%d.%d bad_uuid=%s\n", |
| 964 |
(int) lid, (int) context, (int) subcontext, |
| 965 |
uuid_valid ? "NO" : "YES"); |
| 966 |
break; |
| 967 |
} |
| 968 |
else if (ipsaddr->cstate_to == CSTATE_TO_WAITING_DISC) { |
| 969 |
ipsaddr->cstate_to = CSTATE_TO_DISCONNECTED; |
| 970 |
/* Freed in disconnect() if cstate_from == NONE */ |
| 971 |
} /* else dupe reply */ |
| 972 |
break; |
| 973 |
|
| 974 |
default: |
| 975 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 976 |
"Unexpected/unhandled connect opcode 0x%x\n", |
| 977 |
opcode); |
| 978 |
} |
| 979 |
fail: |
| 980 |
return err; |
| 981 |
} |
| 982 |
|
| 983 |
static |
| 984 |
psm_error_t |
| 985 |
ptl_handle_connect_req(struct ips_proto *proto, psm_epid_t epid, |
| 986 |
psm_epaddr_t epaddr, struct ips_connect_reqrep *req, |
| 987 |
uint32_t paylen, int uuid_valid) |
| 988 |
{ |
| 989 |
ips_epaddr_t *ipsaddr; |
| 990 |
psm_error_t err = PSM_OK; |
| 991 |
uint16_t connect_result = PSM_OK; |
| 992 |
uint16_t psm_verno; |
| 993 |
uint16_t c_verno; |
| 994 |
uint16_t features; |
| 995 |
int newconnect = 0; |
| 996 |
char buf[IPS_MAX_CONNECT_PAYLEN] PSMI_CACHEALIGN; |
| 997 |
|
| 998 |
if (epid == proto->ep->epid) { |
| 999 |
/* For 2.0, we won't expose handling for this error */ |
| 1000 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_EPID_NETWORK_ERROR, |
| 1001 |
"Network connectivity problem: Locally detected duplicate " |
| 1002 |
"LIDs 0x%04x on hosts %s and %s. (Exiting)", |
| 1003 |
(uint32_t) psm_epid_nid(epid), |
| 1004 |
psmi_epaddr_get_hostname(epid), |
| 1005 |
psmi_gethostname()); |
| 1006 |
/* XXX no return */ |
| 1007 |
abort(); |
| 1008 |
} |
| 1009 |
else if (epaddr == NULL) { /* new ep connect before we call into connect */ |
| 1010 |
newconnect = 1; |
| 1011 |
if ((epaddr = ips_alloc_epaddr(proto, epid, req->hostname, |
| 1012 |
5000)) == NULL) { |
| 1013 |
err = PSM_NO_MEMORY; |
| 1014 |
goto fail; |
| 1015 |
} |
| 1016 |
} |
| 1017 |
ipsaddr = epaddr->ptladdr; |
| 1018 |
if (ipsaddr->cstate_from == CSTATE_ESTABLISHED) { |
| 1019 |
/* Duplicate lid detection. */ |
| 1020 |
if (ipsaddr->runid_key == req->runid_key && uuid_valid) |
| 1021 |
goto do_reply; /* duplicate request, not duplicate lid */ |
| 1022 |
else if (uuid_valid) { |
| 1023 |
/* True blue duplicate lid, both connect messages are part of the |
| 1024 |
* same context since they use the same uuid */ |
| 1025 |
/* For 2.0, we won't expose handling for this error */ |
| 1026 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_EPID_NETWORK_ERROR, |
| 1027 |
"Network connectivity problem: Detected duplicate " |
| 1028 |
"LIDs 0x%x on hosts %s (key=%d) and %s (key=%d). (Exiting)", |
| 1029 |
(uint32_t) psm_epid_nid(ipsaddr->epaddr->epid), |
| 1030 |
psmi_epaddr_get_hostname(epid), |
| 1031 |
ipsaddr->runid_key, |
| 1032 |
req->hostname, |
| 1033 |
req->runid_key); |
| 1034 |
} |
| 1035 |
else { /* Some out of context message. Just drop it */ |
| 1036 |
if (!proto->done_warning) { |
| 1037 |
psmi_syslog(proto->ep, 1, LOG_INFO, |
| 1038 |
"Non-fatal connection problem: Received an out-of-context " |
| 1039 |
"connection message from host %s LID=0x%x context=%d. (Ignoring)", |
| 1040 |
req->hostname, (int) psm_epid_nid(epid), psm_epid_context(epid)); |
| 1041 |
proto->done_warning = 1; |
| 1042 |
} |
| 1043 |
goto no_reply; |
| 1044 |
} |
| 1045 |
} |
| 1046 |
psmi_assert_always(ipsaddr->cstate_from == CSTATE_NONE); |
| 1047 |
|
| 1048 |
/* Save requestor's connection and psm version numbers */ |
| 1049 |
c_verno = __be16_to_cpu(req->hdr.connect_verno); |
| 1050 |
psm_verno = __be16_to_cpu(req->hdr.psm_verno); |
| 1051 |
features = __be16_to_cpu(req->features); |
| 1052 |
|
| 1053 |
/* On PSM pre-2.0, just print message and exit if the connect version |
| 1054 |
* number is not at least 0x0201 */ |
| 1055 |
if (c_verno < 0x0201) { |
| 1056 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_EPID_INVALID_VERSION, |
| 1057 |
"Connect protocol (%x,%x) is obsolete and incompatible", |
| 1058 |
(c_verno >> 8) & 0xff, c_verno & 0xff); |
| 1059 |
connect_result = PSM_EPID_INVALID_CONNECT; |
| 1060 |
} |
| 1061 |
/* Whenever there's a protocol change, adjust handling here */ |
| 1062 |
else if ((IPS_CONNECT_VERNO & 0xff00) != (ipsaddr->connect_verno & 0xff00)) { |
| 1063 |
connect_result = PSM_EPID_INVALID_VERSION; |
| 1064 |
} |
| 1065 |
else if (!node_matches_bitendian(proto->ep, features)) |
| 1066 |
connect_result = PSM_EPID_INVALID_NODE; |
| 1067 |
else if (!psmi_verno_isinteroperable(__be16_to_cpu(req->hdr.psm_verno))) { |
| 1068 |
connect_result = PSM_EPID_INVALID_VERSION; |
| 1069 |
} |
| 1070 |
else if (!(proto->flags & IPS_PROTO_FLAG_QUERY_PATH_REC) && |
| 1071 |
proto->epinfo.ep_pkey != IPATH_DEFAULT_P_KEY && |
| 1072 |
proto->epinfo.ep_pkey != req->job_pkey) { |
| 1073 |
connect_result = PSM_EPID_INVALID_PKEY; |
| 1074 |
} |
| 1075 |
else if (!uuid_valid) { |
| 1076 |
char ep_key[37], req_key[37]; |
| 1077 |
connect_result = PSM_EPID_INVALID_UUID_KEY; |
| 1078 |
psmi_uuid_unparse(proto->ep->key, ep_key); |
| 1079 |
psmi_uuid_unparse(req->hdr.uuid, req_key); |
| 1080 |
_IPATH_PRDBG("UUID key mismatch request key=%s endpoint key=%s\n", |
| 1081 |
req_key, ep_key); |
| 1082 |
} |
| 1083 |
else if (!psmi_verno_isinteroperable(ipsaddr->psm_verno)) { |
| 1084 |
connect_result = PSM_INIT_BAD_API_VERSION; |
| 1085 |
} |
| 1086 |
else { |
| 1087 |
connect_result = PSM_OK; |
| 1088 |
if (ipsaddr->cstate_to == CSTATE_NONE) { |
| 1089 |
ips_epstate_idx idx; |
| 1090 |
psmi_assert_always(newconnect == 1); |
| 1091 |
err = ips_epstate_add(proto->epstate, ipsaddr, &idx); |
| 1092 |
if (err) |
| 1093 |
goto fail; |
| 1094 |
ipsaddr->epr.epr_commidx_from = idx; |
| 1095 |
} |
| 1096 |
} |
| 1097 |
ipsaddr->connect_verno = c_verno; |
| 1098 |
ipsaddr->psm_verno = psm_verno; |
| 1099 |
|
| 1100 |
/* Incoming connection request */ |
| 1101 |
if (ipsaddr->cstate_to != CSTATE_ESTABLISHED) { |
| 1102 |
err = ips_ipsaddr_set_req_params(proto, ipsaddr, req, paylen); |
| 1103 |
if (err) goto fail; |
| 1104 |
} |
| 1105 |
ipsaddr->cstate_from = CSTATE_ESTABLISHED; |
| 1106 |
ipsaddr->cerror_from = connect_result; |
| 1107 |
|
| 1108 |
ipsaddr->runid_key = req->runid_key; |
| 1109 |
ipsaddr->flags |= features & EP_FEATURES_RCVTHREAD ? |
| 1110 |
SESS_FLAG_HAS_RCVTHREAD : 0; |
| 1111 |
ipsaddr->flags |= proto->ep->context.runtime_flags & PSMI_RUNTIME_RCVTHREAD ? |
| 1112 |
SESS_FLAG_LOCK_SESS : 0; |
| 1113 |
ipsaddr->flags |= features & EP_FEATURES_MULTIFLOW ? |
| 1114 |
SESS_FLAG_HAS_FLOWID : 0; |
| 1115 |
|
| 1116 |
pthread_mutex_init(&ipsaddr->sesslock, NULL); |
| 1117 |
|
| 1118 |
proto->num_connected_from++; |
| 1119 |
|
| 1120 |
do_reply: |
| 1121 |
ips_proto_send_ctrl_message_reply(&ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO], |
| 1122 |
OPCODE_CONNECT_REPLY, |
| 1123 |
&ipsaddr->ctrl_msg_queued, buf); |
| 1124 |
no_reply: |
| 1125 |
fail: |
| 1126 |
return err; |
| 1127 |
} |
| 1128 |
|
| 1129 |
psm_error_t |
| 1130 |
ips_proto_connect(struct ips_proto *proto, int numep, |
| 1131 |
const psm_epid_t *array_of_epid, |
| 1132 |
const int *array_of_epid_mask, psm_error_t *array_of_errors, |
| 1133 |
psm_epaddr_t *array_of_epaddr, uint64_t timeout_in) |
| 1134 |
{ |
| 1135 |
int i, n, n_first; |
| 1136 |
psm_error_t err = PSM_OK; |
| 1137 |
psm_epaddr_t epaddr; |
| 1138 |
ips_epaddr_t *ipsaddr = NULL; |
| 1139 |
int numep_toconnect = 0, numep_left; |
| 1140 |
char buf[IPS_MAX_CONNECT_PAYLEN] PSMI_CACHEALIGN; |
| 1141 |
union psmi_envvar_val credits_intval; |
| 1142 |
int connect_credits; |
| 1143 |
|
| 1144 |
psmi_getenv("PSM_CONNECT_CREDITS", |
| 1145 |
"End-point connect request credits.", |
| 1146 |
PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_UINT, |
| 1147 |
(union psmi_envvar_val) 100, |
| 1148 |
&credits_intval); |
| 1149 |
|
| 1150 |
connect_credits = credits_intval.e_uint; |
| 1151 |
|
| 1152 |
PSMI_PLOCK_ASSERT(); |
| 1153 |
|
| 1154 |
/* All timeout values are in cycles */ |
| 1155 |
uint64_t t_start = get_cycles(); |
| 1156 |
/* Print a timeout at the warning interval */ |
| 1157 |
union psmi_envvar_val warn_intval; |
| 1158 |
uint64_t to_warning_interval; |
| 1159 |
uint64_t to_warning_next; |
| 1160 |
|
| 1161 |
/* Setup warning interval */ |
| 1162 |
psmi_getenv("PSM_CONNECT_WARN_INTERVAL", |
| 1163 |
"Period in seconds to warn if connections are not completed." |
| 1164 |
"Default is 300 seconds, 0 to disable", |
| 1165 |
PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT, |
| 1166 |
(union psmi_envvar_val) 300, |
| 1167 |
&warn_intval); |
| 1168 |
|
| 1169 |
to_warning_interval = nanosecs_to_cycles(warn_intval.e_uint * SEC_ULL); |
| 1170 |
to_warning_next = t_start + to_warning_interval; |
| 1171 |
|
| 1172 |
/* Some sanity checks */ |
| 1173 |
psmi_assert_always(sizeof(struct connect_msghdr) == IPS_CONNECT_MSGHDR_SIZE); |
| 1174 |
psmi_assert_always(array_of_epid_mask != NULL); |
| 1175 |
psmi_assert_always(sizeof(struct ips_connect_reqrep) >= |
| 1176 |
IPS_CONNECT_REQREP_MINIMUM_SIZE); |
| 1177 |
psmi_assert_always(sizeof(struct ips_disconnect_reqrep) >= |
| 1178 |
IPS_DISCONNECT_REQREP_MINIMUM_SIZE); |
| 1179 |
|
| 1180 |
/* First pass: make sure array of errors is at least fully defined */ |
| 1181 |
for (i = 0; i < numep; i++) { |
| 1182 |
uint64_t lid, context, subcontext; |
| 1183 |
|
| 1184 |
lid = PSMI_EPID_GET_LID(array_of_epid[i]); |
| 1185 |
context = PSMI_EPID_GET_CONTEXT(array_of_epid[i]); |
| 1186 |
subcontext = PSMI_EPID_GET_SUBCONTEXT(array_of_epid[i]); |
| 1187 |
_IPATH_VDBG("epid-connect=%s connect to %ld:%ld:%ld\n", |
| 1188 |
array_of_epid_mask[i] ? "YES" : " NO", |
| 1189 |
(long) lid, (long) context, (long) subcontext); |
| 1190 |
if (array_of_epid_mask[i]) { |
| 1191 |
array_of_errors[i] = PSM_EPID_UNKNOWN; |
| 1192 |
array_of_epaddr[i] = NULL; |
| 1193 |
} |
| 1194 |
} |
| 1195 |
|
| 1196 |
/* Second pass: see what to connect and what is connectable. */ |
| 1197 |
for (i = 0, numep_toconnect = 0; i < numep; i++) { |
| 1198 |
if (!array_of_epid_mask[i]) |
| 1199 |
continue; |
| 1200 |
/* Can't send to epid on same lid */ |
| 1201 |
if (psm_epid_nid(proto->ep->epid) == psm_epid_nid(array_of_epid[i])) { |
| 1202 |
array_of_errors[i] = PSM_EPID_UNREACHABLE; |
| 1203 |
continue; |
| 1204 |
} |
| 1205 |
|
| 1206 |
err = ips_get_addr_from_epid(proto, array_of_epid[i], 30000, &epaddr); |
| 1207 |
if (err) |
| 1208 |
goto fail; |
| 1209 |
if (epaddr == NULL) { |
| 1210 |
ips_epstate_idx idx; |
| 1211 |
/* We're sending a connect request message before some other node |
| 1212 |
* has sent its connect message */ |
| 1213 |
epaddr = ips_alloc_epaddr(proto, array_of_epid[i], |
| 1214 |
NULL, (timeout_in / 1000000UL)); |
| 1215 |
if (epaddr == NULL) { |
| 1216 |
err = PSM_NO_MEMORY; |
| 1217 |
goto fail; |
| 1218 |
} |
| 1219 |
ipsaddr = epaddr->ptladdr; |
| 1220 |
err = ips_epstate_add(proto->epstate, ipsaddr, &idx); |
| 1221 |
if (err) |
| 1222 |
goto fail; |
| 1223 |
ipsaddr->epr.epr_commidx_from = idx; |
| 1224 |
ipsaddr->cstate_from = CSTATE_NONE; |
| 1225 |
} else if (epaddr->ptladdr->cstate_to != CSTATE_NONE) { /* already connected */ |
| 1226 |
psmi_assert_always(epaddr->ptladdr->cstate_to == CSTATE_ESTABLISHED); |
| 1227 |
array_of_errors[i] = PSM_EPID_ALREADY_CONNECTED; |
| 1228 |
array_of_epaddr[i] = epaddr; |
| 1229 |
continue; |
| 1230 |
} else { |
| 1231 |
/* We've already received a connect request message from a remote |
| 1232 |
* peer, it's time to send our own. */ |
| 1233 |
ipsaddr = epaddr->ptladdr; |
| 1234 |
/* No re-entrancy sanity check and makes sure we are not connected |
| 1235 |
* twice (caller's precondition) */ |
| 1236 |
psmi_assert_always(ipsaddr->cstate_to == CSTATE_NONE); |
| 1237 |
psmi_assert_always(ipsaddr->cstate_from != CSTATE_NONE); |
| 1238 |
#if 0 |
| 1239 |
psmi_assert_always(ipsaddr->cerror_from != PSM_OK || |
| 1240 |
!COMMIDX_IS_UNKNOWN(ptl, ipsaddr->commidx_from)); |
| 1241 |
psmi_assert_always(!COMMIDX_IS_UNKNOWN(ptl, ipsaddr->commidx_to)); |
| 1242 |
#endif |
| 1243 |
} |
| 1244 |
|
| 1245 |
ipsaddr->cstate_to = CSTATE_TO_WAITING; |
| 1246 |
ipsaddr->cerror_to = PSM_OK; |
| 1247 |
array_of_epaddr[i] = epaddr; |
| 1248 |
ipsaddr->s_timeout = get_cycles(); |
| 1249 |
ipsaddr->delay_in_ms = 1; |
| 1250 |
ipsaddr->credit = 0; |
| 1251 |
numep_toconnect++; |
| 1252 |
} |
| 1253 |
|
| 1254 |
/* Second pass: do the actual connect. |
| 1255 |
* PSM_EPID_UNKNOWN: Not connected yet. |
| 1256 |
* PSM_EPID_UNREACHABLE: Not to be connected. |
| 1257 |
* PSM_OK: Successfully connected. |
| 1258 |
* Start sending connect messages at a random index between 0 and numep-1 |
| 1259 |
*/ |
| 1260 |
numep_left = numep_toconnect; |
| 1261 |
n_first = ((uint32_t) get_cycles()) % numep; |
| 1262 |
while (numep_left > 0) { |
| 1263 |
for (n = 0; n < numep; n++) { |
| 1264 |
int keep_polling = 1; |
| 1265 |
i = (n_first + n) % numep; |
| 1266 |
if (!array_of_epid_mask[i]) |
| 1267 |
continue; |
| 1268 |
switch (array_of_errors[i]) { |
| 1269 |
case PSM_EPID_UNREACHABLE: |
| 1270 |
case PSM_EPID_ALREADY_CONNECTED: |
| 1271 |
case PSM_OK: |
| 1272 |
continue; |
| 1273 |
default: |
| 1274 |
break; |
| 1275 |
} |
| 1276 |
psmi_assert_always(array_of_epaddr[i] != NULL); |
| 1277 |
ipsaddr = array_of_epaddr[i]->ptladdr; |
| 1278 |
if (ipsaddr->cstate_to == CSTATE_ESTABLISHED) { |
| 1279 |
/* This is not the real error code, we only set OK here |
| 1280 |
* so we know to stop polling for the reply. The actual |
| 1281 |
* error is in ipsaddr->cerror_to */ |
| 1282 |
array_of_errors[i] = PSM_OK; |
| 1283 |
numep_left--; |
| 1284 |
connect_credits++; |
| 1285 |
ipsaddr->credit = 0; |
| 1286 |
continue; |
| 1287 |
} |
| 1288 |
while (keep_polling) { |
| 1289 |
if (!psmi_cycles_left(t_start, timeout_in)) { |
| 1290 |
err = PSM_TIMEOUT; |
| 1291 |
goto err_timeout; |
| 1292 |
} |
| 1293 |
if (to_warning_interval && get_cycles() >= to_warning_next) { |
| 1294 |
uint64_t waiting_time = |
| 1295 |
cycles_to_nanosecs(get_cycles() - t_start) / SEC_ULL; |
| 1296 |
const char *first_name = NULL; |
| 1297 |
int num_waiting = 0; |
| 1298 |
|
| 1299 |
for (i = 0; i < numep; i++) { |
| 1300 |
if (!array_of_epid_mask[i] || |
| 1301 |
array_of_errors[i] != PSM_EPID_UNKNOWN) |
| 1302 |
continue; |
| 1303 |
if (!first_name) |
| 1304 |
first_name = psmi_epaddr_get_name(array_of_epid[i]); |
| 1305 |
num_waiting++; |
| 1306 |
} |
| 1307 |
if (first_name) { |
| 1308 |
_IPATH_INFO("Couldn't connect to %s (and %d others). " |
| 1309 |
"Time elapsed %02i:%02i:%02i. Still trying...\n", |
| 1310 |
first_name, num_waiting, |
| 1311 |
(int) (waiting_time / 3600), |
| 1312 |
(int) ((waiting_time / 60) - |
| 1313 |
((waiting_time / 3600) * 60)), |
| 1314 |
(int) (waiting_time - ((waiting_time / 60) * 60))); |
| 1315 |
} |
| 1316 |
to_warning_next = get_cycles() + to_warning_interval; |
| 1317 |
} |
| 1318 |
|
| 1319 |
if (get_cycles() > ipsaddr->s_timeout) { |
| 1320 |
if (!ipsaddr->credit && connect_credits) { |
| 1321 |
ipsaddr->credit = 1; |
| 1322 |
connect_credits--; |
| 1323 |
} |
| 1324 |
if (ipsaddr->credit) { |
| 1325 |
_IPATH_VDBG("Connect req to %u:%u:%u\n", |
| 1326 |
__be16_to_cpu(ipsaddr->epr.epr_base_lid), |
| 1327 |
ipsaddr->epr.epr_context, |
| 1328 |
ipsaddr->epr.epr_subcontext); |
| 1329 |
if (ips_proto_send_ctrl_message(&ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO], |
| 1330 |
OPCODE_CONNECT_REQUEST, |
| 1331 |
&ipsaddr->ctrl_msg_queued, |
| 1332 |
buf) == PSM_OK) { |
| 1333 |
keep_polling = 0; |
| 1334 |
ipsaddr->delay_in_ms = |
| 1335 |
min(100, ipsaddr->delay_in_ms << 1); |
| 1336 |
ipsaddr->s_timeout = get_cycles() + |
| 1337 |
nanosecs_to_cycles(ipsaddr->delay_in_ms * MSEC_ULL); |
| 1338 |
} |
| 1339 |
/* If not, send got "busy", keep trying */ |
| 1340 |
} |
| 1341 |
else { |
| 1342 |
keep_polling = 0; |
| 1343 |
} |
| 1344 |
} |
| 1345 |
|
| 1346 |
if ((err = psmi_err_only(psmi_poll_internal(proto->ep, 1)))) |
| 1347 |
goto fail; |
| 1348 |
|
| 1349 |
if (ipsaddr->cstate_to == CSTATE_ESTABLISHED) { |
| 1350 |
/* This is not the real error code, we only set OK here |
| 1351 |
* so we know to stop polling for the reply. The actual |
| 1352 |
* error is in ipsaddr->cerror_to */ |
| 1353 |
array_of_errors[i] = PSM_OK; |
| 1354 |
numep_left--; |
| 1355 |
connect_credits++; |
| 1356 |
ipsaddr->credit = 0; |
| 1357 |
break; |
| 1358 |
} |
| 1359 |
} |
| 1360 |
} |
| 1361 |
} |
| 1362 |
|
| 1363 |
err_timeout: |
| 1364 |
/* Find the worst error to report */ |
| 1365 |
for (i = 0; i < numep; i++) { |
| 1366 |
if (!array_of_epid_mask[i]) |
| 1367 |
continue; |
| 1368 |
switch (array_of_errors[i]) { |
| 1369 |
/* These are benign */ |
| 1370 |
case PSM_EPID_UNREACHABLE: |
| 1371 |
case PSM_EPID_ALREADY_CONNECTED: |
| 1372 |
break; |
| 1373 |
case PSM_EPID_UNKNOWN: |
| 1374 |
array_of_errors[i] = PSM_TIMEOUT; |
| 1375 |
err = psmi_error_cmp(err, PSM_TIMEOUT); |
| 1376 |
break; |
| 1377 |
case PSM_OK: |
| 1378 |
/* Restore the real connect error */ |
| 1379 |
ipsaddr = array_of_epaddr[i]->ptladdr; |
| 1380 |
array_of_errors[i] = ipsaddr->cerror_to; |
| 1381 |
psmi_assert_always( |
| 1382 |
array_of_epaddr[i]->ptladdr->cstate_to == CSTATE_ESTABLISHED); |
| 1383 |
if (ipsaddr->cerror_to != PSM_OK) { |
| 1384 |
err = psmi_error_cmp(err, ipsaddr->cerror_to); |
| 1385 |
ips_free_epaddr(array_of_epaddr[i]->ptladdr); |
| 1386 |
array_of_epaddr[i] = NULL; |
| 1387 |
} |
| 1388 |
else { |
| 1389 |
proto->num_connected_to++; |
| 1390 |
psmi_assert_always(ipsaddr->epr.epr_path[IPS_PATH_HIGH_PRIORITY][0]->epr_mtu > 0); |
| 1391 |
} |
| 1392 |
break; |
| 1393 |
default: |
| 1394 |
break; |
| 1395 |
} |
| 1396 |
} |
| 1397 |
|
| 1398 |
fail: |
| 1399 |
return err; |
| 1400 |
} |
| 1401 |
|
| 1402 |
/* Repercutions on MQ. |
| 1403 |
* |
| 1404 |
* If num_connected==0, everything that exists in the posted queue should |
| 1405 |
* complete and the error must be marked epid_was_closed. |
| 1406 |
* |
| 1407 |
*/ |
| 1408 |
|
| 1409 |
psm_error_t |
| 1410 |
ips_proto_disconnect(struct ips_proto *proto, int force, int numep, |
| 1411 |
const psm_epaddr_t array_of_epaddr[], |
| 1412 |
const int array_of_epaddr_mask[], |
| 1413 |
psm_error_t array_of_errors[], |
| 1414 |
uint64_t timeout_in) |
| 1415 |
{ |
| 1416 |
ips_epaddr_t *ipsaddr; |
| 1417 |
int numep_left, numep_todisc, i, n; |
| 1418 |
int n_first; |
| 1419 |
int cstate; |
| 1420 |
int has_pending; |
| 1421 |
uint64_t timeout; |
| 1422 |
psm_error_t err = PSM_OK; |
| 1423 |
char buf[IPS_MAX_CONNECT_PAYLEN] PSMI_CACHEALIGN; |
| 1424 |
uint64_t reqs_sent = 0; |
| 1425 |
union psmi_envvar_val credits_intval; |
| 1426 |
int disconnect_credits; |
| 1427 |
uint64_t t_warning, t_start; |
| 1428 |
union psmi_envvar_val warn_intval; |
| 1429 |
unsigned warning_secs; |
| 1430 |
|
| 1431 |
psmi_assert_always(numep > 0); |
| 1432 |
|
| 1433 |
psmi_getenv("PSM_DISCONNECT_CREDITS", |
| 1434 |
"End-point disconnect request credits.", |
| 1435 |
PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_UINT, |
| 1436 |
(union psmi_envvar_val) 100, |
| 1437 |
&credits_intval); |
| 1438 |
|
| 1439 |
disconnect_credits = credits_intval.e_uint; |
| 1440 |
|
| 1441 |
/* Setup warning interval */ |
| 1442 |
psmi_getenv("PSM_DISCONNECT_WARN_INTERVAL", |
| 1443 |
"Period in seconds to warn if disconnections are not completed." |
| 1444 |
"Default is 300 seconds, 0 to disable.", |
| 1445 |
PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT, |
| 1446 |
(union psmi_envvar_val) 300, |
| 1447 |
&warn_intval); |
| 1448 |
|
| 1449 |
warning_secs = warn_intval.e_uint; |
| 1450 |
|
| 1451 |
PSMI_PLOCK_ASSERT(); |
| 1452 |
|
| 1453 |
/* First pass: see what to disconnect and what is disconnectable */ |
| 1454 |
for (i = 0, numep_todisc = 0; i < numep; i++) { |
| 1455 |
if (!array_of_epaddr_mask[i]) |
| 1456 |
continue; |
| 1457 |
psmi_assert_always(array_of_epaddr[i]->ptl == proto->ptl); |
| 1458 |
cstate = array_of_epaddr[i]->ptladdr->cstate_to; |
| 1459 |
array_of_epaddr[i]->ptladdr->credit = 0; |
| 1460 |
if (cstate == CSTATE_NONE) { |
| 1461 |
array_of_errors[i] = PSM_OK; |
| 1462 |
continue; |
| 1463 |
} |
| 1464 |
else { |
| 1465 |
psmi_assert_always(cstate == CSTATE_ESTABLISHED); |
| 1466 |
} |
| 1467 |
_IPATH_VDBG("disconnecting %p\n", array_of_epaddr[i]); |
| 1468 |
array_of_errors[i] = PSM_EPID_UNKNOWN; |
| 1469 |
numep_todisc++; |
| 1470 |
} |
| 1471 |
if (numep_todisc == 0) |
| 1472 |
goto success; |
| 1473 |
|
| 1474 |
/* Wait for everyone to ack previous packets before putting */ |
| 1475 |
if (timeout_in == 0) |
| 1476 |
timeout = ~0ULL; |
| 1477 |
else |
| 1478 |
timeout = get_cycles() + nanosecs_to_cycles(timeout_in); |
| 1479 |
|
| 1480 |
t_start = get_cycles(); |
| 1481 |
t_warning = t_start + nanosecs_to_cycles(warning_secs * SEC_ULL); |
| 1482 |
|
| 1483 |
n_first = ((uint32_t) get_cycles()) % numep; |
| 1484 |
if (!force) { |
| 1485 |
numep_left = numep_todisc; |
| 1486 |
do { |
| 1487 |
for (n = 0; n < numep; n++) { |
| 1488 |
i = (n_first + n) % numep; |
| 1489 |
if (!array_of_epaddr_mask[i] || array_of_errors[i] == PSM_OK) |
| 1490 |
continue; |
| 1491 |
ipsaddr = array_of_epaddr[i]->ptladdr; |
| 1492 |
switch (ipsaddr->cstate_to) { |
| 1493 |
case CSTATE_TO_DISCONNECTED: |
| 1494 |
array_of_errors[i] = PSM_OK; |
| 1495 |
numep_left--; |
| 1496 |
disconnect_credits++; |
| 1497 |
ipsaddr->credit = 0; |
| 1498 |
continue; |
| 1499 |
case CSTATE_TO_WAITING_DISC: |
| 1500 |
if (ipsaddr->s_timeout > get_cycles()) |
| 1501 |
continue; |
| 1502 |
ipsaddr->delay_in_ms = |
| 1503 |
min(100, ipsaddr->delay_in_ms << 1); |
| 1504 |
ipsaddr->s_timeout = get_cycles() + |
| 1505 |
nanosecs_to_cycles(ipsaddr->delay_in_ms*MSEC_ULL); |
| 1506 |
ips_proto_send_ctrl_message_request(proto, &ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO], |
| 1507 |
OPCODE_DISCONNECT_REQUEST, |
| 1508 |
&ipsaddr->ctrl_msg_queued, |
| 1509 |
buf, timeout); |
| 1510 |
reqs_sent++; |
| 1511 |
break; |
| 1512 |
case CSTATE_ESTABLISHED: |
| 1513 |
/* Still pending acks, hold off for now */ |
| 1514 |
ips_ptladdr_lock(ipsaddr); |
| 1515 |
has_pending = |
| 1516 |
!STAILQ_EMPTY(&ipsaddr-> |
| 1517 |
flows[EP_FLOW_GO_BACK_N_PIO].scb_unacked) || |
| 1518 |
!STAILQ_EMPTY(&ipsaddr-> |
| 1519 |
flows[EP_FLOW_GO_BACK_N_DMA].scb_unacked) || |
| 1520 |
!STAILQ_EMPTY(&ipsaddr-> |
| 1521 |
flows[EP_FLOW_GO_BACK_N_AM_REQ].scb_unacked) || |
| 1522 |
!STAILQ_EMPTY(&ipsaddr-> |
| 1523 |
flows[EP_FLOW_GO_BACK_N_AM_RSP].scb_unacked); |
| 1524 |
ips_ptladdr_unlock(ipsaddr); |
| 1525 |
if (has_pending) |
| 1526 |
continue; |
| 1527 |
if (!ipsaddr->credit && disconnect_credits) { |
| 1528 |
ipsaddr->credit = 1; |
| 1529 |
disconnect_credits--; |
| 1530 |
} |
| 1531 |
if (!ipsaddr->credit) |
| 1532 |
continue; |
| 1533 |
ipsaddr->delay_in_ms = 1; |
| 1534 |
ipsaddr->cstate_to = CSTATE_TO_WAITING_DISC; |
| 1535 |
ipsaddr->s_timeout = get_cycles() + |
| 1536 |
nanosecs_to_cycles(MSEC_ULL); |
| 1537 |
ips_proto_send_ctrl_message_request(proto, &ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO], |
| 1538 |
OPCODE_DISCONNECT_REQUEST, |
| 1539 |
&ipsaddr->ctrl_msg_queued, |
| 1540 |
buf, timeout); |
| 1541 |
reqs_sent++; |
| 1542 |
break; |
| 1543 |
default: |
| 1544 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 1545 |
"Unhandled/unknown close state %d", |
| 1546 |
ipsaddr->cstate_to); |
| 1547 |
break; |
| 1548 |
} |
| 1549 |
} |
| 1550 |
if (numep_left == 0) |
| 1551 |
break; |
| 1552 |
|
| 1553 |
if ((err = psmi_err_only(psmi_poll_internal(proto->ep, 1)))) |
| 1554 |
goto fail; |
| 1555 |
|
| 1556 |
if (warning_secs && get_cycles() > t_warning) { |
| 1557 |
_IPATH_INFO("graceful close in progress for %d/%d peers " |
| 1558 |
"(elapsed=%d millisecs,timeout=%d millisecs,reqs=%lld)\n", numep_left, numep_todisc, |
| 1559 |
(int) (cycles_to_nanosecs(get_cycles() - t_start) / MSEC_ULL), |
| 1560 |
(int) (timeout_in / MSEC_ULL), |
| 1561 |
(unsigned long long) reqs_sent); |
| 1562 |
t_warning = get_cycles() + nanosecs_to_cycles(warning_secs * SEC_ULL); |
| 1563 |
} |
| 1564 |
} |
| 1565 |
while (timeout > get_cycles()); |
| 1566 |
|
| 1567 |
if (numep_left > 0) { |
| 1568 |
err = PSM_TIMEOUT; |
| 1569 |
for (i = 0; i < numep; i++) { |
| 1570 |
if (!array_of_epaddr_mask[i]) |
| 1571 |
continue; |
| 1572 |
if (array_of_errors[i] == PSM_EPID_UNKNOWN) { |
| 1573 |
array_of_errors[i] = PSM_TIMEOUT; |
| 1574 |
_IPATH_VDBG("disc timeout on index %d, epaddr %s\n", |
| 1575 |
i, psmi_epaddr_get_name(array_of_epaddr[i]->epid)); |
| 1576 |
} |
| 1577 |
} |
| 1578 |
_IPATH_PRDBG("graceful close incomplete for %d/%d peers " |
| 1579 |
"(elapsed=%d millisecs,timeout=%d millisecs,reqs=%lld)\n", numep_left, numep_todisc, |
| 1580 |
(int) (cycles_to_nanosecs(get_cycles() - t_start) / MSEC_ULL), |
| 1581 |
(int) (timeout_in / MSEC_ULL), |
| 1582 |
(unsigned long long) reqs_sent); |
| 1583 |
} |
| 1584 |
else |
| 1585 |
_IPATH_PRDBG("graceful close complete from %d peers in %d millisecs, reqs_sent=%lld\n", |
| 1586 |
numep_todisc, |
| 1587 |
(int) (cycles_to_nanosecs(get_cycles() - t_start) / MSEC_ULL), |
| 1588 |
(unsigned long long) reqs_sent); |
| 1589 |
} else { |
| 1590 |
for (n = 0; n < numep; n++) { |
| 1591 |
i = (n_first + n) % numep; |
| 1592 |
if (!array_of_epaddr_mask[i]) |
| 1593 |
continue; |
| 1594 |
ipsaddr = array_of_epaddr[i]->ptladdr; |
| 1595 |
psmi_assert_always(ipsaddr->cstate_to == CSTATE_ESTABLISHED); |
| 1596 |
ips_proto_send_ctrl_message(&ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO], |
| 1597 |
OPCODE_DISCONNECT_REQUEST, |
| 1598 |
&ipsaddr->ctrl_msg_queued, |
| 1599 |
buf); |
| 1600 |
/* Force state to DISCONNECTED */ |
| 1601 |
ipsaddr->cstate_to = CSTATE_TO_DISCONNECTED; |
| 1602 |
array_of_errors[i] = PSM_OK; |
| 1603 |
} |
| 1604 |
_IPATH_VDBG("non-graceful close complete from %d peers\n", numep); |
| 1605 |
} |
| 1606 |
|
| 1607 |
for (i = 0; i < numep; i++) { |
| 1608 |
if (!array_of_epaddr_mask[i] || array_of_errors[i] != PSM_OK) |
| 1609 |
continue; |
| 1610 |
ipsaddr = array_of_epaddr[i]->ptladdr; |
| 1611 |
if (ipsaddr->cstate_to == CSTATE_NONE) |
| 1612 |
continue; |
| 1613 |
psmi_assert_always(ipsaddr->cstate_to == CSTATE_TO_DISCONNECTED); |
| 1614 |
proto->num_connected_to--; |
| 1615 |
/* Remote disconnect req arrived already, remove this epid. If it |
| 1616 |
* hasn't arrived yet, that's okay, we'll pick it up later and just |
| 1617 |
* mark our connect-to status as being "none". */ |
| 1618 |
if (ipsaddr->cstate_from == CSTATE_NONE) { |
| 1619 |
ips_free_epaddr(ipsaddr); |
| 1620 |
} |
| 1621 |
else |
| 1622 |
ipsaddr->cstate_to = CSTATE_NONE; |
| 1623 |
} |
| 1624 |
|
| 1625 |
fail: |
| 1626 |
success: |
| 1627 |
return err; |
| 1628 |
} |
| 1629 |
|
| 1630 |
int |
| 1631 |
ips_proto_isconnected(ips_epaddr_t *ipsaddr) |
| 1632 |
{ |
| 1633 |
if (ipsaddr->cstate_to == CSTATE_ESTABLISHED || |
| 1634 |
ipsaddr->cstate_from == CSTATE_ESTABLISHED) |
| 1635 |
return 1; |
| 1636 |
else |
| 1637 |
return 0; |
| 1638 |
} |
| 1639 |
|