| 1 |
/* |
| 2 |
* Copyright (c) 2013. Intel Corporation. All rights reserved. |
| 3 |
* Copyright (c) 2006-2012. QLogic Corporation. All rights reserved. |
| 4 |
* Copyright (c) 2003-2006, PathScale, Inc. All rights reserved. |
| 5 |
* |
| 6 |
* This software is available to you under a choice of one of two |
| 7 |
* licenses. You may choose to be licensed under the terms of the GNU |
| 8 |
* General Public License (GPL) Version 2, available from the file |
| 9 |
* COPYING in the main directory of this source tree, or the |
| 10 |
* OpenIB.org BSD license below: |
| 11 |
* |
| 12 |
* Redistribution and use in source and binary forms, with or |
| 13 |
* without modification, are permitted provided that the following |
| 14 |
* conditions are met: |
| 15 |
* |
| 16 |
* - Redistributions of source code must retain the above |
| 17 |
* copyright notice, this list of conditions and the following |
| 18 |
* disclaimer. |
| 19 |
* |
| 20 |
* - Redistributions in binary form must reproduce the above |
| 21 |
* copyright notice, this list of conditions and the following |
| 22 |
* disclaimer in the documentation and/or other materials |
| 23 |
* provided with the distribution. |
| 24 |
* |
| 25 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 26 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 27 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 28 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 29 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 30 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 31 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 32 |
* SOFTWARE. |
| 33 |
*/ |
| 34 |
|
| 35 |
/* This file implements the PSM PTL for ips */ |
| 36 |
#include "psm_user.h" |
| 37 |
#include "ptl_ips.h" |
| 38 |
#include "ipserror.h" |
| 39 |
|
| 40 |
int ips_ptl_recvq_isempty(const struct ptl *ptl); |
| 41 |
|
| 42 |
#define PSMI_CONTEXT_STATUS_CHECK_INTERVAL_MSECS 250 |
| 43 |
|
| 44 |
static |
| 45 |
int |
| 46 |
ips_subcontext_ignore(const struct ips_recvhdrq_event *rcv_ev, uint32_t subcontext) |
| 47 |
{ |
| 48 |
return IPS_RECVHDRQ_CONTINUE; |
| 49 |
} |
| 50 |
|
| 51 |
static |
| 52 |
int |
| 53 |
ips_subcontext_process(const struct ips_recvhdrq_event *rcv_ev, uint32_t subcontext) |
| 54 |
{ |
| 55 |
struct ptl_shared *recvshc = rcv_ev->proto->ptl->recvshc; |
| 56 |
if_pt (subcontext != recvshc->subcontext && |
| 57 |
subcontext < recvshc->subcontext_cnt) { |
| 58 |
return ips_writehdrq_append(&recvshc->writeq[subcontext], rcv_ev); |
| 59 |
} |
| 60 |
else { |
| 61 |
_IPATH_VDBG("Drop pkt for subcontext %d out of %d (I am %d) : errors 0x%x\n", |
| 62 |
(int) subcontext, (int) recvshc->subcontext_cnt, |
| 63 |
(int) recvshc->subcontext, (unsigned) rcv_ev->error_flags); |
| 64 |
return IPS_RECVHDRQ_BREAK; |
| 65 |
} |
| 66 |
} |
| 67 |
|
| 68 |
static |
| 69 |
void |
| 70 |
recvhdrq_hw_params(const psmi_context_t *context, |
| 71 |
struct ips_recvq_params *hdrq, |
| 72 |
struct ips_recvq_params *egrq, |
| 73 |
int is_shared_context, int subcontext) |
| 74 |
{ |
| 75 |
const struct ipath_base_info *base_info = &context->base_info; |
| 76 |
|
| 77 |
hdrq->elemcnt = base_info->spi_rcvhdr_cnt; |
| 78 |
hdrq->elemsz = base_info->spi_rcvhdrent_size; |
| 79 |
|
| 80 |
egrq->elemsz = base_info->spi_rcv_egrbufsize; /* bytes */ |
| 81 |
egrq->elemcnt = base_info->spi_tidegrcnt; /* words */ |
| 82 |
|
| 83 |
if (!is_shared_context) { |
| 84 |
volatile uint64_t *uregbase = /* HW registers */ |
| 85 |
(volatile uint64_t *) (uintptr_t) base_info->spi_uregbase; |
| 86 |
hdrq->base_addr = (uint32_t *)(uintptr_t) base_info->spi_rcvhdr_base; |
| 87 |
hdrq->head_register = (volatile __le32 *) &uregbase[ur_rcvhdrhead]; |
| 88 |
hdrq->tail_register = (volatile __le32 *) (uintptr_t) |
| 89 |
base_info->spi_rcvhdr_tailaddr; |
| 90 |
egrq->base_addr = (void *) (uintptr_t) base_info->spi_rcv_egrbufs; |
| 91 |
egrq->head_register = (volatile __le32 *) |
| 92 |
&uregbase[ur_rcvegrindexhead]; |
| 93 |
egrq->tail_register = (volatile __le32 *) |
| 94 |
&uregbase[ur_rcvegrindextail]; |
| 95 |
} |
| 96 |
else { |
| 97 |
/* Subcontexts mimic the HW registers but use different addresses |
| 98 |
* to avoid cache contention. */ |
| 99 |
volatile uint64_t *subcontext_uregbase; |
| 100 |
uint32_t *rcv_hdr; |
| 101 |
void *rcv_egr; |
| 102 |
unsigned pagesize = getpagesize(); |
| 103 |
unsigned hdrsize, egrsize; |
| 104 |
unsigned i = pagesize - 1; |
| 105 |
hdrsize = (base_info->spi_rcvhdr_cnt * sizeof(uint32_t) * |
| 106 |
base_info->spi_rcvhdrent_size + i) & ~i; |
| 107 |
egrsize = base_info->spi_rcv_egrbuftotlen; |
| 108 |
subcontext_uregbase = (uint64_t *) |
| 109 |
(((uintptr_t) base_info->spi_subctxt_uregbase) + |
| 110 |
(pagesize * subcontext)); |
| 111 |
rcv_hdr = (uint32_t *) |
| 112 |
(((uintptr_t) base_info->spi_subctxt_rcvhdr_base + |
| 113 |
(hdrsize * subcontext))); |
| 114 |
rcv_egr = (void *) |
| 115 |
(((uintptr_t) base_info->spi_subctxt_rcvegrbuf + |
| 116 |
(egrsize * subcontext))); |
| 117 |
hdrq->base_addr = (uint32_t *) rcv_hdr; |
| 118 |
hdrq->head_register = (volatile __le32 *) |
| 119 |
&subcontext_uregbase[ur_rcvhdrhead * 8]; |
| 120 |
hdrq->tail_register = (volatile __le32 *) (uintptr_t) |
| 121 |
&subcontext_uregbase[ur_rcvhdrtail * 8]; |
| 122 |
egrq->base_addr = rcv_egr; |
| 123 |
egrq->head_register = (volatile __le32 *) |
| 124 |
&subcontext_uregbase[ur_rcvegrindexhead * 8]; |
| 125 |
egrq->tail_register = (volatile __le32 *) |
| 126 |
&subcontext_uregbase[ur_rcvegrindextail * 8]; |
| 127 |
} |
| 128 |
} |
| 129 |
|
| 130 |
static psm_error_t shrecvq_init(ptl_t *ptl, const psmi_context_t *context); |
| 131 |
static psm_error_t shrecvq_fini(ptl_t *ptl); |
| 132 |
|
| 133 |
static |
| 134 |
size_t |
| 135 |
ips_ptl_sizeof(void) |
| 136 |
{ |
| 137 |
return sizeof(ptl_t); |
| 138 |
} |
| 139 |
|
| 140 |
static |
| 141 |
int |
| 142 |
ips_ptl_epaddr_stats_num(void) |
| 143 |
{ |
| 144 |
return sizeof(struct ptl_epaddr_stats) / sizeof (uint64_t); |
| 145 |
} |
| 146 |
|
| 147 |
static |
| 148 |
int |
| 149 |
ips_ptl_epaddr_stats_init(char **desc, uint16_t *flags) |
| 150 |
{ |
| 151 |
int num_stats = sizeof(struct ptl_epaddr_stats) / sizeof (uint64_t); |
| 152 |
int i; |
| 153 |
|
| 154 |
/* All stats are uint64_t */ |
| 155 |
for (i = 0; i < num_stats; i++) |
| 156 |
flags[i] = MPSPAWN_STATS_REDUCTION_ALL | |
| 157 |
MPSPAWN_STATS_SKIP_IF_ZERO; |
| 158 |
|
| 159 |
desc[0] = "errchecks sent"; |
| 160 |
desc[1] = "errchecks recv"; |
| 161 |
desc[2] = "naks sent"; |
| 162 |
desc[3] = "naks recv"; |
| 163 |
desc[4] = "connect reqs sent"; |
| 164 |
desc[5] = "disconnect reqs sent"; |
| 165 |
desc[6] = "tid grants sent"; |
| 166 |
desc[7] = "tid grants recv"; |
| 167 |
desc[8] = "send rexmit"; |
| 168 |
desc[9] = "congestion packets"; |
| 169 |
|
| 170 |
return num_stats; |
| 171 |
} |
| 172 |
|
| 173 |
int |
| 174 |
ips_ptl_epaddr_stats_get(psm_epaddr_t epaddr, uint64_t *stats_o) |
| 175 |
{ |
| 176 |
struct ptl_epaddr *ipsaddr = epaddr->ptladdr; |
| 177 |
int i, num_stats = sizeof(struct ptl_epaddr_stats) / sizeof (uint64_t); |
| 178 |
uint64_t *stats_i = (uint64_t *) &ipsaddr->stats; |
| 179 |
|
| 180 |
for (i = 0; i < num_stats; i++) |
| 181 |
stats_o[i] = stats_i[i]; |
| 182 |
|
| 183 |
return num_stats; |
| 184 |
} |
| 185 |
|
| 186 |
static psm_error_t |
| 187 |
psmi_context_check_status_callback(struct psmi_timer *t, uint64_t current) |
| 188 |
{ |
| 189 |
struct ptl *ptl = (struct ptl *) t->context; |
| 190 |
const uint64_t current_count = get_cycles(); |
| 191 |
psm_error_t err; |
| 192 |
|
| 193 |
err = psmi_context_check_status(ptl->context); |
| 194 |
psmi_timer_request_always(&ptl->timerq, &ptl->status_timer, |
| 195 |
current_count + ptl->status_cyc_timeout); |
| 196 |
|
| 197 |
return err; |
| 198 |
} |
| 199 |
|
| 200 |
static |
| 201 |
psm_error_t |
| 202 |
ips_ptl_init(const psm_ep_t ep, ptl_t *ptl, ptl_ctl_t *ctl) |
| 203 |
{ |
| 204 |
psm_error_t err = PSM_OK; |
| 205 |
uint32_t num_of_send_bufs = ep->ipath_num_sendbufs; |
| 206 |
uint32_t num_of_send_desc = ep->ipath_num_descriptors; |
| 207 |
uint32_t imm_size = ep->ipath_imm_size; |
| 208 |
const psmi_context_t *context = &ep->context; |
| 209 |
const struct ipath_user_info *user_info = &context->user_info; |
| 210 |
const int enable_shcontexts = (user_info->spu_subcontext_cnt > 0); |
| 211 |
const uint64_t current_count = get_cycles(); |
| 212 |
|
| 213 |
/* Preconditions */ |
| 214 |
psmi_assert_always(ep != NULL); |
| 215 |
psmi_assert_always(ep->epaddr != NULL); |
| 216 |
psmi_assert_always(ep->epid != 0); |
| 217 |
psmi_assert_always(ep->ipath_num_sendbufs > 0); |
| 218 |
|
| 219 |
memset(ptl, 0, sizeof(struct ptl)); |
| 220 |
|
| 221 |
ptl->ep = ep; /* back pointer */ |
| 222 |
ptl->epid = ep->epid; /* cache epid */ |
| 223 |
ptl->epaddr = ep->epaddr; /* cache a copy */ |
| 224 |
ptl->ctl = ctl; |
| 225 |
ptl->context = context; |
| 226 |
ptl->runtime_flags = context->runtime_flags; |
| 227 |
|
| 228 |
memset(ctl, 0, sizeof(*ctl)); |
| 229 |
/* Fill in the control structure */ |
| 230 |
ctl->ptl = ptl; |
| 231 |
ctl->ep_poll = enable_shcontexts ? ips_ptl_shared_poll : ips_ptl_poll; |
| 232 |
ctl->ep_connect = ips_ptl_connect; |
| 233 |
ctl->ep_disconnect = ips_ptl_disconnect; |
| 234 |
ctl->mq_send = ips_proto_mq_send; |
| 235 |
ctl->mq_isend = ips_proto_mq_isend; |
| 236 |
|
| 237 |
ctl->am_short_request = ips_am_short_request; |
| 238 |
ctl->am_short_reply = ips_am_short_reply; |
| 239 |
|
| 240 |
ctl->epaddr_stats_num = ips_ptl_epaddr_stats_num; |
| 241 |
ctl->epaddr_stats_init = ips_ptl_epaddr_stats_init; |
| 242 |
ctl->epaddr_stats_get = ips_ptl_epaddr_stats_get; |
| 243 |
|
| 244 |
/* |
| 245 |
* Runtime flags in 'ptl' are different from runtime flags in 'context'. |
| 246 |
* In 'context', runtime flags reflect what the driver is capable of. |
| 247 |
* In 'ptl', runtime flags reflect the features we can or want to use in |
| 248 |
* the driver's supported runtime flags. |
| 249 |
*/ |
| 250 |
|
| 251 |
/* |
| 252 |
* This timer is to be used to check the context's status at every |
| 253 |
* PSMI_CONTEXT_STATUS_CHECK_INTERVAL_MSECS. This is useful to detect when |
| 254 |
* the link transitions from the DOWN state to the UP state. We can thus |
| 255 |
* stop aggregating link failure messages once we detect that the link is |
| 256 |
* up. |
| 257 |
*/ |
| 258 |
psmi_timer_entry_init(&ptl->status_timer, |
| 259 |
psmi_context_check_status_callback, ptl); |
| 260 |
|
| 261 |
/* cache the context's status timeout in cycles */ |
| 262 |
ptl->status_cyc_timeout = |
| 263 |
ms_2_cycles(PSMI_CONTEXT_STATUS_CHECK_INTERVAL_MSECS); |
| 264 |
|
| 265 |
/* |
| 266 |
* Retransmissions and pending operations are kept in a timer structure |
| 267 |
* (queue). The timerq is shared to various internal IPS interfaces so |
| 268 |
* that they too may schedule events on the timer queue. The timerq is |
| 269 |
* drained in the progress function. |
| 270 |
*/ |
| 271 |
if ((err = psmi_timer_init(&ptl->timerq))) |
| 272 |
goto fail; |
| 273 |
|
| 274 |
/* start the context's status timer */ |
| 275 |
psmi_timer_request_always(&ptl->timerq, &ptl->status_timer, |
| 276 |
current_count + ptl->status_cyc_timeout); |
| 277 |
|
| 278 |
/* |
| 279 |
* Hardware send pio used by eager and control messages. |
| 280 |
*/ |
| 281 |
if ((err = ips_spio_init(context, ptl, &ptl->spioc))) |
| 282 |
goto fail; |
| 283 |
|
| 284 |
/* |
| 285 |
* Epstate maps endpoint ids (epid integers) to ipsaddr (structs). Mappings |
| 286 |
* are added/removed by the connect portion of the ips protocol and lookup |
| 287 |
* is made by the receive queue processing component. |
| 288 |
*/ |
| 289 |
if ((err = ips_epstate_init(&ptl->epstate, context))) |
| 290 |
goto fail; |
| 291 |
|
| 292 |
/* |
| 293 |
* Actual ips protocol handling. |
| 294 |
*/ |
| 295 |
if ((err = ips_proto_init(context, ptl, num_of_send_bufs, num_of_send_desc, |
| 296 |
imm_size, &ptl->timerq, &ptl->epstate, |
| 297 |
&ptl->spioc, &ptl->proto))) |
| 298 |
goto fail; |
| 299 |
|
| 300 |
/* |
| 301 |
* Hardware receive hdr/egr queue, services incoming packets and issues |
| 302 |
* callbacks for protocol handling in proto_recv. It uses the epstate |
| 303 |
* interface to determine if a packet is known or unknown. |
| 304 |
*/ |
| 305 |
if (!enable_shcontexts) { |
| 306 |
struct ips_recvhdrq_callbacks recvq_callbacks; |
| 307 |
struct ips_recvq_params hdrq, egrq; |
| 308 |
recvhdrq_hw_params(context, &hdrq, &egrq, 0, 0); |
| 309 |
recvq_callbacks.callback_packet_unknown = ips_proto_process_unknown; |
| 310 |
recvq_callbacks.callback_subcontext = ips_subcontext_ignore; |
| 311 |
recvq_callbacks.callback_error = ips_proto_process_packet_error; |
| 312 |
if ((err = ips_recvhdrq_init(context, &ptl->epstate, &ptl->proto, |
| 313 |
&hdrq, &egrq, &recvq_callbacks, |
| 314 |
ptl->runtime_flags, 0, |
| 315 |
&ptl->recvq, &ptl->recvq_state))) |
| 316 |
goto fail; |
| 317 |
} |
| 318 |
|
| 319 |
/* |
| 320 |
* Software receive hdr/egr queue, used in shared contexts. |
| 321 |
*/ |
| 322 |
if (enable_shcontexts && (err = shrecvq_init(ptl, context))) |
| 323 |
goto fail; |
| 324 |
|
| 325 |
/* |
| 326 |
* Receive thread, always initialized but not necessary creates a |
| 327 |
* pthread. |
| 328 |
*/ |
| 329 |
if ((err = ips_ptl_rcvthread_init(ptl, &ptl->recvq))) |
| 330 |
goto fail; |
| 331 |
fail: |
| 332 |
return err; |
| 333 |
} |
| 334 |
|
| 335 |
static |
| 336 |
psm_error_t |
| 337 |
ips_ptl_fini(ptl_t *ptl, int force, uint64_t timeout_in) |
| 338 |
{ |
| 339 |
const struct ipath_user_info *user_info = &ptl->context->user_info; |
| 340 |
const int enable_shcontexts = (user_info->spu_subcontext_cnt > 0); |
| 341 |
psm_error_t err = PSM_OK; |
| 342 |
|
| 343 |
if ((err = ips_proto_fini(&ptl->proto, force, timeout_in))) |
| 344 |
goto fail; |
| 345 |
|
| 346 |
/* We have to cancel the thread after terminating the protocol because |
| 347 |
* connect/disconnect packets use interrupts and the kernel doesn't |
| 348 |
* like to have no pollers waiting */ |
| 349 |
if ((err = ips_ptl_rcvthread_fini(ptl))) |
| 350 |
goto fail; |
| 351 |
|
| 352 |
if ((err = ips_epstate_fini(&ptl->epstate))) |
| 353 |
goto fail; |
| 354 |
|
| 355 |
if ((err = ips_spio_fini(&ptl->spioc))) |
| 356 |
goto fail; |
| 357 |
|
| 358 |
if ((err = psmi_timer_fini(&ptl->timerq))) |
| 359 |
goto fail; |
| 360 |
|
| 361 |
if (!enable_shcontexts && (err = ips_recvhdrq_fini(&ptl->recvq))) |
| 362 |
goto fail; |
| 363 |
|
| 364 |
if (enable_shcontexts && (err = shrecvq_fini(ptl))) |
| 365 |
goto fail; |
| 366 |
|
| 367 |
fail: |
| 368 |
return err; |
| 369 |
} |
| 370 |
|
| 371 |
static |
| 372 |
psm_error_t |
| 373 |
ips_ptl_optctl(const void *core_obj, int optname, |
| 374 |
void *optval, uint64_t *optlen, int get) |
| 375 |
{ |
| 376 |
psm_error_t err = PSM_OK; |
| 377 |
|
| 378 |
switch(optname) { |
| 379 |
case PSM_IB_OPT_EP_SL: |
| 380 |
{ |
| 381 |
/* Core object is psm_epaddr */ |
| 382 |
psm_epaddr_t epaddr = (psm_epaddr_t) core_obj; |
| 383 |
ips_epaddr_t *ipsaddr = epaddr->ptladdr; |
| 384 |
|
| 385 |
/* If endpoint does not use IB ignore for set, complain for get */ |
| 386 |
if (epaddr->ptlctl->ep_connect != ips_ptl_connect) { |
| 387 |
if (get) |
| 388 |
err = psmi_handle_error(PSMI_EP_LOGEVENT, |
| 389 |
PSM_PARAM_ERR, "Invalid EP transport"); |
| 390 |
goto exit_fn; |
| 391 |
} |
| 392 |
|
| 393 |
/* Sanity check option length */ |
| 394 |
if (*optlen < sizeof(uint8_t)) { |
| 395 |
err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR, |
| 396 |
"Option value length error"); |
| 397 |
*optlen = sizeof(unsigned); |
| 398 |
goto exit_fn; |
| 399 |
} |
| 400 |
|
| 401 |
if (get) { |
| 402 |
/* Get returns the SL for the PIO flow */ |
| 403 |
*((uint8_t *) optval) = |
| 404 |
(uint8_t) ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO].sl; |
| 405 |
} |
| 406 |
else { |
| 407 |
uint16_t new_sl; |
| 408 |
|
| 409 |
/* Sanity check if SL is within range */ |
| 410 |
new_sl = (uint16_t) *(uint8_t*) optval; |
| 411 |
if (new_sl > 15) { |
| 412 |
err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR, |
| 413 |
"Invalid SL value %u. 0 <= SL <= 15.",new_sl); |
| 414 |
goto exit_fn; |
| 415 |
} |
| 416 |
|
| 417 |
/* Set new SL for all flows */ |
| 418 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO].sl = new_sl; |
| 419 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_DMA].sl = new_sl; |
| 420 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_AM_REQ].sl = new_sl; |
| 421 |
ipsaddr->flows[EP_FLOW_GO_BACK_N_AM_RSP].sl = new_sl; |
| 422 |
} |
| 423 |
} |
| 424 |
break; |
| 425 |
case PSM_IB_OPT_DF_SL: |
| 426 |
{ |
| 427 |
/* Set default SL to be used by an endpoint for all communication */ |
| 428 |
/* Core object is psm_epaddr */ |
| 429 |
psm_ep_t ep = (psm_ep_t) core_obj; |
| 430 |
|
| 431 |
/* Make sure ep is specified */ |
| 432 |
if (!ep) { |
| 433 |
err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR, |
| 434 |
"Invalid PSM Endpoint"); |
| 435 |
goto exit_fn; |
| 436 |
} |
| 437 |
|
| 438 |
/* Sanity check option length */ |
| 439 |
if (*optlen < sizeof(uint8_t)) { |
| 440 |
err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR, |
| 441 |
"Option value length error"); |
| 442 |
*optlen = sizeof(uint8_t); |
| 443 |
goto exit_fn; |
| 444 |
} |
| 445 |
|
| 446 |
if (get) { |
| 447 |
*((uint8_t *) optval) = ep->ptl_ips.ptl->proto.epinfo.ep_sl; |
| 448 |
} |
| 449 |
else { |
| 450 |
uint16_t new_sl; |
| 451 |
|
| 452 |
/* Sanity check if SL is within range */ |
| 453 |
new_sl = (uint16_t) *(uint8_t*) optval; |
| 454 |
if (new_sl > 15) { |
| 455 |
err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR, |
| 456 |
"Invalid SL value %u. 0 <= SL <= 15.",new_sl); |
| 457 |
goto exit_fn; |
| 458 |
} |
| 459 |
|
| 460 |
ep->ptl_ips.ptl->proto.epinfo.ep_sl = (uint8_t) new_sl; |
| 461 |
} |
| 462 |
} |
| 463 |
break; |
| 464 |
default: |
| 465 |
err = psmi_handle_error(NULL, PSM_PARAM_ERR, "Unknown PSM_IB option %u.", optname); |
| 466 |
} |
| 467 |
|
| 468 |
exit_fn: |
| 469 |
return err; |
| 470 |
} |
| 471 |
|
| 472 |
static |
| 473 |
psm_error_t |
| 474 |
ips_ptl_setopt(const void *component_obj, int optname, |
| 475 |
const void *optval, uint64_t optlen) |
| 476 |
{ |
| 477 |
return ips_ptl_optctl(component_obj, optname, (void*) optval, &optlen, 0); |
| 478 |
} |
| 479 |
|
| 480 |
static |
| 481 |
psm_error_t |
| 482 |
ips_ptl_getopt(const void *component_obj, int optname, |
| 483 |
void *optval, uint64_t *optlen) |
| 484 |
{ |
| 485 |
return ips_ptl_optctl(component_obj, optname, optval, optlen, 1); |
| 486 |
} |
| 487 |
|
| 488 |
psm_error_t __recvpath |
| 489 |
ips_ptl_poll(ptl_t *ptl, int _ignored) |
| 490 |
{ |
| 491 |
const uint64_t current_count = get_cycles(); |
| 492 |
const int do_lock = PSMI_PLOCK_DISABLED && |
| 493 |
(ptl->runtime_flags & PSMI_RUNTIME_RCVTHREAD); |
| 494 |
psm_error_t err = PSM_OK_NO_PROGRESS; |
| 495 |
psm_error_t err2; |
| 496 |
|
| 497 |
if (!ips_recvhdrq_isempty(&ptl->recvq)) { |
| 498 |
if (do_lock && !ips_recvhdrq_trylock(&ptl->recvq)) |
| 499 |
return err; |
| 500 |
err = ips_recvhdrq_progress(&ptl->recvq); |
| 501 |
if (do_lock) |
| 502 |
ips_recvhdrq_unlock(&ptl->recvq); |
| 503 |
if_pf (err > PSM_OK_NO_PROGRESS) |
| 504 |
return err; |
| 505 |
err2 = psmi_timer_process_if_expired(&(ptl->timerq), current_count); |
| 506 |
if (err2 != PSM_OK_NO_PROGRESS) |
| 507 |
return err2; |
| 508 |
else |
| 509 |
return err; |
| 510 |
} |
| 511 |
|
| 512 |
/* |
| 513 |
* Process timer expirations after servicing receive queues (some packets |
| 514 |
* may have been acked, some requests-to-send may have been queued). |
| 515 |
* |
| 516 |
* It's safe to look at the timer without holding the lock because it's not |
| 517 |
* incorrect to be wrong some of the time. |
| 518 |
*/ |
| 519 |
if (psmi_timer_is_expired(&(ptl->timerq), current_count)) { |
| 520 |
if (do_lock) |
| 521 |
ips_recvhdrq_lock(&ptl->recvq); |
| 522 |
err = psmi_timer_process_expired(&(ptl->timerq), current_count); |
| 523 |
if (do_lock) |
| 524 |
ips_recvhdrq_unlock(&ptl->recvq); |
| 525 |
} |
| 526 |
|
| 527 |
return err; |
| 528 |
} |
| 529 |
|
| 530 |
PSMI_INLINE( |
| 531 |
int |
| 532 |
ips_try_lock_shared_context (struct ptl_shared *recvshc)) |
| 533 |
{ |
| 534 |
return pthread_spin_trylock(recvshc->context_lock); |
| 535 |
} |
| 536 |
|
| 537 |
PSMI_INLINE( |
| 538 |
void |
| 539 |
ips_lock_shared_context (struct ptl_shared *recvshc)) |
| 540 |
{ |
| 541 |
pthread_spin_lock(recvshc->context_lock); |
| 542 |
} |
| 543 |
|
| 544 |
PSMI_INLINE( |
| 545 |
void |
| 546 |
ips_unlock_shared_context (struct ptl_shared *recvshc)) |
| 547 |
{ |
| 548 |
pthread_spin_unlock(recvshc->context_lock); |
| 549 |
} |
| 550 |
|
| 551 |
psm_error_t __recvpath |
| 552 |
ips_ptl_shared_poll(ptl_t *ptl, int _ignored) |
| 553 |
{ |
| 554 |
const uint64_t current_count = get_cycles(); |
| 555 |
psm_error_t err = PSM_OK_NO_PROGRESS; |
| 556 |
psm_error_t err2; |
| 557 |
struct ptl_shared *recvshc = ptl->recvshc; |
| 558 |
psmi_assert(recvshc != NULL); |
| 559 |
|
| 560 |
/* The following header queue checks are speculative (but safe) |
| 561 |
* until this process has acquired the lock. The idea is to |
| 562 |
* minimize lock contention due to processes spinning on the |
| 563 |
* shared context. */ |
| 564 |
if (ips_recvhdrq_isempty(&recvshc->recvq)) { |
| 565 |
if (!ips_recvhdrq_isempty(&ptl->recvq) && |
| 566 |
ips_try_lock_shared_context(recvshc) == 0) { |
| 567 |
/* check that subcontext is empty while under lock to avoid |
| 568 |
* re-ordering of incoming packets (since packets from |
| 569 |
* hardware context will be processed immediately). */ |
| 570 |
if_pt (ips_recvhdrq_isempty(&recvshc->recvq)) { |
| 571 |
err = ips_recvhdrq_progress(&ptl->recvq); |
| 572 |
} |
| 573 |
ips_unlock_shared_context(recvshc); |
| 574 |
} |
| 575 |
} |
| 576 |
|
| 577 |
if_pf (err > PSM_OK_NO_PROGRESS) |
| 578 |
return err; |
| 579 |
|
| 580 |
if (!ips_recvhdrq_isempty(&recvshc->recvq)) { |
| 581 |
err2 = ips_recvhdrq_progress(&recvshc->recvq); |
| 582 |
if (err2 != PSM_OK_NO_PROGRESS) { |
| 583 |
err = err2; |
| 584 |
} |
| 585 |
} |
| 586 |
|
| 587 |
if_pf (err > PSM_OK_NO_PROGRESS) |
| 588 |
return err; |
| 589 |
|
| 590 |
/* |
| 591 |
* Process timer expirations after servicing receive queues (some packets |
| 592 |
* may have been acked, some requests-to-send may have been queued). |
| 593 |
*/ |
| 594 |
err2 = psmi_timer_process_if_expired(&(ptl->timerq), current_count); |
| 595 |
if (err2 != PSM_OK_NO_PROGRESS) |
| 596 |
err = err2; |
| 597 |
|
| 598 |
return err; |
| 599 |
} |
| 600 |
|
| 601 |
int __recvpath |
| 602 |
ips_ptl_recvq_isempty(const ptl_t *ptl) |
| 603 |
{ |
| 604 |
struct ptl_shared *recvshc = ptl->recvshc; |
| 605 |
|
| 606 |
if (recvshc != NULL && !ips_recvhdrq_isempty(&recvshc->recvq)) |
| 607 |
return 0; |
| 608 |
return ips_recvhdrq_isempty(&ptl->recvq); |
| 609 |
} |
| 610 |
|
| 611 |
/* |
| 612 |
* Legacy ips_get_stat -- do nothing. |
| 613 |
*/ |
| 614 |
int ips_get_stat(psm_epaddr_t epaddr, ips_sess_stat * stats) |
| 615 |
{ |
| 616 |
memset(stats, 0, sizeof (ips_sess_stat)); |
| 617 |
return 0; |
| 618 |
} |
| 619 |
|
| 620 |
static |
| 621 |
psm_error_t |
| 622 |
shrecvq_init(ptl_t *ptl, const psmi_context_t *context) |
| 623 |
{ |
| 624 |
const struct ipath_base_info *base_info = &context->base_info; |
| 625 |
const struct ipath_user_info *user_info = &context->user_info; |
| 626 |
struct ips_recvhdrq_callbacks recvq_callbacks; |
| 627 |
struct ips_recvq_params hdrq, egrq; |
| 628 |
psm_error_t err = PSM_OK; |
| 629 |
struct ptl_shared *recvshc; |
| 630 |
int i; |
| 631 |
|
| 632 |
psmi_assert_always(user_info->spu_subcontext_cnt > 0); |
| 633 |
|
| 634 |
recvshc = (struct ptl_shared *) |
| 635 |
psmi_calloc(context->ep, UNDEFINED, 1, sizeof(struct ptl_shared)); |
| 636 |
if (recvshc == NULL) { |
| 637 |
err = PSM_NO_MEMORY; |
| 638 |
goto fail; |
| 639 |
} |
| 640 |
|
| 641 |
ptl->recvshc = recvshc; |
| 642 |
recvshc->ptl = ptl; |
| 643 |
|
| 644 |
/* Initialize recvshc fields */ |
| 645 |
recvshc->subcontext = base_info->spi_subcontext; |
| 646 |
recvshc->subcontext_cnt = user_info->spu_subcontext_cnt; |
| 647 |
psmi_assert_always(recvshc->subcontext_cnt <= INFINIPATH_MAX_SUBCONTEXT); |
| 648 |
psmi_assert_always(recvshc->subcontext < recvshc->subcontext_cnt); |
| 649 |
|
| 650 |
if ((err = ips_subcontext_ureg_get(ptl, context, recvshc->subcontext_ureg, |
| 651 |
recvshc->subcontext_cnt))) |
| 652 |
goto fail; |
| 653 |
if ((err = ips_subcontext_ureg_initialize( |
| 654 |
ptl, recvshc->subcontext, recvshc->subcontext_ureg[recvshc->subcontext]))) |
| 655 |
goto fail; |
| 656 |
recvshc->context_lock = &recvshc->subcontext_ureg[0]->context_lock; |
| 657 |
|
| 658 |
/* Initialize (shared) hardware context recvq (ptl->recvq) */ |
| 659 |
/* NOTE: uses recvq in ptl structure for shared h/w context */ |
| 660 |
recvhdrq_hw_params(context, &hdrq, &egrq, 0, 0); |
| 661 |
recvq_callbacks.callback_packet_unknown = ips_proto_process_unknown; |
| 662 |
recvq_callbacks.callback_subcontext = ips_subcontext_process; |
| 663 |
recvq_callbacks.callback_error = ips_proto_process_packet_error; |
| 664 |
if ((err = ips_recvhdrq_init(context, &ptl->epstate, &ptl->proto, |
| 665 |
&hdrq, &egrq, &recvq_callbacks, |
| 666 |
ptl->runtime_flags, recvshc->subcontext, |
| 667 |
&ptl->recvq, |
| 668 |
&recvshc->subcontext_ureg[0]->recvq_state))) { |
| 669 |
goto fail; |
| 670 |
} |
| 671 |
|
| 672 |
/* Initialize software subcontext (recvshc->recvq). Subcontexts do */ |
| 673 |
/* not require the rcvhdr copy feature. */ |
| 674 |
recvhdrq_hw_params(context, &hdrq, &egrq, 1, recvshc->subcontext); |
| 675 |
recvq_callbacks.callback_subcontext = ips_subcontext_ignore; |
| 676 |
if ((err = ips_recvhdrq_init(context, &ptl->epstate, &ptl->proto, |
| 677 |
&hdrq, &egrq, &recvq_callbacks, |
| 678 |
ptl->runtime_flags & ~IPATH_RUNTIME_RCVHDR_COPY, |
| 679 |
recvshc->subcontext, |
| 680 |
&recvshc->recvq, |
| 681 |
&recvshc->recvq_state))) { |
| 682 |
goto fail; |
| 683 |
} |
| 684 |
|
| 685 |
/* Initialize each recvshc->writeq for shared contexts */ |
| 686 |
for (i = 0; i < recvshc->subcontext_cnt; i++) { |
| 687 |
recvhdrq_hw_params(context, &hdrq, &egrq, 1, i); |
| 688 |
if ((err = ips_writehdrq_init(context, &hdrq, &egrq, |
| 689 |
&recvshc->writeq[i], |
| 690 |
&recvshc->subcontext_ureg[i]->writeq_state, |
| 691 |
ptl->runtime_flags & ~IPATH_RUNTIME_RCVHDR_COPY))) { |
| 692 |
goto fail; |
| 693 |
} |
| 694 |
} |
| 695 |
|
| 696 |
if (err == PSM_OK) |
| 697 |
_IPATH_DBG("Context sharing in use: lid %d, context %d, sub-context %d\n", |
| 698 |
(int) psm_epid_nid(ptl->epid), base_info->spi_context, |
| 699 |
recvshc->subcontext); |
| 700 |
fail: |
| 701 |
return err; |
| 702 |
} |
| 703 |
|
| 704 |
static |
| 705 |
psm_error_t |
| 706 |
shrecvq_fini(ptl_t *ptl) |
| 707 |
{ |
| 708 |
psm_error_t err = PSM_OK; |
| 709 |
int i; |
| 710 |
|
| 711 |
/* disable my write header queue before deallocation */ |
| 712 |
i = ptl->recvshc->subcontext; |
| 713 |
ptl->recvshc->subcontext_ureg[i]->writeq_state.enabled = 0; |
| 714 |
|
| 715 |
if ((err = ips_recvhdrq_fini(&ptl->recvq))) |
| 716 |
goto fail; |
| 717 |
|
| 718 |
if ((err = ips_recvhdrq_fini(&ptl->recvshc->recvq))) |
| 719 |
goto fail; |
| 720 |
|
| 721 |
for (i = 0; i < ptl->recvshc->subcontext_cnt; i++) { |
| 722 |
if ((err = ips_writehdrq_fini(&ptl->recvshc->writeq[i]))) { |
| 723 |
goto fail; |
| 724 |
} |
| 725 |
} |
| 726 |
|
| 727 |
psmi_free(ptl->recvshc); |
| 728 |
|
| 729 |
fail: |
| 730 |
return err; |
| 731 |
} |
| 732 |
|
| 733 |
psm_error_t |
| 734 |
ips_ptl_connect(ptl_t *ptl, int numep, const psm_epid_t *array_of_epid, |
| 735 |
const int *array_of_epid_mask, psm_error_t *array_of_errors, |
| 736 |
psm_epaddr_t *array_of_epaddr, uint64_t timeout_in) |
| 737 |
{ |
| 738 |
psm_error_t err; |
| 739 |
psm_ep_t ep; |
| 740 |
psm_epid_t *epid_array = NULL; |
| 741 |
psm_error_t *error_array = NULL; |
| 742 |
psm_epaddr_t *epaddr_array = NULL; |
| 743 |
int *mask_array = NULL; |
| 744 |
int i, count; |
| 745 |
|
| 746 |
PSMI_PLOCK_ASSERT(); |
| 747 |
err = ips_proto_connect(&ptl->proto, numep, array_of_epid, |
| 748 |
array_of_epid_mask, array_of_errors, |
| 749 |
array_of_epaddr, timeout_in); |
| 750 |
if (err) return err; |
| 751 |
|
| 752 |
psmi_assert_always(ptl->ep->mctxt_master == ptl->ep); |
| 753 |
if (ptl->ep->mctxt_next == ptl->ep) return err; |
| 754 |
|
| 755 |
/* make the additional mutil-context connections. */ |
| 756 |
epid_array = (psm_epid_t *) |
| 757 |
psmi_malloc(ptl->ep, UNDEFINED, sizeof(psm_epid_t)*numep); |
| 758 |
mask_array = (int *) |
| 759 |
psmi_malloc(ptl->ep, UNDEFINED, sizeof(int)*numep); |
| 760 |
error_array = (psm_error_t *) |
| 761 |
psmi_malloc(ptl->ep, UNDEFINED, sizeof(psm_error_t)*numep); |
| 762 |
epaddr_array = (psm_epaddr_t *) |
| 763 |
psmi_malloc(ptl->ep, UNDEFINED, sizeof(psm_epaddr_t)*numep); |
| 764 |
if (!epid_array || !mask_array || !error_array || !epaddr_array) { |
| 765 |
goto fail; |
| 766 |
} |
| 767 |
|
| 768 |
count = 0; |
| 769 |
ep = ptl->ep->mctxt_next; |
| 770 |
while (ep != ep->mctxt_master) { |
| 771 |
|
| 772 |
/* Setup the mask array and epid array. */ |
| 773 |
for (i = 0; i < numep; i++) { |
| 774 |
if (array_of_epid_mask[i] |
| 775 |
&& array_of_errors[i] == PSM_OK |
| 776 |
&& count < array_of_epaddr[i]->mctxt_epcount) { |
| 777 |
if (ep->gid_hi != array_of_epaddr[i]->mctxt_gidhi[count]) { |
| 778 |
mask_array[i] = 0; |
| 779 |
_IPATH_INFO("Subnet ID mismatch, ignore...\n"); |
| 780 |
} else { |
| 781 |
mask_array[i] = 1; |
| 782 |
epid_array[i] = array_of_epaddr[i]->mctxt_epid[count]; |
| 783 |
} |
| 784 |
} else { |
| 785 |
mask_array[i] = 0; |
| 786 |
} |
| 787 |
} |
| 788 |
|
| 789 |
/* Make the real protocol connections. */ |
| 790 |
err = ips_proto_connect(&ep->ptl_ips.ptl->proto, numep, epid_array, |
| 791 |
mask_array, error_array, |
| 792 |
epaddr_array, timeout_in); |
| 793 |
if (err) goto fail; |
| 794 |
|
| 795 |
/* Make the epaddr linklist for this peer. */ |
| 796 |
for (i = 0; i < numep; i++) { |
| 797 |
if (!mask_array[i]) continue; |
| 798 |
|
| 799 |
/* In rare case, when the peer exits psm_ep_connect() |
| 800 |
* and sends a message, it is received by this epaddr, |
| 801 |
* because the epaddr->mctxt_master is still itself (linked |
| 802 |
* and changed by below macro), epaddr->mctxt_recv_seqnum |
| 803 |
* is increased, not the master's mctxt_recv_seqnum. |
| 804 |
* when this happens, we need to apply this mctxt_recv_seqnum |
| 805 |
* to master's mctxt_recv_seqnum, otherwise, the message |
| 806 |
* sequence number doesnot match master's mctxt_recv_seqnum, |
| 807 |
* and causes code hanging. |
| 808 |
* This case only happens in the last rail of multi-rail. |
| 809 |
*/ |
| 810 |
if (epaddr_array[i]->mctxt_recv_seqnum) { |
| 811 |
array_of_epaddr[i]->mctxt_recv_seqnum += |
| 812 |
epaddr_array[i]->mctxt_recv_seqnum; |
| 813 |
epaddr_array[i]->mctxt_recv_seqnum = 0; |
| 814 |
} |
| 815 |
|
| 816 |
PSM_MCTXT_APPEND(array_of_epaddr[i], epaddr_array[i]); |
| 817 |
|
| 818 |
/* randomize the rail to start traffic */ |
| 819 |
if ((random()%(count+2)) == 0) { |
| 820 |
array_of_epaddr[i]->mctxt_current = epaddr_array[i]; |
| 821 |
} |
| 822 |
|
| 823 |
/* Set the # slave connections so far */ |
| 824 |
array_of_epaddr[i]->mctxt_nsconn++; |
| 825 |
} |
| 826 |
|
| 827 |
count++; |
| 828 |
ep = ep->mctxt_next; |
| 829 |
} |
| 830 |
|
| 831 |
fail: |
| 832 |
if (epid_array) psmi_free(epid_array); |
| 833 |
if (mask_array) psmi_free(mask_array); |
| 834 |
if (error_array) psmi_free(error_array); |
| 835 |
if (epaddr_array) psmi_free(epaddr_array); |
| 836 |
|
| 837 |
return err; |
| 838 |
} |
| 839 |
|
| 840 |
psm_error_t |
| 841 |
ips_ptl_disconnect(ptl_t *ptl, int force, int numep, |
| 842 |
const psm_epaddr_t array_of_epaddr[], |
| 843 |
const int array_of_epaddr_mask[], |
| 844 |
psm_error_t array_of_errors[], uint64_t timeout_in) |
| 845 |
{ |
| 846 |
psm_error_t err; |
| 847 |
|
| 848 |
fprintf(stderr, "Aiee! ips_proto_disconnect() called.\n"); |
| 849 |
PSMI_PLOCK_ASSERT(); |
| 850 |
err = ips_proto_disconnect(&ptl->proto, force, numep, array_of_epaddr, |
| 851 |
array_of_epaddr_mask, array_of_errors, |
| 852 |
timeout_in); |
| 853 |
return err; |
| 854 |
} |
| 855 |
|
| 856 |
/* Only symbol we expose out of here */ |
| 857 |
struct ptl_ctl_init |
| 858 |
psmi_ptl_ips = { |
| 859 |
ips_ptl_sizeof, ips_ptl_init, ips_ptl_fini, ips_ptl_setopt, ips_ptl_getopt |
| 860 |
}; |