| 1 |
/* |
| 2 |
* Copyright (c) 2013. Intel Corporation. All rights reserved. |
| 3 |
* Copyright (c) 2006-2012. QLogic Corporation. All rights reserved. |
| 4 |
* Copyright (c) 2003-2006, PathScale, Inc. All rights reserved. |
| 5 |
* |
| 6 |
* This software is available to you under a choice of one of two |
| 7 |
* licenses. You may choose to be licensed under the terms of the GNU |
| 8 |
* General Public License (GPL) Version 2, available from the file |
| 9 |
* COPYING in the main directory of this source tree, or the |
| 10 |
* OpenIB.org BSD license below: |
| 11 |
* |
| 12 |
* Redistribution and use in source and binary forms, with or |
| 13 |
* without modification, are permitted provided that the following |
| 14 |
* conditions are met: |
| 15 |
* |
| 16 |
* - Redistributions of source code must retain the above |
| 17 |
* copyright notice, this list of conditions and the following |
| 18 |
* disclaimer. |
| 19 |
* |
| 20 |
* - Redistributions in binary form must reproduce the above |
| 21 |
* copyright notice, this list of conditions and the following |
| 22 |
* disclaimer in the documentation and/or other materials |
| 23 |
* provided with the distribution. |
| 24 |
* |
| 25 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 26 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 27 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 28 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 29 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 30 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 31 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 32 |
* SOFTWARE. |
| 33 |
*/ |
| 34 |
|
| 35 |
/* included header files */ |
| 36 |
#include <stdlib.h> |
| 37 |
#include <stddef.h> |
| 38 |
#include <stdint.h> |
| 39 |
#include <stdio.h> |
| 40 |
#include <sched.h> |
| 41 |
|
| 42 |
#include "ips_proto.h" |
| 43 |
#include "ips_proto_internal.h" |
| 44 |
#include "ips_spio.h" |
| 45 |
#include "ipserror.h" /* ips error codes */ |
| 46 |
#include "ips_proto_params.h" |
| 47 |
#include "ipath_byteorder.h" |
| 48 |
|
| 49 |
|
| 50 |
#define SPIO_INUSE_MASK 0xAAAAAAAAAAAAAAAAULL |
| 51 |
#define SPIO_CHECK_MASK 0x5555555555555555ULL |
| 52 |
|
| 53 |
/* Report PIO stalls every 20 seconds at the least */ |
| 54 |
#define SPIO_STALL_WARNING_INTERVAL (nanosecs_to_cycles(20e9)) |
| 55 |
#define SPIO_MAX_CONSECUTIVE_SEND_FAIL (1<<20) /* 1M */ |
| 56 |
/* RESYNC_CONSECUTIVE_SEND_FAIL has to be a multiple of MAX_CONSECUTIVE */ |
| 57 |
#define SPIO_RESYNC_CONSECUTIVE_SEND_FAIL (1<<4) /* 16 */ |
| 58 |
|
| 59 |
static void spio_report_stall(struct ips_spio *ctrl, |
| 60 |
uint64_t t_cyc_now, |
| 61 |
uint64_t send_failures); |
| 62 |
|
| 63 |
static void spio_handle_stall(struct ips_spio *ctrl, |
| 64 |
uint64_t send_failures); |
| 65 |
|
| 66 |
static inline |
| 67 |
uint64_t |
| 68 |
ips_spio_read_avail_index(struct ips_spio *ctrl, int index) |
| 69 |
{ |
| 70 |
if (ctrl->runtime_flags & IPATH_RUNTIME_PIO_REGSWAPPED && index > 3) { |
| 71 |
return __le64_to_cpu(ctrl->spio_avail_addr[index ^ 1]); |
| 72 |
} |
| 73 |
else |
| 74 |
return __le64_to_cpu(ctrl->spio_avail_addr[index]); |
| 75 |
} |
| 76 |
|
| 77 |
psm_error_t |
| 78 |
ips_spio_init(const struct psmi_context *context, const struct ptl *ptl, |
| 79 |
struct ips_spio *ctrl) |
| 80 |
{ |
| 81 |
psm_error_t err = PSM_OK; |
| 82 |
const struct ipath_base_info *base_info = &context->base_info; |
| 83 |
unsigned wc_unordered; |
| 84 |
char *order_str = "undefined"; |
| 85 |
int i, last_shadow_index; |
| 86 |
int num_shadow_index = sizeof(ctrl->spio_avail_shadow) / |
| 87 |
sizeof(ctrl->spio_avail_shadow[0]); |
| 88 |
|
| 89 |
ctrl->ptl = ptl; |
| 90 |
ctrl->context = context; |
| 91 |
/* Copy runtime flags */ |
| 92 |
ctrl->runtime_flags = ptl->runtime_flags; |
| 93 |
ctrl->unit_id = context->ep->unit_id; |
| 94 |
ctrl->portnum = context->ep->portnum; |
| 95 |
pthread_spin_init(&ctrl->spio_lock, 0); |
| 96 |
ctrl->spio_avail_addr = |
| 97 |
(__le64 *)(ptrdiff_t)base_info->spi_pioavailaddr; |
| 98 |
ctrl->spio_buffer_base = |
| 99 |
(uint32_t *)(ptrdiff_t)base_info->spi_piobufbase; |
| 100 |
ctrl->spio_sendbuf_status = |
| 101 |
(unsigned long *)(ptrdiff_t)base_info->spi_sendbuf_status; |
| 102 |
|
| 103 |
ctrl->spio_buffer_spacing = base_info->spi_pioalign >> 2; |
| 104 |
ctrl->spio_first_buffer = ctrl->spio_current_buffer = |
| 105 |
base_info->spi_pioindex; |
| 106 |
ctrl->spio_last_buffer = |
| 107 |
ctrl->spio_first_buffer + base_info->spi_piocnt - 1; |
| 108 |
ctrl->spio_num_of_buffer = base_info->spi_piocnt; |
| 109 |
|
| 110 |
ctrl->spio_consecutive_failures = 0; |
| 111 |
ctrl->spio_num_stall = 0ULL; |
| 112 |
ctrl->spio_next_stall_warning = 0ULL; |
| 113 |
ctrl->spio_last_stall_cyc = 0ULL; |
| 114 |
ctrl->spio_init_cyc = get_cycles(); |
| 115 |
|
| 116 |
last_shadow_index = ctrl->spio_last_buffer / 32; |
| 117 |
last_shadow_index += (ctrl->spio_last_buffer % 32) ? 1 : 0; |
| 118 |
if (last_shadow_index > num_shadow_index) |
| 119 |
{ |
| 120 |
err = psmi_handle_error(ctrl->context->ep, PSM_EP_DEVICE_FAILURE, |
| 121 |
"Number of buffer avail registers is wrong; " |
| 122 |
"have %u, expected %u (1st %u, piocnt %u, last %u)", |
| 123 |
last_shadow_index, |
| 124 |
(uint32_t)(sizeof(ctrl->spio_avail_shadow) / |
| 125 |
sizeof(ctrl->spio_avail_shadow[0])), |
| 126 |
base_info->spi_pioindex, ctrl->spio_last_buffer, |
| 127 |
base_info->spi_piocnt); |
| 128 |
goto fail; |
| 129 |
} |
| 130 |
|
| 131 |
/* update the shadow copy with the current contents of hardware |
| 132 |
* available registers */ |
| 133 |
for (i = 0; i < num_shadow_index; i++) |
| 134 |
ctrl->spio_avail_shadow[i] = ips_spio_read_avail_index(ctrl, i); |
| 135 |
|
| 136 |
/* Figure out the type of ordering we require for pio writes. Update the |
| 137 |
* routine we use for copies according to the type of pio write required */ |
| 138 |
wc_unordered = base_info->spi_runtime_flags; |
| 139 |
wc_unordered &= IPATH_RUNTIME_FORCE_WC_ORDER; |
| 140 |
|
| 141 |
if (base_info->spi_runtime_flags & IPATH_RUNTIME_SPECIAL_TRIGGER) { |
| 142 |
/* For now all PIO packets are < 2K and use the 2K trigger function. */ |
| 143 |
ctrl->spio_copy_fn = ipath_write_pio_special_trigger2k; |
| 144 |
order_str = "natural CPU (w/ 2k special trigger)"; |
| 145 |
} |
| 146 |
else { |
| 147 |
switch ( wc_unordered ) { |
| 148 |
case 0: |
| 149 |
#ifdef __MIC__ |
| 150 |
ctrl->spio_copy_fn = getenv("IPATH_MIC_DWORD_PIO")? |
| 151 |
ipath_write_pio:ipath_write_pio_vector; |
| 152 |
#else |
| 153 |
ctrl->spio_copy_fn = ipath_write_pio; |
| 154 |
#endif |
| 155 |
order_str = "natural CPU"; |
| 156 |
break; |
| 157 |
|
| 158 |
case IPATH_RUNTIME_FORCE_WC_ORDER: |
| 159 |
default: // any other non-zero |
| 160 |
ctrl->spio_copy_fn = ipath_write_pio_force_order; |
| 161 |
order_str = "forced"; |
| 162 |
break; |
| 163 |
} |
| 164 |
} |
| 165 |
|
| 166 |
_IPATH_PRDBG("PIO copy uses %s ordering\n", order_str); |
| 167 |
|
| 168 |
fail: |
| 169 |
return err; |
| 170 |
} |
| 171 |
|
| 172 |
psm_error_t |
| 173 |
ips_spio_fini(struct ips_spio *ctrl) |
| 174 |
{ |
| 175 |
spio_report_stall(ctrl, get_cycles(), 0ULL); |
| 176 |
return PSM_OK; |
| 177 |
} |
| 178 |
|
| 179 |
static |
| 180 |
void |
| 181 |
spio_report_stall(struct ips_spio *ctrl, uint64_t t_cyc_now, |
| 182 |
uint64_t send_failures) |
| 183 |
{ |
| 184 |
int last, i; |
| 185 |
size_t off = 0; |
| 186 |
char buf[1024]; |
| 187 |
|
| 188 |
if (ctrl->spio_num_stall == 0) |
| 189 |
return; |
| 190 |
|
| 191 |
last = ctrl->spio_last_buffer/32; |
| 192 |
|
| 193 |
if (send_failures > 0) { |
| 194 |
char bufctr[128]; |
| 195 |
uint64_t tx_stat, rx_stat; |
| 196 |
int ret; |
| 197 |
|
| 198 |
off = snprintf(buf, sizeof buf - 1, |
| 199 |
"PIO Send Bufs context %d with %d bufs from %d to %d. PIO avail regs: ", |
| 200 |
(int) psm_epid_context(ctrl->context->epid), |
| 201 |
ctrl->spio_num_of_buffer, ctrl->spio_first_buffer, |
| 202 |
ctrl->spio_last_buffer); |
| 203 |
|
| 204 |
for (i = 0; i < 8; i++) { |
| 205 |
uint64_t avail = ips_spio_read_avail_index(ctrl, i); |
| 206 |
off += snprintf(buf+off, sizeof buf - off - 1, " <%d>=(%llx) ", |
| 207 |
i, (long long) avail); |
| 208 |
} |
| 209 |
off += snprintf(buf+off, sizeof buf - off - 1, ". PIO shadow regs: "); |
| 210 |
for (i = ctrl->spio_first_buffer/32; i <= last; i++) { |
| 211 |
off += snprintf(buf+off, sizeof buf - off - 1, " <%d>=(%llx) ", |
| 212 |
i, (long long)ctrl->spio_avail_shadow[i]); |
| 213 |
} |
| 214 |
buf[off] = '\0'; |
| 215 |
|
| 216 |
/* In case ipathfs isn't running */ |
| 217 |
ret = infinipath_get_single_portctr(ctrl->unit_id, ctrl->portnum, |
| 218 |
"TxPkt", &tx_stat); |
| 219 |
if (ret != -1) { |
| 220 |
ret = infinipath_get_single_portctr(ctrl->unit_id, |
| 221 |
ctrl->portnum, "RxPkt", |
| 222 |
&rx_stat); |
| 223 |
if (ret != -1) { |
| 224 |
snprintf(bufctr, sizeof bufctr - 1, |
| 225 |
"(TxPktCnt=%llu,RxPktCnt=%llu)", |
| 226 |
(unsigned long long) tx_stat, |
| 227 |
(unsigned long long) rx_stat); |
| 228 |
bufctr[sizeof bufctr - 1] = '\0'; |
| 229 |
} else |
| 230 |
bufctr[0] = '\0'; |
| 231 |
} else |
| 232 |
bufctr[0] = '\0'; |
| 233 |
_IPATH_DBG("PIO Send Stall after at least %.2fM failed send attempts " |
| 234 |
"(elapsed=%.3fs, last=%.3fs, pio_stall_count=%lld) %s %s\n", |
| 235 |
send_failures / 1e6, |
| 236 |
PSMI_CYCLES_TO_SECSF(t_cyc_now - ctrl->spio_init_cyc), |
| 237 |
PSMI_CYCLES_TO_SECSF(t_cyc_now - ctrl->spio_last_stall_cyc), |
| 238 |
(unsigned long long) ctrl->spio_num_stall, |
| 239 |
bufctr[0] != '\0' ? bufctr : "", buf); |
| 240 |
} |
| 241 |
else { |
| 242 |
_IPATH_DBG( |
| 243 |
"PIO Send Stall Summary: count=%llu, last=%.3fs, elapsed=%.3fs", |
| 244 |
(unsigned long long) ctrl->spio_num_stall, |
| 245 |
PSMI_CYCLES_TO_SECSF(t_cyc_now - ctrl->spio_init_cyc), |
| 246 |
PSMI_CYCLES_TO_SECSF(t_cyc_now - ctrl->spio_last_stall_cyc)); |
| 247 |
} |
| 248 |
|
| 249 |
return; |
| 250 |
} |
| 251 |
|
| 252 |
static void |
| 253 |
spio_handle_stall(struct ips_spio *ctrl, |
| 254 |
uint64_t send_failures) |
| 255 |
{ |
| 256 |
uint64_t t_cyc_now = get_cycles(); |
| 257 |
int i, last; |
| 258 |
|
| 259 |
/* We handle the pio-stall every time but only report something every 20 |
| 260 |
* seconds. We print a summary at the end while closing the device */ |
| 261 |
ctrl->spio_num_stall++; |
| 262 |
ctrl->spio_num_stall_total++; |
| 263 |
|
| 264 |
if (ctrl->spio_next_stall_warning <= t_cyc_now) { |
| 265 |
/* If context status is ok (i.e. no cables pulled or anything) */ |
| 266 |
if (psmi_context_check_status(ctrl->context) == PSM_OK) |
| 267 |
spio_report_stall(ctrl, t_cyc_now, send_failures); |
| 268 |
ctrl->spio_next_stall_warning = |
| 269 |
get_cycles() + SPIO_STALL_WARNING_INTERVAL; |
| 270 |
} |
| 271 |
|
| 272 |
/* re-initialize our shadow from the real registers; by this time, |
| 273 |
* we know the hardware has to have done the update. |
| 274 |
* Also, kernel check may have changed things. |
| 275 |
*/ |
| 276 |
last = ctrl->spio_last_buffer/32; |
| 277 |
for (i = 0; i <= last; i++) { |
| 278 |
uint64_t mask, avail, shadow_avail; |
| 279 |
|
| 280 |
avail = ips_spio_read_avail_index(ctrl, i); |
| 281 |
shadow_avail = ctrl->spio_avail_shadow[i]; |
| 282 |
mask = (~(avail ^ shadow_avail) & SPIO_CHECK_MASK) << 1; |
| 283 |
shadow_avail &= ~mask; /* clear all possible in-use bits */ |
| 284 |
shadow_avail |= (avail & mask); |
| 285 |
ctrl->spio_avail_shadow[i] = shadow_avail; |
| 286 |
} |
| 287 |
|
| 288 |
ctrl->spio_last_stall_cyc = t_cyc_now; |
| 289 |
|
| 290 |
return; |
| 291 |
} |
| 292 |
|
| 293 |
/* |
| 294 |
* Update our shadow of the PIO available bitfield at index 'index' |
| 295 |
*/ |
| 296 |
static void __sendpath |
| 297 |
spio_update_shadow(struct ips_spio *ctrl, int index) |
| 298 |
{ |
| 299 |
register uint64_t mask, avail, shadow_avail; |
| 300 |
|
| 301 |
if_pf (*ctrl->spio_sendbuf_status) { |
| 302 |
__u64 event_mask; |
| 303 |
struct ips_proto *proto = (struct ips_proto*) &ctrl->ptl->proto; |
| 304 |
|
| 305 |
/* Get event mask for PSM to process */ |
| 306 |
event_mask = (uint64_t) *ctrl->spio_sendbuf_status; |
| 307 |
|
| 308 |
/* First ack the driver the receipt of the events */ |
| 309 |
_IPATH_VDBG("Acking event(s) 0x%"PRIx64" to qib driver.\n", (uint64_t) event_mask); |
| 310 |
ipath_event_ack(ctrl->context->ctrl, event_mask); |
| 311 |
|
| 312 |
if (event_mask & IPATH_EVENT_DISARM_BUFS) { |
| 313 |
/* Just acking event has disarmed all buffers */ |
| 314 |
_IPATH_VDBG("Disarm of send buffers completed.\n"); |
| 315 |
} |
| 316 |
|
| 317 |
if (event_mask & IPATH_EVENT_LINKDOWN) { |
| 318 |
/* A link down event can clear the LMC and SL2VL change as those |
| 319 |
* events are implicitly handled in the link up/down event handler. |
| 320 |
*/ |
| 321 |
event_mask &= ~(IPATH_EVENT_LMC_CHANGE | IPATH_EVENT_SL2VL_CHANGE); |
| 322 |
ips_ibta_link_updown_event(proto); |
| 323 |
_IPATH_VDBG("Link down detected.\n"); |
| 324 |
} |
| 325 |
|
| 326 |
if (event_mask & IPATH_EVENT_LID_CHANGE) { |
| 327 |
/* Display a warning that LID change has occurred during the run. This |
| 328 |
* is not supported in the current implementation and in general is |
| 329 |
* bad for the SM to re-assign LIDs during a run. |
| 330 |
*/ |
| 331 |
int lid, olid; |
| 332 |
|
| 333 |
lid = |
| 334 |
ipath_get_port_lid(proto->ep->context.base_info.spi_unit, |
| 335 |
proto->ep->context.base_info.spi_port); |
| 336 |
olid = PSMI_EPID_GET_LID(ctrl->context->epid); |
| 337 |
|
| 338 |
_IPATH_INFO("Warning! LID change detected during run. Old LID: %x, New Lid: %x\n", olid, lid); |
| 339 |
} |
| 340 |
|
| 341 |
if (event_mask & IPATH_EVENT_LMC_CHANGE) { |
| 342 |
_IPATH_INFO("Fabric LMC changed.\n"); |
| 343 |
} |
| 344 |
|
| 345 |
if (event_mask & IPATH_EVENT_SL2VL_CHANGE) { |
| 346 |
_IPATH_INFO("SL2VL mapping changed for port.\n"); |
| 347 |
ips_ibta_init_sl2vl_table(proto); |
| 348 |
} |
| 349 |
} |
| 350 |
|
| 351 |
index &= 0x7; // max spio_avail_shadow[] index. |
| 352 |
avail = ips_spio_read_avail_index(ctrl, index); |
| 353 |
|
| 354 |
do { |
| 355 |
shadow_avail = ctrl->spio_avail_shadow[index]; |
| 356 |
mask = (~(avail ^ shadow_avail) & SPIO_CHECK_MASK) << 1; |
| 357 |
shadow_avail &= ~mask; /* clear all possible in-use bits */ |
| 358 |
shadow_avail |= (avail & mask); |
| 359 |
} |
| 360 |
#ifndef PSMI_USE_THREADS |
| 361 |
while (0); |
| 362 |
ctrl->spio_avail_shadow[index] = shadow_avail; |
| 363 |
#else |
| 364 |
while (ips_cswap(...)); |
| 365 |
#endif |
| 366 |
} |
| 367 |
|
| 368 |
static void |
| 369 |
spio_handle_resync(struct ips_spio *ctrl, |
| 370 |
uint64_t consecutive_send_failed) |
| 371 |
{ |
| 372 |
if (ctrl->runtime_flags & IPATH_RUNTIME_FORCE_PIOAVAIL) |
| 373 |
ipath_force_pio_avail_update(ctrl->context->ctrl); |
| 374 |
if (!(consecutive_send_failed & (SPIO_MAX_CONSECUTIVE_SEND_FAIL - 1))) |
| 375 |
spio_handle_stall(ctrl, consecutive_send_failed); |
| 376 |
} |
| 377 |
|
| 378 |
/* |
| 379 |
* This function attempts to write a packet to a PIO. |
| 380 |
* |
| 381 |
* Recoverable errors: |
| 382 |
* PSM_OK: Packet triggered through PIO. |
| 383 |
* PSM_EP_NO_RESOURCES: No PIO bufs available or cable pulled. |
| 384 |
* |
| 385 |
* Unrecoverable errors: |
| 386 |
* PSM_EP_NO_NETWORK: No network, no lid, ... |
| 387 |
* PSM_EP_DEVICE_FAILURE: Chip failures, rxe/txe parity, etc. |
| 388 |
*/ |
| 389 |
psm_error_t __sendpath |
| 390 |
ips_spio_transfer_frame(struct ips_spio *ctrl, struct ips_flow *flow, |
| 391 |
void *header, void *payload, int length, |
| 392 |
uint32_t isCtrlMsg, uint32_t cksum_valid,uint32_t cksum) |
| 393 |
{ |
| 394 |
uint32_t *current_pio_buffer; |
| 395 |
const uint64_t toggle_bits = 3ULL; |
| 396 |
psm_error_t err = PSM_OK; |
| 397 |
int tries; |
| 398 |
int do_lock = (ctrl->runtime_flags & PSMI_RUNTIME_RCVTHREAD); |
| 399 |
struct ipath_pio_params pio_params; |
| 400 |
struct ips_message_header *p_hdr = (struct ips_message_header*) header; |
| 401 |
|
| 402 |
if (do_lock) |
| 403 |
pthread_spin_lock(&ctrl->spio_lock); |
| 404 |
|
| 405 |
if_pf (PSMI_FAULTINJ_ENABLED()) { |
| 406 |
PSMI_FAULTINJ_STATIC_DECL(fi_lost, "piosend", 1, IPS_FAULTINJ_PIOLOST); |
| 407 |
PSMI_FAULTINJ_STATIC_DECL(fi_busy, "piobusy", 1, IPS_FAULTINJ_PIOBUSY); |
| 408 |
if (psmi_faultinj_is_fault(fi_lost)) { |
| 409 |
if (do_lock) |
| 410 |
pthread_spin_unlock(&ctrl->spio_lock); |
| 411 |
return PSM_OK; |
| 412 |
} |
| 413 |
else if (psmi_faultinj_is_fault(fi_busy)) |
| 414 |
goto fi_busy; |
| 415 |
/* else fall through normal processing path, i.e. no faults */ |
| 416 |
} |
| 417 |
|
| 418 |
if (ctrl->spio_avail_shadow[ctrl->spio_current_buffer / 32] & |
| 419 |
(1ULL<<(((ctrl->spio_current_buffer) % 32 * 2) + 1))) |
| 420 |
{ |
| 421 |
/* |
| 422 |
* If the bit was already set, we couldn't get the pio buf. Update our |
| 423 |
* shadow copy. |
| 424 |
*/ |
| 425 |
spio_update_shadow(ctrl, ctrl->spio_current_buffer / 32); |
| 426 |
|
| 427 |
tries = ctrl->spio_num_of_buffer; |
| 428 |
|
| 429 |
while (tries && (ctrl->spio_avail_shadow[ctrl->spio_current_buffer / 32] & |
| 430 |
(1ULL<<(((ctrl->spio_current_buffer % 32) * 2) + 1)))) |
| 431 |
{ |
| 432 |
/* advance spio_current_buffer to next buffer */ |
| 433 |
if (++ctrl->spio_current_buffer > ctrl->spio_last_buffer) { |
| 434 |
ctrl->spio_current_buffer = ctrl->spio_first_buffer; |
| 435 |
spio_update_shadow(ctrl, ctrl->spio_current_buffer / 32); |
| 436 |
} |
| 437 |
else if ( (ctrl->spio_current_buffer % 32) == 0 ) |
| 438 |
spio_update_shadow(ctrl, ctrl->spio_current_buffer / 32); |
| 439 |
tries--; |
| 440 |
} |
| 441 |
|
| 442 |
if_pf ( !tries ) { |
| 443 |
/* Check unit status */ |
| 444 |
fi_busy: |
| 445 |
if ((err = psmi_context_check_status(ctrl->context)) == PSM_OK) { |
| 446 |
if (0 == (++ctrl->spio_consecutive_failures & |
| 447 |
(SPIO_RESYNC_CONSECUTIVE_SEND_FAIL-1))) |
| 448 |
spio_handle_resync(ctrl, ctrl->spio_consecutive_failures); |
| 449 |
err = PSM_EP_NO_RESOURCES; |
| 450 |
} |
| 451 |
/* If cable is pulled, we don't count it as a consecutive failure, |
| 452 |
* we just make it as though no send pio was available */ |
| 453 |
else if (err == PSM_OK_NO_PROGRESS) |
| 454 |
err = PSM_EP_NO_RESOURCES; |
| 455 |
/* else something bad happened in check_status */ |
| 456 |
if (do_lock) |
| 457 |
pthread_spin_unlock(&ctrl->spio_lock); |
| 458 |
return err; |
| 459 |
} |
| 460 |
} |
| 461 |
if (ctrl->spio_num_stall) // now able to send, so clear if set |
| 462 |
ctrl->spio_num_stall = 0; |
| 463 |
|
| 464 |
/* toggle the Generation bit and set the busy bit. |
| 465 |
* If we detected a flip, toggle busy but not GenBit (0x2) |
| 466 |
* If we didn't detect the flip, toggle busy but not the GenBit (0x3) */ |
| 467 |
ctrl->spio_avail_shadow[ctrl->spio_current_buffer / 32] ^= |
| 468 |
(toggle_bits<<(((ctrl->spio_current_buffer % 32) * 2))); |
| 469 |
|
| 470 |
current_pio_buffer = (uint32_t *) ctrl->spio_buffer_base + |
| 471 |
(ctrl->spio_buffer_spacing * |
| 472 |
(ctrl->spio_current_buffer - ctrl->spio_first_buffer)); |
| 473 |
|
| 474 |
/* advance spio_current_buffer to next buffer */ |
| 475 |
if (++ctrl->spio_current_buffer > ctrl->spio_last_buffer) |
| 476 |
ctrl->spio_current_buffer = ctrl->spio_first_buffer; |
| 477 |
|
| 478 |
ctrl->spio_consecutive_failures = 0; |
| 479 |
|
| 480 |
if (do_lock) |
| 481 |
pthread_spin_unlock(&ctrl->spio_lock); |
| 482 |
|
| 483 |
pio_params.length = length; |
| 484 |
pio_params.vl = (__be16_to_cpu(p_hdr->lrh[0]) >> LRH_VL_SHIFT) & 0xf; |
| 485 |
pio_params.port = ctrl->portnum; |
| 486 |
pio_params.cksum_is_valid = cksum_valid; |
| 487 |
pio_params.cksum = cksum; |
| 488 |
|
| 489 |
/* For matched send/receive rates and control messages IPD is not |
| 490 |
* required. |
| 491 |
*/ |
| 492 |
if_pf (!isCtrlMsg && flow->path->epr_active_ipd) |
| 493 |
pio_params.rate = |
| 494 |
ips_proto_pbc_static_rate(flow, |
| 495 |
(length + sizeof(struct ips_message_header))); |
| 496 |
else |
| 497 |
pio_params.rate = 0; |
| 498 |
|
| 499 |
/* Copy buffer using PIO */ |
| 500 |
ctrl->spio_copy_fn(current_pio_buffer, &pio_params, header, payload); |
| 501 |
|
| 502 |
return PSM_OK; |
| 503 |
} // ips_spio_transfer_frame() |
| 504 |
|