Debian Codegraph @

ptl_ips/ips_spio.c

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 /* included header files */
36 #include <stdlib.h>
37 #include <stddef.h>
38 #include <stdint.h>
39 #include <stdio.h>
40 #include <sched.h>
41
42 #include "ips_proto.h"
43 #include "ips_proto_internal.h"
44 #include "ips_spio.h"
45 #include "ipserror.h" /* ips error codes */
46 #include "ips_proto_params.h"
47 #include "ipath_byteorder.h"
48
49
50 #define SPIO_INUSE_MASK 0xAAAAAAAAAAAAAAAAULL
51 #define SPIO_CHECK_MASK 0x5555555555555555ULL
52
53 /* Report PIO stalls every 20 seconds at the least */
54 #define SPIO_STALL_WARNING_INTERVAL (nanosecs_to_cycles(20e9))
55 #define SPIO_MAX_CONSECUTIVE_SEND_FAIL (1<<20) /* 1M */
56 /* RESYNC_CONSECUTIVE_SEND_FAIL has to be a multiple of MAX_CONSECUTIVE */
57 #define SPIO_RESYNC_CONSECUTIVE_SEND_FAIL (1<<4) /* 16 */
58
59 static void spio_report_stall(struct ips_spio *ctrl,
60 uint64_t t_cyc_now,
61 uint64_t send_failures);
62
63 static void spio_handle_stall(struct ips_spio *ctrl,
64 uint64_t send_failures);
65
66 static inline
67 uint64_t
68 ips_spio_read_avail_index(struct ips_spio *ctrl, int index)
69 {
70 if (ctrl->runtime_flags & IPATH_RUNTIME_PIO_REGSWAPPED && index > 3) {
71 return __le64_to_cpu(ctrl->spio_avail_addr[index ^ 1]);
72 }
73 else
74 return __le64_to_cpu(ctrl->spio_avail_addr[index]);
75 }
76
77 psm_error_t
78 ips_spio_init(const struct psmi_context *context, const struct ptl *ptl,
79 struct ips_spio *ctrl)
80 {
81 psm_error_t err = PSM_OK;
82 const struct ipath_base_info *base_info = &context->base_info;
83 unsigned wc_unordered;
84 char *order_str = "undefined";
85 int i, last_shadow_index;
86 int num_shadow_index = sizeof(ctrl->spio_avail_shadow) /
87 sizeof(ctrl->spio_avail_shadow[0]);
88
89 ctrl->ptl = ptl;
90 ctrl->context = context;
91 /* Copy runtime flags */
92 ctrl->runtime_flags = ptl->runtime_flags;
93 ctrl->unit_id = context->ep->unit_id;
94 ctrl->portnum = context->ep->portnum;
95 pthread_spin_init(&ctrl->spio_lock, 0);
96 ctrl->spio_avail_addr =
97 (__le64 *)(ptrdiff_t)base_info->spi_pioavailaddr;
98 ctrl->spio_buffer_base =
99 (uint32_t *)(ptrdiff_t)base_info->spi_piobufbase;
100 ctrl->spio_sendbuf_status =
101 (unsigned long *)(ptrdiff_t)base_info->spi_sendbuf_status;
102
103 ctrl->spio_buffer_spacing = base_info->spi_pioalign >> 2;
104 ctrl->spio_first_buffer = ctrl->spio_current_buffer =
105 base_info->spi_pioindex;
106 ctrl->spio_last_buffer =
107 ctrl->spio_first_buffer + base_info->spi_piocnt - 1;
108 ctrl->spio_num_of_buffer = base_info->spi_piocnt;
109
110 ctrl->spio_consecutive_failures = 0;
111 ctrl->spio_num_stall = 0ULL;
112 ctrl->spio_next_stall_warning = 0ULL;
113 ctrl->spio_last_stall_cyc = 0ULL;
114 ctrl->spio_init_cyc = get_cycles();
115
116 last_shadow_index = ctrl->spio_last_buffer / 32;
117 last_shadow_index += (ctrl->spio_last_buffer % 32) ? 1 : 0;
118 if (last_shadow_index > num_shadow_index)
119 {
120 err = psmi_handle_error(ctrl->context->ep, PSM_EP_DEVICE_FAILURE,
121 "Number of buffer avail registers is wrong; "
122 "have %u, expected %u (1st %u, piocnt %u, last %u)",
123 last_shadow_index,
124 (uint32_t)(sizeof(ctrl->spio_avail_shadow) /
125 sizeof(ctrl->spio_avail_shadow[0])),
126 base_info->spi_pioindex, ctrl->spio_last_buffer,
127 base_info->spi_piocnt);
128 goto fail;
129 }
130
131 /* update the shadow copy with the current contents of hardware
132 * available registers */
133 for (i = 0; i < num_shadow_index; i++)
134 ctrl->spio_avail_shadow[i] = ips_spio_read_avail_index(ctrl, i);
135
136 /* Figure out the type of ordering we require for pio writes. Update the
137 * routine we use for copies according to the type of pio write required */
138 wc_unordered = base_info->spi_runtime_flags;
139 wc_unordered &= IPATH_RUNTIME_FORCE_WC_ORDER;
140
141 if (base_info->spi_runtime_flags & IPATH_RUNTIME_SPECIAL_TRIGGER) {
142 /* For now all PIO packets are < 2K and use the 2K trigger function. */
143 ctrl->spio_copy_fn = ipath_write_pio_special_trigger2k;
144 order_str = "natural CPU (w/ 2k special trigger)";
145 }
146 else {
147 switch ( wc_unordered ) {
148 case 0:
149 #ifdef __MIC__
150 ctrl->spio_copy_fn = getenv("IPATH_MIC_DWORD_PIO")?
151 ipath_write_pio:ipath_write_pio_vector;
152 #else
153 ctrl->spio_copy_fn = ipath_write_pio;
154 #endif
155 order_str = "natural CPU";
156 break;
157
158 case IPATH_RUNTIME_FORCE_WC_ORDER:
159 default: // any other non-zero
160 ctrl->spio_copy_fn = ipath_write_pio_force_order;
161 order_str = "forced";
162 break;
163 }
164 }
165
166 _IPATH_PRDBG("PIO copy uses %s ordering\n", order_str);
167
168 fail:
169 return err;
170 }
171
172 psm_error_t
173 ips_spio_fini(struct ips_spio *ctrl)
174 {
175 spio_report_stall(ctrl, get_cycles(), 0ULL);
176 return PSM_OK;
177 }
178
179 static
180 void
181 spio_report_stall(struct ips_spio *ctrl, uint64_t t_cyc_now,
182 uint64_t send_failures)
183 {
184 int last, i;
185 size_t off = 0;
186 char buf[1024];
187
188 if (ctrl->spio_num_stall == 0)
189 return;
190
191 last = ctrl->spio_last_buffer/32;
192
193 if (send_failures > 0) {
194 char bufctr[128];
195 uint64_t tx_stat, rx_stat;
196 int ret;
197
198 off = snprintf(buf, sizeof buf - 1,
199 "PIO Send Bufs context %d with %d bufs from %d to %d. PIO avail regs: ",
200 (int) psm_epid_context(ctrl->context->epid),
201 ctrl->spio_num_of_buffer, ctrl->spio_first_buffer,
202 ctrl->spio_last_buffer);
203
204 for (i = 0; i < 8; i++) {
205 uint64_t avail = ips_spio_read_avail_index(ctrl, i);
206 off += snprintf(buf+off, sizeof buf - off - 1, " <%d>=(%llx) ",
207 i, (long long) avail);
208 }
209 off += snprintf(buf+off, sizeof buf - off - 1, ". PIO shadow regs: ");
210 for (i = ctrl->spio_first_buffer/32; i <= last; i++) {
211 off += snprintf(buf+off, sizeof buf - off - 1, " <%d>=(%llx) ",
212 i, (long long)ctrl->spio_avail_shadow[i]);
213 }
214 buf[off] = '\0';
215
216 /* In case ipathfs isn't running */
217 ret = infinipath_get_single_portctr(ctrl->unit_id, ctrl->portnum,
218 "TxPkt", &tx_stat);
219 if (ret != -1) {
220 ret = infinipath_get_single_portctr(ctrl->unit_id,
221 ctrl->portnum, "RxPkt",
222 &rx_stat);
223 if (ret != -1) {
224 snprintf(bufctr, sizeof bufctr - 1,
225 "(TxPktCnt=%llu,RxPktCnt=%llu)",
226 (unsigned long long) tx_stat,
227 (unsigned long long) rx_stat);
228 bufctr[sizeof bufctr - 1] = '\0';
229 } else
230 bufctr[0] = '\0';
231 } else
232 bufctr[0] = '\0';
233 _IPATH_DBG("PIO Send Stall after at least %.2fM failed send attempts "
234 "(elapsed=%.3fs, last=%.3fs, pio_stall_count=%lld) %s %s\n",
235 send_failures / 1e6,
236 PSMI_CYCLES_TO_SECSF(t_cyc_now - ctrl->spio_init_cyc),
237 PSMI_CYCLES_TO_SECSF(t_cyc_now - ctrl->spio_last_stall_cyc),
238 (unsigned long long) ctrl->spio_num_stall,
239 bufctr[0] != '\0' ? bufctr : "", buf);
240 }
241 else {
242 _IPATH_DBG(
243 "PIO Send Stall Summary: count=%llu, last=%.3fs, elapsed=%.3fs",
244 (unsigned long long) ctrl->spio_num_stall,
245 PSMI_CYCLES_TO_SECSF(t_cyc_now - ctrl->spio_init_cyc),
246 PSMI_CYCLES_TO_SECSF(t_cyc_now - ctrl->spio_last_stall_cyc));
247 }
248
249 return;
250 }
251
252 static void
253 spio_handle_stall(struct ips_spio *ctrl,
254 uint64_t send_failures)
255 {
256 uint64_t t_cyc_now = get_cycles();
257 int i, last;
258
259 /* We handle the pio-stall every time but only report something every 20
260 * seconds. We print a summary at the end while closing the device */
261 ctrl->spio_num_stall++;
262 ctrl->spio_num_stall_total++;
263
264 if (ctrl->spio_next_stall_warning <= t_cyc_now) {
265 /* If context status is ok (i.e. no cables pulled or anything) */
266 if (psmi_context_check_status(ctrl->context) == PSM_OK)
267 spio_report_stall(ctrl, t_cyc_now, send_failures);
268 ctrl->spio_next_stall_warning =
269 get_cycles() + SPIO_STALL_WARNING_INTERVAL;
270 }
271
272 /* re-initialize our shadow from the real registers; by this time,
273 * we know the hardware has to have done the update.
274 * Also, kernel check may have changed things.
275 */
276 last = ctrl->spio_last_buffer/32;
277 for (i = 0; i <= last; i++) {
278 uint64_t mask, avail, shadow_avail;
279
280 avail = ips_spio_read_avail_index(ctrl, i);
281 shadow_avail = ctrl->spio_avail_shadow[i];
282 mask = (~(avail ^ shadow_avail) & SPIO_CHECK_MASK) << 1;
283 shadow_avail &= ~mask; /* clear all possible in-use bits */
284 shadow_avail |= (avail & mask);
285 ctrl->spio_avail_shadow[i] = shadow_avail;
286 }
287
288 ctrl->spio_last_stall_cyc = t_cyc_now;
289
290 return;
291 }
292
293 /*
294 * Update our shadow of the PIO available bitfield at index 'index'
295 */
296 static void __sendpath
297 spio_update_shadow(struct ips_spio *ctrl, int index)
298 {
299 register uint64_t mask, avail, shadow_avail;
300
301 if_pf (*ctrl->spio_sendbuf_status) {
302 __u64 event_mask;
303 struct ips_proto *proto = (struct ips_proto*) &ctrl->ptl->proto;
304
305 /* Get event mask for PSM to process */
306 event_mask = (uint64_t) *ctrl->spio_sendbuf_status;
307
308 /* First ack the driver the receipt of the events */
309 _IPATH_VDBG("Acking event(s) 0x%"PRIx64" to qib driver.\n", (uint64_t) event_mask);
310 ipath_event_ack(ctrl->context->ctrl, event_mask);
311
312 if (event_mask & IPATH_EVENT_DISARM_BUFS) {
313 /* Just acking event has disarmed all buffers */
314 _IPATH_VDBG("Disarm of send buffers completed.\n");
315 }
316
317 if (event_mask & IPATH_EVENT_LINKDOWN) {
318 /* A link down event can clear the LMC and SL2VL change as those
319 * events are implicitly handled in the link up/down event handler.
320 */
321 event_mask &= ~(IPATH_EVENT_LMC_CHANGE | IPATH_EVENT_SL2VL_CHANGE);
322 ips_ibta_link_updown_event(proto);
323 _IPATH_VDBG("Link down detected.\n");
324 }
325
326 if (event_mask & IPATH_EVENT_LID_CHANGE) {
327 /* Display a warning that LID change has occurred during the run. This
328 * is not supported in the current implementation and in general is
329 * bad for the SM to re-assign LIDs during a run.
330 */
331 int lid, olid;
332
333 lid =
334 ipath_get_port_lid(proto->ep->context.base_info.spi_unit,
335 proto->ep->context.base_info.spi_port);
336 olid = PSMI_EPID_GET_LID(ctrl->context->epid);
337
338 _IPATH_INFO("Warning! LID change detected during run. Old LID: %x, New Lid: %x\n", olid, lid);
339 }
340
341 if (event_mask & IPATH_EVENT_LMC_CHANGE) {
342 _IPATH_INFO("Fabric LMC changed.\n");
343 }
344
345 if (event_mask & IPATH_EVENT_SL2VL_CHANGE) {
346 _IPATH_INFO("SL2VL mapping changed for port.\n");
347 ips_ibta_init_sl2vl_table(proto);
348 }
349 }
350
351 index &= 0x7; // max spio_avail_shadow[] index.
352 avail = ips_spio_read_avail_index(ctrl, index);
353
354 do {
355 shadow_avail = ctrl->spio_avail_shadow[index];
356 mask = (~(avail ^ shadow_avail) & SPIO_CHECK_MASK) << 1;
357 shadow_avail &= ~mask; /* clear all possible in-use bits */
358 shadow_avail |= (avail & mask);
359 }
360 #ifndef PSMI_USE_THREADS
361 while (0);
362 ctrl->spio_avail_shadow[index] = shadow_avail;
363 #else
364 while (ips_cswap(...));
365 #endif
366 }
367
368 static void
369 spio_handle_resync(struct ips_spio *ctrl,
370 uint64_t consecutive_send_failed)
371 {
372 if (ctrl->runtime_flags & IPATH_RUNTIME_FORCE_PIOAVAIL)
373 ipath_force_pio_avail_update(ctrl->context->ctrl);
374 if (!(consecutive_send_failed & (SPIO_MAX_CONSECUTIVE_SEND_FAIL - 1)))
375 spio_handle_stall(ctrl, consecutive_send_failed);
376 }
377
378 /*
379 * This function attempts to write a packet to a PIO.
380 *
381 * Recoverable errors:
382 * PSM_OK: Packet triggered through PIO.
383 * PSM_EP_NO_RESOURCES: No PIO bufs available or cable pulled.
384 *
385 * Unrecoverable errors:
386 * PSM_EP_NO_NETWORK: No network, no lid, ...
387 * PSM_EP_DEVICE_FAILURE: Chip failures, rxe/txe parity, etc.
388 */
389 psm_error_t __sendpath
390 ips_spio_transfer_frame(struct ips_spio *ctrl, struct ips_flow *flow,
391 void *header, void *payload, int length,
392 uint32_t isCtrlMsg, uint32_t cksum_valid,uint32_t cksum)
393 {
394 uint32_t *current_pio_buffer;
395 const uint64_t toggle_bits = 3ULL;
396 psm_error_t err = PSM_OK;
397 int tries;
398 int do_lock = (ctrl->runtime_flags & PSMI_RUNTIME_RCVTHREAD);
399 struct ipath_pio_params pio_params;
400 struct ips_message_header *p_hdr = (struct ips_message_header*) header;
401
402 if (do_lock)
403 pthread_spin_lock(&ctrl->spio_lock);
404
405 if_pf (PSMI_FAULTINJ_ENABLED()) {
406 PSMI_FAULTINJ_STATIC_DECL(fi_lost, "piosend", 1, IPS_FAULTINJ_PIOLOST);
407 PSMI_FAULTINJ_STATIC_DECL(fi_busy, "piobusy", 1, IPS_FAULTINJ_PIOBUSY);
408 if (psmi_faultinj_is_fault(fi_lost)) {
409 if (do_lock)
410 pthread_spin_unlock(&ctrl->spio_lock);
411 return PSM_OK;
412 }
413 else if (psmi_faultinj_is_fault(fi_busy))
414 goto fi_busy;
415 /* else fall through normal processing path, i.e. no faults */
416 }
417
418 if (ctrl->spio_avail_shadow[ctrl->spio_current_buffer / 32] &
419 (1ULL<<(((ctrl->spio_current_buffer) % 32 * 2) + 1)))
420 {
421 /*
422 * If the bit was already set, we couldn't get the pio buf. Update our
423 * shadow copy.
424 */
425 spio_update_shadow(ctrl, ctrl->spio_current_buffer / 32);
426
427 tries = ctrl->spio_num_of_buffer;
428
429 while (tries && (ctrl->spio_avail_shadow[ctrl->spio_current_buffer / 32] &
430 (1ULL<<(((ctrl->spio_current_buffer % 32) * 2) + 1))))
431 {
432 /* advance spio_current_buffer to next buffer */
433 if (++ctrl->spio_current_buffer > ctrl->spio_last_buffer) {
434 ctrl->spio_current_buffer = ctrl->spio_first_buffer;
435 spio_update_shadow(ctrl, ctrl->spio_current_buffer / 32);
436 }
437 else if ( (ctrl->spio_current_buffer % 32) == 0 )
438 spio_update_shadow(ctrl, ctrl->spio_current_buffer / 32);
439 tries--;
440 }
441
442 if_pf ( !tries ) {
443 /* Check unit status */
444 fi_busy:
445 if ((err = psmi_context_check_status(ctrl->context)) == PSM_OK) {
446 if (0 == (++ctrl->spio_consecutive_failures &
447 (SPIO_RESYNC_CONSECUTIVE_SEND_FAIL-1)))
448 spio_handle_resync(ctrl, ctrl->spio_consecutive_failures);
449 err = PSM_EP_NO_RESOURCES;
450 }
451 /* If cable is pulled, we don't count it as a consecutive failure,
452 * we just make it as though no send pio was available */
453 else if (err == PSM_OK_NO_PROGRESS)
454 err = PSM_EP_NO_RESOURCES;
455 /* else something bad happened in check_status */
456 if (do_lock)
457 pthread_spin_unlock(&ctrl->spio_lock);
458 return err;
459 }
460 }
461 if (ctrl->spio_num_stall) // now able to send, so clear if set
462 ctrl->spio_num_stall = 0;
463
464 /* toggle the Generation bit and set the busy bit.
465 * If we detected a flip, toggle busy but not GenBit (0x2)
466 * If we didn't detect the flip, toggle busy but not the GenBit (0x3) */
467 ctrl->spio_avail_shadow[ctrl->spio_current_buffer / 32] ^=
468 (toggle_bits<<(((ctrl->spio_current_buffer % 32) * 2)));
469
470 current_pio_buffer = (uint32_t *) ctrl->spio_buffer_base +
471 (ctrl->spio_buffer_spacing *
472 (ctrl->spio_current_buffer - ctrl->spio_first_buffer));
473
474 /* advance spio_current_buffer to next buffer */
475 if (++ctrl->spio_current_buffer > ctrl->spio_last_buffer)
476 ctrl->spio_current_buffer = ctrl->spio_first_buffer;
477
478 ctrl->spio_consecutive_failures = 0;
479
480 if (do_lock)
481 pthread_spin_unlock(&ctrl->spio_lock);
482
483 pio_params.length = length;
484 pio_params.vl = (__be16_to_cpu(p_hdr->lrh[0]) >> LRH_VL_SHIFT) & 0xf;
485 pio_params.port = ctrl->portnum;
486 pio_params.cksum_is_valid = cksum_valid;
487 pio_params.cksum = cksum;
488
489 /* For matched send/receive rates and control messages IPD is not
490 * required.
491 */
492 if_pf (!isCtrlMsg && flow->path->epr_active_ipd)
493 pio_params.rate =
494 ips_proto_pbc_static_rate(flow,
495 (length + sizeof(struct ips_message_header)));
496 else
497 pio_params.rate = 0;
498
499 /* Copy buffer using PIO */
500 ctrl->spio_copy_fn(current_pio_buffer, &pio_params, header, payload);
501
502 return PSM_OK;
503 } // ips_spio_transfer_frame()
504