Debian Codegraph @

ptl_ips/ips_proto.c

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 /*
36 * IPS - Interconnect Protocol Stack.
37 */
38
39 #include <assert.h>
40 #include <sys/uio.h> /* writev */
41 #include "psm_user.h"
42 #include "ipserror.h"
43 #include "ips_proto.h"
44 #include "ips_proto_internal.h"
45 #include "ips_proto_help.h"
46
47 /*
48 * host ipv4 and pid used in ERR_CHK messages to detect stray processes
49 */
50 static uint32_t host_ipv4addr = 0; /* be */
51 static uint32_t host_pid = 0; /* be */
52
53 /*
54 * Control message types have their own flag to determine whether a message of
55 * that type is queued or not. These flags are kept in a state bitfield.
56 */
57 #define CTRL_MSG_ACK_QUEUED 0x0001
58 #define CTRL_MSG_NAK_QUEUED 0x0002
59 #define CTRL_MSG_ERR_CHK_QUEUED 0x0004
60 #define CTRL_MSG_ERR_CHK_PLS_QUEUED 0x0008
61 #define CTRL_MSG_CONNECT_REQUEST_QUEUED 0x0010
62 #define CTRL_MSG_CONNECT_REPLY_QUEUED 0x0020
63 #define CTRL_MSG_DISCONNECT_REQUEST_QUEUED 0x0040
64 #define CTRL_MSG_DISCONNECT_REPLY_QUEUED 0x0080
65 #define CTRL_MSG_TIDS_RELEASE_QUEUED 0x0100
66 #define CTRL_MSG_TIDS_RELEASE_CONFIRM_QUEUED 0x0200
67 #define CTRL_MSG_CLOSE_QUEUED 0x0400
68 #define CTRL_MSG_CLOSE_ACK_QUEUED 0x0800
69 #define CTRL_MSG_ABORT_QUEUED 0x1000
70 #define CTRL_MSG_TIDS_GRANT_QUEUED 0x2000
71 #define CTRL_MSG_TIDS_GRANT_ACK_QUEUED 0x4000
72 #define CTRL_MSG_ERR_CHK_GEN_QUEUED 0x8000
73 #define CTRL_MSG_FLOW_CCA_BECN 0x10000
74
75 #define CTRL_MSG_QUEUE_ALWAYS 0x80000000
76
77 #define _desc_idx u32w0
78 #define _desc_genc u32w1
79
80 static void ctrlq_init(struct ips_ctrlq *ctrlq, int flowid,
81 struct ips_proto *proto);
82 static psm_error_t proto_sdma_init(struct ips_proto *proto,
83 const psmi_context_t *context);
84
85 psm_error_t
86 ips_proto_init(const psmi_context_t *context, const ptl_t *ptl,
87 int num_of_send_bufs, int num_of_send_desc, uint32_t imm_size,
88 const struct psmi_timer_ctrl *timerq,
89 const struct ips_epstate *epstate,
90 const struct ips_spio *spioc,
91 struct ips_proto *proto)
92 {
93 const struct ipath_base_info *base_info = &context->base_info;
94 uint32_t protoexp_flags, cksum_sz = 0;
95 union psmi_envvar_val env_tid, env_cksum, env_mtu;
96 psm_error_t err = PSM_OK;
97
98 /*
99 * Checksum packets within PSM. Default is off.
100 * This is heavy weight and done in software so not recommended for
101 * production runs.
102 */
103
104 psmi_getenv("PSM_CHECKSUM",
105 "Enable checksum of messages (0 disables checksum)",
106 PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT_FLAGS,
107 (union psmi_envvar_val) 0,
108 &env_cksum);
109
110 memset(proto, 0, sizeof(struct ips_proto));
111 proto->ptl = (ptl_t *) ptl;
112 proto->ep = context->ep; /* cached */
113 proto->mq = context->ep->mq; /* cached */
114 proto->fd = context->fd; /* cached */
115 proto->pend_sends.proto = proto;
116 psmi_timer_entry_init(&proto->pend_sends.timer,
117 ips_proto_timer_pendq_callback, &proto->pend_sends);
118 STAILQ_INIT(&proto->pend_sends.pendq);
119 proto->epstate = (struct ips_epstate *) epstate;
120 proto->timerq = (struct psmi_timer_ctrl *) timerq;
121 proto->spioc = (struct ips_spio *) spioc;
122
123 proto->epinfo.ep_baseqp = base_info->spi_qpair;
124 proto->epinfo.ep_context = base_info->spi_context; /* "real" context */
125
126 proto->epinfo.ep_subcontext = base_info->spi_subcontext;
127 proto->epinfo.ep_hca_type = psmi_epid_hca_type(context->epid);
128
129 proto->epinfo.ep_unit = base_info->spi_unit;
130 proto->epinfo.ep_hdrq_msg_size = (IPS_HEADER_QUEUE_HWORDS +
131 IPS_HEADER_QUEUE_IWORDS +
132 IPS_HEADER_QUEUE_UWORDS_MIN) << 2;
133
134 /* If checksums enabled we insert checksum at end of packet */
135 cksum_sz = env_cksum.e_uint ? PSM_CRC_SIZE_IN_BYTES : 0;
136
137 proto->epinfo.ep_mtu = base_info->spi_mtu -
138 proto->epinfo.ep_hdrq_msg_size -
139 CRC_SIZE_IN_BYTES - PCB_SIZE_IN_BYTES;
140 proto->epinfo.ep_mtu = ips_next_low_pow2(proto->epinfo.ep_mtu);
141 /* Decrement checksum accounting AFTER lowering power of two */
142 proto->epinfo.ep_mtu -= cksum_sz;
143
144 /* See if user specifies a lower MTU to use */
145 if (!psmi_getenv("PSM_MTU", "MTU specified by user: 1-5,256-4096[4/2048]",
146 PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_INT,
147 (union psmi_envvar_val) -1,
148 &env_mtu)) {
149 if (env_mtu.e_int != 256 && env_mtu.e_int != 512
150 && env_mtu.e_int != 1024 && env_mtu.e_int != 2048
151 && env_mtu.e_int != 4096) {
152 if (env_mtu.e_int < 1 || env_mtu.e_int > 5) env_mtu.e_int = 4;
153 env_mtu.e_int = ibta_mtu_enum_to_int((enum ibta_mtu)env_mtu.e_int);
154 }
155 if (proto->epinfo.ep_mtu > env_mtu.e_int)
156 proto->epinfo.ep_mtu = env_mtu.e_int;
157 }
158
159 proto->epinfo.ep_piosize = base_info->spi_piosize -
160 proto->epinfo.ep_hdrq_msg_size -
161 CRC_SIZE_IN_BYTES - PCB_SIZE_IN_BYTES - cksum_sz;
162
163 /* Keep PIO as multiple of cache line size */
164 if (proto->epinfo.ep_piosize > PSM_CACHE_LINE_BYTES)
165 proto->epinfo.ep_piosize &= ~(PSM_CACHE_LINE_BYTES - 1);
166
167
168 proto->timeout_send = us_2_cycles(IPS_PROTO_SPIO_RETRY_US_DEFAULT);
169
170 proto->iovec_cntr_next_inflight = 0;
171 proto->iovec_thresh_eager= proto->iovec_thresh_eager_blocking = ~0U;
172 proto->scb_max_inflight = 2*num_of_send_desc;
173 proto->scb_bufsize = PSMI_ALIGNUP(max(base_info->spi_piosize,
174 base_info->spi_mtu),
175 PSMI_PAGESIZE),
176 proto->t_init = get_cycles();
177 proto->t_fini = 0;
178 proto->flags = env_cksum.e_uint ?
179 IPS_PROTO_FLAG_CKSUM : 0;
180
181 proto->num_connected_to = 0;
182 proto->num_connected_from = 0;
183 proto->num_disconnect_requests = 0;
184 proto->stray_warn_interval = (uint64_t) -1;
185 proto->done_warning = 0;
186 proto->done_once = 0;
187 proto->num_bogus_warnings = 0;
188 proto->psmi_logevent_tid_send_reqs.interval_secs = 15;
189 proto->psmi_logevent_tid_send_reqs.next_warning = 0;
190 proto->psmi_logevent_tid_send_reqs.count = 0;
191
192 /* Initialize IBTA related stuff (path record, SL2VL, CCA etc.) */
193 if ((err = ips_ibta_init(proto)))
194 goto fail;
195
196 {
197 /* Disable coalesced ACKs? */
198 union psmi_envvar_val env_coalesce_acks;
199
200 psmi_getenv("PSM_COALESCE_ACKS",
201 "Coalesce ACKs on the wire (default is enabled i.e. 1)",
202 PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_UINT_FLAGS,
203 (union psmi_envvar_val) 1, /* Enabled by default */
204 &env_coalesce_acks);
205
206 if (env_coalesce_acks.e_uint)
207 proto->flags |= IPS_PROTO_FLAG_COALESCE_ACKS;
208 }
209
210 {
211 /* Number of credits per flow */
212 union psmi_envvar_val env_flow_credits;
213 int df_flow_credits = min(PSM_FLOW_CREDITS, num_of_send_desc);
214
215 psmi_getenv("PSM_FLOW_CREDITS",
216 "Number of unacked packets (credits) per flow (default is 64)",
217 PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT,
218 (union psmi_envvar_val) df_flow_credits,
219 &env_flow_credits);
220 proto->flow_credits = env_flow_credits.e_uint;
221 }
222
223 if ((context->runtime_flags & IPATH_RUNTIME_SDMA))
224 if ((err = proto_sdma_init(proto, context)))
225 goto fail;
226
227 /*
228 * Clone sendreq mpool configuration for pend sends config
229 */
230 {
231 uint32_t chunks, maxsz;
232
233 psmi_assert_always(proto->ep->mq->sreq_pool != NULL);
234 psmi_mpool_get_obj_info(proto->ep->mq->sreq_pool, &chunks, &maxsz);
235
236 proto->pend_sends_pool =
237 psmi_mpool_create(sizeof(struct ips_pend_sreq), chunks, maxsz,
238 0, DESCRIPTORS, NULL, NULL);
239 if (proto->pend_sends_pool == NULL) {
240 err = PSM_NO_MEMORY;
241 goto fail;
242 }
243 }
244
245 /*
246 * Register ips protocol statistics
247 *
248 * We put a (*) in the output to denote stats that may cause a drop in
249 * performance.
250 *
251 * We put a (**) in the output of those stats that "should never happen"
252 */
253 {
254 struct psmi_stats_entry entries[] = {
255 PSMI_STATS_DECLU64("pio busy count",
256 &proto->stats.pio_busy_cnt),
257 /* Throttling by kernel */
258 PSMI_STATS_DECLU64("writev busy cnt",
259 &proto->stats.writev_busy_cnt),
260 /* When local dma completion is in the way... */
261 PSMI_STATS_DECLU64("writev compl. eagain",
262 &proto->stats.writev_compl_eagain),
263 /* When remote completion happens before local completion */
264 PSMI_STATS_DECLU64("writev compl. delay (*)",
265 &proto->stats.writev_compl_delay),
266 PSMI_STATS_DECLU64("scb unavail eager count",
267 &proto->stats.scb_egr_unavail_cnt),
268 PSMI_STATS_DECLU64("scb unavail exp count",
269 &proto->stats.scb_exp_unavail_cnt),
270 PSMI_STATS_DECLU64("rcvhdr overflows", /* Normal egr/hdr ovflw */
271 &proto->stats.hdr_overflow),
272 PSMI_STATS_DECLU64("rcveager overflows",
273 &proto->stats.egr_overflow),
274 PSMI_STATS_DECLU64("lid zero errs (**)", /* shouldn't happen */
275 &proto->stats.lid_zero_errs),
276 PSMI_STATS_DECLU64("unknown packets (**)", /* shouldn't happen */
277 &proto->stats.unknown_packets),
278 PSMI_STATS_DECLU64("stray packets (*)",
279 &proto->stats.stray_packets),
280 PSMI_STATS_DECLU64("send dma misaligns (*)",
281 &proto->stats.send_dma_misaligns),
282 PSMI_STATS_DECLU64("amreply no bufs (*)",
283 &proto->proto_am.amreply_nobufs),
284 PSMI_STATS_DECLU64("pio stalls (*)", /* shouldn't happen too often */
285 &proto->spioc->spio_num_stall_total),
286 PSMI_STATS_DECLU64("Invariant CRC error (*)",
287 &proto->error_stats.num_icrc_err),
288 PSMI_STATS_DECLU64("Variant CRC error (*)",
289 &proto->error_stats.num_vcrc_err),
290 PSMI_STATS_DECLU64("ECC error ",
291 &proto->error_stats.num_ecc_err),
292 PSMI_STATS_DECLU64("IB Len error",
293 &proto->error_stats.num_len_err),
294 PSMI_STATS_DECLU64("IB MTU error ",
295 &proto->error_stats.num_mtu_err),
296 PSMI_STATS_DECLU64("KDETH error ",
297 &proto->error_stats.num_khdr_err),
298 PSMI_STATS_DECLU64("TID error ",
299 &proto->error_stats.num_tid_err),
300 PSMI_STATS_DECLU64("MK error ",
301 &proto->error_stats.num_mk_err),
302 PSMI_STATS_DECLU64("IB error ",
303 &proto->error_stats.num_ib_err),
304
305 };
306
307 err = psmi_stats_register_type("InfiniPath low-level protocol stats",
308 PSMI_STATSTYPE_IPSPROTO,
309 entries,
310 PSMI_STATS_HOWMANY(entries),
311 NULL);
312 if (err != PSM_OK)
313 goto fail;
314 }
315
316 /*
317 * Control Queue and messaging
318 */
319 {
320 int idx;
321
322 for (idx = 0; idx < EP_FLOW_LAST; idx++)
323 ctrlq_init(&proto->ctrlq[idx], idx, proto);
324 }
325
326 /*
327 * Receive-side handling
328 */
329 if ((err = ips_proto_recv_init(proto)))
330 goto fail;
331
332 /*
333 * Eager buffers. We don't care to receive a callback when eager buffers
334 * are newly released since we actively poll for new bufs.
335 */
336 if ((err = ips_scbctrl_init(context, num_of_send_desc,
337 num_of_send_bufs, imm_size, proto->scb_bufsize,
338 NULL, NULL, &proto->scbc_egr)))
339 goto fail;
340
341 /*
342 * Expected protocol handling.
343 * If we enable tid-based expected rendezvous, the expected protocol code
344 * handles its own rv scb buffers. If not, we have to enable eager-based
345 * rendezvous and we allocate scb buffers for it.
346 */
347 psmi_getenv("PSM_TID",
348 "Tid proto flags (0 disables protocol)",
349 PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT_FLAGS,
350 (union psmi_envvar_val) IPS_PROTOEXP_FLAGS_DEFAULT,
351 &env_tid);
352 protoexp_flags = env_tid.e_uint;
353
354 if (protoexp_flags & IPS_PROTOEXP_FLAG_ENABLED) {
355 proto->scbc_rv = NULL;
356 if ((err = ips_protoexp_init(context, proto, protoexp_flags,
357 num_of_send_bufs, num_of_send_desc,
358 &proto->protoexp)))
359 goto fail;
360 }
361 else {
362 proto->protoexp = NULL;
363 proto->scbc_rv = (struct ips_scbctrl *)
364 psmi_calloc(proto->ep, DESCRIPTORS,
365 1, sizeof(struct ips_scbctrl));
366 if (proto->scbc_rv == NULL) {
367 err = PSM_NO_MEMORY;
368 goto fail;
369 }
370 /*
371 * Rendezvous buffers. We want to get a callback for rendezvous bufs
372 * since we asynchronously try to make progress on these sends and only
373 * schedule them on the timerq if there are pending sends and available
374 * bufs.
375 */
376 if ((err = ips_scbctrl_init(context, num_of_send_desc, 0 /* no bufs */,
377 0, 0 /* bufsize==0 */, ips_proto_rv_scbavail_callback,
378 proto, proto->scbc_rv)))
379 goto fail;
380 }
381
382 /*
383 * Parse the tid error settings from the environment.
384 * <interval_secs>:<max_count_before_exit>
385 */
386 {
387 int tvals[2];
388 char *tid_err;
389 union psmi_envvar_val env_tiderr;
390
391 tid_err = "-1:0"; /* no tiderr warnings, never exits */
392 tvals[0] = -1;
393 tvals[1] = 0;
394
395 if (!psmi_getenv("PSM_TID_ERROR",
396 "Tid error control <intval_secs:max_errors>",
397 PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_STR,
398 (union psmi_envvar_val) tid_err,
399 &env_tiderr)) {
400 /* not using default values */
401 tid_err = env_tiderr.e_str;
402 psmi_parse_str_tuples(tid_err, 2, tvals);
403 }
404 if (tvals[0] >= 0)
405 proto->tiderr_warn_interval = sec_2_cycles(tvals[0]);
406 else
407 proto->tiderr_warn_interval = UINT64_MAX;
408 proto->tiderr_max = tvals[1];
409 _IPATH_PRDBG("Tid error control: warning every %d secs%s, "
410 "fatal error after %d tid errors%s\n",
411 tvals[0], (tvals[0] < 0) ? " (no warnings)" : "",
412 tvals[1], (tvals[1] == 0) ? " (never fatal)" : "");
413 }
414
415 /*
416 * Active Message interface. AM requests compete with MQ for eager
417 * buffers, since request establish the amount of buffering in the network
418 * (maximum number of requests in flight). AM replies use the same amount
419 * of request buffers -- we can never run out of AM reply buffers because a
420 * request handler can only be run if we have at least one reply buffer (or
421 * else the AM request is dropped).
422 */
423 if ((err = ips_proto_am_init(proto, num_of_send_bufs, num_of_send_desc,
424 imm_size, &proto->proto_am)))
425 goto fail;
426
427 if (!host_pid) {
428 char ipbuf[INET_ADDRSTRLEN], *p;
429 host_pid = (uint32_t) getpid();
430 host_ipv4addr = psmi_get_ipv4addr(); /* already be */
431 if (host_ipv4addr == 0) {
432 _IPATH_DBG("Unable to obtain local IP address, "
433 "not fatal but some features may be disabled\n");
434 }
435 else if (host_ipv4addr == __cpu_to_be32(0x7f000001)) {
436 _IPATH_INFO("Localhost IP address is set to the "
437 "loopback address 127.0.0.1, "
438 "not fatal but some features may be disabled\n");
439 }
440 else {
441 p = (char *) inet_ntop(AF_INET, (const void *) &host_ipv4addr,
442 ipbuf, sizeof ipbuf);
443 _IPATH_PRDBG("Ethernet Host IP=%s and PID=%d\n", p, host_pid);
444 }
445
446 /* Store in big endian for use in ERR_CHK */
447 host_pid = __cpu_to_be32(host_pid);
448 }
449
450 fail:
451 return err;
452 }
453
454 psm_error_t
455 ips_proto_fini(struct ips_proto *proto, int force, uint64_t timeout_in)
456 {
457 struct psmi_eptab_iterator itor;
458 uint64_t t_start;
459 uint64_t t_grace_start, t_grace_time, t_grace_finish, t_grace_interval;
460 psm_epaddr_t epaddr;
461 psm_error_t err = PSM_OK;
462 int i;
463 union psmi_envvar_val grace_intval;
464
465 psmi_getenv("PSM_CLOSE_GRACE_PERIOD",
466 "Additional grace period in seconds for closing end-point.",
467 PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_UINT,
468 (union psmi_envvar_val) 0,
469 &grace_intval);
470
471 if (getenv("PSM_CLOSE_GRACE_PERIOD")) {
472 t_grace_time = grace_intval.e_uint * SEC_ULL;
473 }
474 else if (timeout_in > 0) {
475 /* default to half of the close time-out */
476 t_grace_time = timeout_in / 2;
477 }
478 else {
479 /* propagate the infinite time-out case */
480 t_grace_time = 0;
481 }
482
483 if (t_grace_time > 0 && t_grace_time < PSMI_MIN_EP_CLOSE_TIMEOUT)
484 t_grace_time = PSMI_MIN_EP_CLOSE_TIMEOUT;
485
486 /* At close we will busy wait for the grace interval to see if any
487 * receive progress is made. If progress is made we will wait for
488 * another grace interval, until either no progress is made or the
489 * entire grace period has passed. If the grace interval is too low
490 * we may miss traffic and exit too early. If the grace interval is
491 * too large the additional time spent while closing the program
492 * will become visible to the user. */
493 psmi_getenv("PSM_CLOSE_GRACE_INTERVAL",
494 "Grace interval in seconds for closing end-point.",
495 PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_UINT,
496 (union psmi_envvar_val) 0,
497 &grace_intval);
498
499 if (getenv("PSM_CLOSE_GRACE_INTERVAL")) {
500 t_grace_interval = grace_intval.e_uint * SEC_ULL;
501 }
502 else {
503 /* A heuristic is used to scale up the timeout linearly with
504 * the number of endpoints, and we allow one second per 1000
505 * endpoints. */
506 t_grace_interval = (proto->ep->connections * SEC_ULL) / 1000;
507 }
508
509 if (t_grace_interval < PSMI_MIN_EP_CLOSE_GRACE_INTERVAL)
510 t_grace_interval = PSMI_MIN_EP_CLOSE_GRACE_INTERVAL;
511 if (t_grace_interval > PSMI_MAX_EP_CLOSE_GRACE_INTERVAL)
512 t_grace_interval = PSMI_MAX_EP_CLOSE_GRACE_INTERVAL;
513
514 PSMI_PLOCK_ASSERT();
515
516 t_start = proto->t_fini = get_cycles();
517
518 /* Close whatever has been left open */
519 if (proto->num_connected_to > 0) {
520 int num_disc = 0;
521 int *mask;
522 psm_error_t *errs;
523 psm_epaddr_t *epaddr_array;
524
525 psmi_epid_itor_init(&itor, proto->ep);
526 while ((epaddr = psmi_epid_itor_next(&itor))) {
527 if (epaddr->ptl == proto->ptl)
528 num_disc++;
529 }
530 psmi_epid_itor_fini(&itor);
531 mask = (int *) psmi_calloc(proto->ep, UNDEFINED, num_disc, sizeof(int));
532 errs = (psm_error_t *)
533 psmi_calloc(proto->ep, UNDEFINED, num_disc, sizeof(psm_error_t));
534 epaddr_array = (psm_epaddr_t *)
535 psmi_calloc(proto->ep, UNDEFINED, num_disc, sizeof(psm_epaddr_t));
536
537 if (errs == NULL || epaddr_array == NULL || mask == NULL) {
538 if (epaddr_array) psmi_free(epaddr_array);
539 if (errs) psmi_free(errs);
540 if (mask) psmi_free(mask);
541 err = PSM_NO_MEMORY;
542 goto fail;
543 }
544 psmi_epid_itor_init(&itor, proto->ep);
545 i = 0;
546 while ((epaddr = psmi_epid_itor_next(&itor))) {
547 if (epaddr->ptl == proto->ptl) {
548 mask[i] = 1;
549 epaddr_array[i] = epaddr;
550 i++;
551 PSM_MCTXT_REMOVE(epaddr);
552 }
553 }
554 psmi_epid_itor_fini(&itor);
555 err = ips_proto_disconnect(proto, force, num_disc, epaddr_array,
556 mask, errs, timeout_in);
557 psmi_free(mask);
558 psmi_free(errs);
559 psmi_free(epaddr_array);
560 }
561
562 t_grace_start = get_cycles();
563
564 while (psmi_cycles_left(t_grace_start, t_grace_time)) {
565 uint64_t t_grace_interval_start = get_cycles();
566 int num_disconnect_requests = proto->num_disconnect_requests;
567 PSMI_BLOCKUNTIL(proto->ep, err,
568 (proto->num_connected_from == 0 ||
569 !psmi_cycles_left(t_start, timeout_in)) &&
570 (!psmi_cycles_left(t_grace_interval_start, t_grace_interval) ||
571 !psmi_cycles_left(t_grace_start, t_grace_time)));
572 if (num_disconnect_requests == proto->num_disconnect_requests) {
573 /* nothing happened in this grace interval so break out early */
574 break;
575 }
576 }
577
578 t_grace_finish = get_cycles();
579
580 _IPATH_PRDBG("Closing endpoint disconnect left to=%d,from=%d after %d millisec of grace (out of %d)\n",
581 proto->num_connected_to, proto->num_connected_from,
582 (int) (cycles_to_nanosecs(t_grace_finish - t_grace_start) / MSEC_ULL),
583 (int) (t_grace_time / MSEC_ULL));
584
585 if ((err = ips_ibta_fini(proto)))
586 goto fail;
587
588 if ((err = ips_proto_am_fini(&proto->proto_am)))
589 goto fail;
590
591 if ((err = ips_scbctrl_fini(&proto->scbc_egr)))
592 goto fail;
593
594 ips_proto_recv_fini(proto);
595
596 if (proto->protoexp) {
597 if ((err = ips_protoexp_fini(proto->protoexp)))
598 goto fail;
599 }
600 else {
601 ips_scbctrl_fini(proto->scbc_rv);
602 psmi_free(proto->scbc_rv);
603 }
604
605 psmi_mpool_destroy(proto->pend_sends_pool);
606
607 fail:
608 proto->t_fini = proto->t_init = 0;
609 return err;
610 }
611
612 static
613 psm_error_t
614 proto_sdma_init(struct ips_proto *proto, const psmi_context_t *context)
615 {
616 union psmi_envvar_val env_sdma, env_ipathegr;
617 char *c;
618 uint32_t defval = IPS_PROTO_FLAGS_DEFAULT & IPS_PROTO_FLAGS_ALL_SDMA;
619 psm_error_t err = PSM_OK;
620 int egrmode;
621
622 /*
623 * Only initialize if RUNTIME_SDMA is enabled.
624 */
625 psmi_assert_always(context->runtime_flags & IPATH_RUNTIME_SDMA);
626
627 if ((c = getenv("PSM_SDMA")) && *c && !strncmp("always", c, 7))
628 defval = IPS_PROTO_FLAGS_ALL_SDMA;
629
630 psmi_getenv("PSM_SDMA",
631 "ipath send dma flags (0 disables send dma)",
632 PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT_FLAGS,
633 (union psmi_envvar_val) defval,
634 &env_sdma);
635
636 if(env_sdma.e_uint != 1)
637 proto->flags |= env_sdma.e_uint & IPS_PROTO_FLAGS_ALL_SDMA;
638
639 /* If anything uses send dma, figure out our max packet threshold to call
640 * send dma with */
641 proto->scb_max_sdma = IPS_SDMA_MAX_SCB;
642 if (proto->flags & IPS_PROTO_FLAGS_ALL_SDMA) {
643 psmi_getenv("PSM_SDMA_THRESH",
644 "ipath send dma max packet per call",
645 PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_UINT,
646 (union psmi_envvar_val) proto->scb_max_sdma,
647 &env_sdma);
648 proto->scb_max_sdma = env_sdma.e_uint;
649 if (proto->scb_max_sdma < 1) {
650 _IPATH_ERROR("Overriding PSM_SDMA_THRESH=%u to be '%u'\n",
651 proto->scb_max_sdma, 1);
652 proto->scb_max_sdma = 1;
653 }
654 }
655
656 egrmode = proto->flags &
657 (IPS_PROTO_FLAG_MQ_ENVELOPE_SDMA|IPS_PROTO_FLAG_MQ_EAGER_SDMA);
658
659 /* Some modes don't make sense or at least, MQ doesn't expect them to
660 * be a functional mode. For example, it's not possible to use DMA
661 * message envelopes with PIO eager data.
662 */
663 if (egrmode == IPS_PROTO_FLAG_MQ_ENVELOPE_SDMA) {
664 err = psmi_handle_error(proto->ep, PSM_PARAM_ERR,
665 "Unsupported Send DMA mode 0x%x: dma envelopes and pio eager",
666 proto->flags);
667 goto fail;
668 }
669 /* Only bother with switchover for pio-envelope,dma-eagerdata */
670 else if (egrmode == IPS_PROTO_FLAG_MQ_EAGER_SDMA) {
671 /* Reduce threshold to use SDMA for QLE73XX as we are PIO limited for
672 * medium message sizes on it.
673 */
674 uint32_t hca_type = psmi_get_hca_type((psmi_context_t*) context);
675
676 defval = (hca_type == PSMI_HCA_TYPE_QLE73XX) ?
677 MQ_IPATH_THRESH_EGR_SDMA_SQ : MQ_IPATH_THRESH_EGR_SDMA;
678 psmi_getenv("PSM_MQ_EAGER_SDMA_SZ",
679 "ipath pio-to-sdma eager switchover",
680 PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT,
681 (union psmi_envvar_val) defval, &env_ipathegr);
682
683 /* Has to be at least 1 MTU */
684 proto->iovec_thresh_eager = proto->iovec_thresh_eager_blocking =
685 max(proto->epinfo.ep_piosize, env_ipathegr.e_uint);
686
687 /* For QLE73XX bump up the eager SDMA threshold for blocking sends if
688 * the user has not explicitly set one.
689 */
690 if ((hca_type == PSMI_HCA_TYPE_QLE73XX) &&
691 (proto->iovec_thresh_eager == defval))
692 proto->iovec_thresh_eager_blocking = MQ_IPATH_THRESH_EGR_SDMA;
693 }
694 else if (egrmode ==
695 (IPS_PROTO_FLAG_MQ_ENVELOPE_SDMA|IPS_PROTO_FLAG_MQ_EAGER_SDMA))
696 {
697 /* Has to be 0 so we never try to split pio and dma */
698 proto->iovec_thresh_eager = proto->iovec_thresh_eager_blocking = 0;
699 }
700 else if (egrmode == 0) { /* all pio */
701 proto->iovec_thresh_eager = proto->iovec_thresh_eager_blocking = ~0U;
702 }
703
704 fail:
705 return err;
706 }
707
708 static
709 void
710 ctrlq_init(struct ips_ctrlq *ctrlq, int flowid, struct ips_proto *proto)
711 {
712 // clear the ctrl send queue
713 memset(ctrlq, 0, sizeof(*ctrlq));
714
715 proto->message_type_to_index[OPCODE_ACK] = CTRL_MSG_ACK_QUEUED;
716 proto->message_type_to_index[OPCODE_NAK] = CTRL_MSG_NAK_QUEUED;
717 proto->message_type_to_index[OPCODE_ERR_CHK] = CTRL_MSG_ERR_CHK_QUEUED;
718 proto->message_type_to_index[OPCODE_ERR_CHK_PLS] = CTRL_MSG_ERR_CHK_PLS_QUEUED;
719 proto->message_type_to_index[OPCODE_CONNECT_REQUEST] =
720 CTRL_MSG_CONNECT_REQUEST_QUEUED;
721 proto->message_type_to_index[OPCODE_CONNECT_REPLY] =
722 CTRL_MSG_CONNECT_REPLY_QUEUED;
723 proto->message_type_to_index[OPCODE_DISCONNECT_REQUEST] =
724 CTRL_MSG_DISCONNECT_REQUEST_QUEUED;
725 proto->message_type_to_index[OPCODE_DISCONNECT_REPLY] =
726 CTRL_MSG_DISCONNECT_REPLY_QUEUED;
727 proto->message_type_to_index[OPCODE_CLOSE] = CTRL_MSG_CLOSE_QUEUED;
728 proto->message_type_to_index[OPCODE_CLOSE_ACK] = CTRL_MSG_CLOSE_ACK_QUEUED;
729 proto->message_type_to_index[OPCODE_ABORT] = CTRL_MSG_ABORT_QUEUED;
730 proto->message_type_to_index[OPCODE_TIDS_GRANT] = CTRL_MSG_TIDS_GRANT_QUEUED;
731 proto->message_type_to_index[OPCODE_TIDS_GRANT_ACK] = CTRL_MSG_TIDS_GRANT_ACK_QUEUED;
732 proto->message_type_to_index[OPCODE_ERR_CHK_GEN] = CTRL_MSG_ERR_CHK_GEN_QUEUED;
733 proto->message_type_to_index[OPCODE_FLOW_CCA_BECN] = CTRL_MSG_FLOW_CCA_BECN;
734
735 ctrlq->ctrlq_head = ctrlq->ctrlq_tail = 0;
736 ctrlq->ctrlq_overflow = 0;
737 ctrlq->ctrlq_proto = proto;
738 ctrlq->ctrlq_flowid = flowid;
739 /* We never enqueue connect messages. They require 512 bytes and we don't
740 * want to stack allocate 512 bytes just when sending back acks.
741 */
742 proto->ctrl_msg_queue_never_enqueue = CTRL_MSG_CONNECT_REQUEST_QUEUED |
743 CTRL_MSG_CONNECT_REPLY_QUEUED |
744 CTRL_MSG_DISCONNECT_REQUEST_QUEUED |
745 CTRL_MSG_DISCONNECT_REPLY_QUEUED |
746 CTRL_MSG_ERR_CHK_GEN_QUEUED |
747 CTRL_MSG_TIDS_GRANT_QUEUED;
748
749 psmi_timer_entry_init(&ctrlq->ctrlq_timer,
750 ips_proto_timer_ctrlq_callback, ctrlq);
751
752 return;
753 }
754
755 static int inline
756 _build_ctrl_message(struct ips_proto *proto,
757 struct ips_proto_ctrl_message *msg,
758 ips_epaddr_t *ipsaddr, uint8_t message_type,
759 struct ips_flow *flow,
760 void *payload, uint8_t *discard_msg)
761 {
762 uint32_t tot_paywords = sizeof(struct ips_message_header) >> 2;
763 struct ips_epinfo *epinfo = &proto->epinfo;
764 struct ips_epinfo_remote *epr = &ipsaddr->epr;
765 uint16_t pkt_flags = IPS_EPSTATE_COMMIDX_PACK(epr->epr_commidx_to);
766 struct ips_message_header *p_hdr = &msg->pbc_hdr.hdr;
767 ips_path_rec_t *ctrl_path = ipsaddr->epr.epr_path[IPS_PATH_HIGH_PRIORITY][ipsaddr->epr.epr_hpp_index];
768 int paylen = 0;
769
770 if ((proto->flags & IPS_PROTO_FLAG_PPOLICY_ADAPTIVE) &&
771 (++ipsaddr->epr.epr_hpp_index >=
772 ipsaddr->epr.epr_num_paths[IPS_PATH_HIGH_PRIORITY]))
773 ipsaddr->epr.epr_hpp_index = 0;
774
775 /* Control messages go over the control path. */
776 p_hdr->lrh[0] = __cpu_to_be16(IPATH_LRH_BTH |
777 (ctrl_path->epr_sl << 4) |
778 (proto->sl2vl[ctrl_path->epr_sl] << LRH_VL_SHIFT));
779 p_hdr->lrh[1] = ctrl_path->epr_dlid;
780 p_hdr->lrh[2] = __cpu_to_be16(tot_paywords + SIZE_OF_CRC);
781 p_hdr->lrh[3] = ctrl_path->epr_slid;
782
783 p_hdr->bth[0] = __cpu_to_be32((IPATH_OPCODE_USER1 << 24) +
784 ctrl_path->epr_pkey);
785
786 /* If flow is congested then generate a BECN for path. */
787 if_pf (flow->flags & IPS_FLOW_FLAG_GEN_BECN) {
788 _IPATH_CCADBG("Generating BECN for flow %x ----> %x. Num congested packets: 0x%"PRIx64". Msg type: %d\n", __be16_to_cpu(flow->path->epr_slid), __be16_to_cpu(flow->path->epr_dlid), ipsaddr->stats.congestion_pkts, message_type);
789 p_hdr->bth[1] = __cpu_to_be32(epr->epr_qp | 1 << BTH_BECN_SHIFT);
790 flow->flags &= ~IPS_FLOW_FLAG_GEN_BECN;
791 }
792 else
793 p_hdr->bth[1] = __cpu_to_be32(epr->epr_qp);
794 p_hdr->bth[2] = 0;
795
796 p_hdr->commidx = epr->epr_commidx_to;
797 p_hdr->sub_opcode = message_type;
798 p_hdr->ack_seq_num = 0;
799 IPS_HEADER_SRCCONTEXT_SET(p_hdr, epinfo->ep_context);
800 p_hdr->src_subcontext = epinfo->ep_subcontext;
801 p_hdr->dst_subcontext = epr->epr_subcontext;
802 p_hdr->flags = 0;
803 p_hdr->mqhdr = 0;
804 p_hdr->flowid = flow->flowid;
805
806 switch (message_type) {
807 case OPCODE_ACK:
808 if_pt (flow->protocol != PSM_PROTOCOL_TIDFLOW)
809 p_hdr->ack_seq_num = flow->recv_seq_num.psn;
810 else {
811 ptl_arg_t *args = (ptl_arg_t*) payload;
812 uint32_t tid_recv_sessid;
813 struct ips_tid_recv_desc *tidrecvc;
814
815 /* TIDFLOW ACK.
816 * args[0] = send descriptor id
817 * args[1] = receive descriptor id
818 */
819 ips_ptladdr_lock(ipsaddr);
820
821 tid_recv_sessid = args[1]._desc_idx;
822 tidrecvc =
823 psmi_mpool_find_obj_by_index(proto->protoexp->tid_desc_recv_pool,
824 tid_recv_sessid);
825 if (tidrecvc == NULL) {
826 *discard_msg = 1;
827 ips_ptladdr_unlock(ipsaddr);
828 break;
829 }
830 if_pf (psmi_mpool_get_obj_gen_count(tidrecvc) != args[1]._desc_genc) {
831 *discard_msg = 1;
832 ips_ptladdr_unlock(ipsaddr);
833 break;
834 }
835
836 p_hdr->data[0].u64 = args[0].u64;
837 p_hdr->ack_seq_num = tidrecvc->tidflow_genseq.psn;
838 ips_ptladdr_unlock(ipsaddr);
839 }
840 break;
841
842 case OPCODE_NAK:
843 if_pf (flow->protocol != PSM_PROTOCOL_TIDFLOW) {
844 p_hdr->ack_seq_num = flow->recv_seq_num.psn;
845 }
846 else {
847 ptl_arg_t *args = (ptl_arg_t*) payload;
848 uint32_t tid_recv_sessid;
849 struct ips_tid_recv_desc *tidrecvc;
850 psmi_seqnum_t ack_seq_num;
851
852 /* TIDFLOW NAK.
853 * args[0] = send descriptor id
854 * args[1] = receive descriptor id
855 * args[2].u16w0 = Old generation to NAK
856 */
857 ips_ptladdr_lock(ipsaddr);
858
859 tid_recv_sessid = args[1]._desc_idx;
860 tidrecvc =
861 psmi_mpool_find_obj_by_index(proto->protoexp->tid_desc_recv_pool,
862 tid_recv_sessid);
863 if (tidrecvc == NULL) {
864 *discard_msg = 1;
865 ips_ptladdr_unlock(ipsaddr);
866 break;
867 }
868 if_pf (psmi_mpool_get_obj_gen_count(tidrecvc) != args[1]._desc_genc) {
869 *discard_msg = 1;
870 ips_ptladdr_unlock(ipsaddr);
871 break;
872 }
873
874 p_hdr->data[0].u64 = args[0].u64; /* Send descriptor id */
875 p_hdr->data[1].u32w0 = tidrecvc->tidflow_genseq.val; /*New flowgenseq*/
876
877 /* Ack seqnum contains the old generation we are acking for */
878 ack_seq_num = tidrecvc->tidflow_genseq;
879 ack_seq_num.gen = args[2].u16w0;
880 p_hdr->ack_seq_num = ack_seq_num.psn;
881
882 ips_ptladdr_unlock(ipsaddr);
883 }
884 break;
885
886 case OPCODE_ERR_CHK:
887 {
888 psmi_seqnum_t err_chk_seq;
889 ips_ptladdr_lock(ipsaddr);
890
891 err_chk_seq = (SLIST_EMPTY(&flow->scb_pend)) ?
892 flow->xmit_seq_num : SLIST_FIRST(&flow->scb_pend)->seq_num;
893 err_chk_seq.pkt -= 1;
894 p_hdr->bth[2] = __cpu_to_be32(err_chk_seq.psn);
895 ips_ptladdr_unlock(ipsaddr);
896 p_hdr->data[0].u32w0 = host_ipv4addr;
897 p_hdr->data[0].u32w1 = host_pid;
898
899 if (ipsaddr->flags & SESS_FLAG_HAS_RCVTHREAD)
900 pkt_flags |= INFINIPATH_KPF_INTR;
901 }
902 break;
903
904 case OPCODE_ERR_CHK_GEN:
905 {
906 struct ips_scb_unackedq *unackedq = &flow->scb_unacked;
907
908 /* TIDFLOW ERR_CHK_GEN
909 * args[0] = receive descriptor id
910 * args[1] = send descriptor id
911 */
912 if (!STAILQ_EMPTY(unackedq)) {
913 ips_scb_t *scb = STAILQ_FIRST(unackedq);
914 psmi_seqnum_t err_chk_seq;
915
916 ips_ptladdr_lock(ipsaddr);
917
918 psmi_assert_always(scb->tidsendc);
919
920 err_chk_seq = (SLIST_EMPTY(&flow->scb_pend)) ?
921 flow->xmit_seq_num : SLIST_FIRST(&flow->scb_pend)->seq_num;
922 err_chk_seq.seq -= 1;
923
924 /* NOTE: If error check gen is cached and we get a NAK
925 * the scbs are flushed again. This can increase the DMA counter
926 * as scb's are retransmitted which we don't check for here.
927 * One way is never cache the ERR_CHK_GEN messages so it's only
928 * called from the ack timeout callback. Other way is that we
929 * send the ERR_CHK_GEN message over SDMA so they are serialized with
930 * respect to each other. Note: In this case we don't need to
931 * wait for the DMA completion counters in the ack timeout.
932 */
933 p_hdr->bth[2] = __cpu_to_be32(err_chk_seq.psn);
934
935 /* Receive descriptor index */
936 p_hdr->data[0].u64 = scb->tidsendc->tid_list.tsess_descid.u64;
937 /* Send descriptor index */
938 p_hdr->data[1].u64 = scb->tidsendc->descid.u64;
939
940 ips_ptladdr_unlock(ipsaddr);
941
942 if (ipsaddr->flags & SESS_FLAG_HAS_RCVTHREAD)
943 pkt_flags |= INFINIPATH_KPF_INTR;
944 }
945 else
946 *discard_msg = 1;
947 }
948 break;
949
950 case OPCODE_FLOW_CCA_BECN:
951 _IPATH_CCADBG("Generating Explicit BECN for flow %x ----> %x. Num congested packets: 0x%"PRIx64"\n", __be16_to_cpu(flow->path->epr_slid), __be16_to_cpu(flow->path->epr_dlid), ipsaddr->stats.congestion_pkts);
952 p_hdr->bth[1] = __cpu_to_be32(epr->epr_qp | 1 << BTH_BECN_SHIFT);
953 p_hdr->data[0].u32w0 = flow->cca_ooo_pkts;
954 break;
955
956 case OPCODE_ERR_CHK_BAD:
957 p_hdr->data[0].u32w0 = host_ipv4addr;
958 p_hdr->data[0].u32w1 = host_pid;
959 break;
960
961 case OPCODE_STARTUP:
962 case OPCODE_STARTUP_ACK:
963 case OPCODE_STARTUP_EXT:
964 case OPCODE_STARTUP_ACK_EXT:
965 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
966 "Unexpected use of old connect protocol");
967 break;
968
969 case OPCODE_CONNECT_REQUEST:
970 case OPCODE_CONNECT_REPLY:
971 p_hdr->hdr_dlen = (epinfo->ep_hdrq_msg_size>>2) -
972 IPS_HEADER_QUEUE_IWORDS - IPS_HEADER_QUEUE_HWORDS;
973 p_hdr->bth[0] = __cpu_to_be32((IPATH_OPCODE_USER1 << 24) +
974 ctrl_path->epr_pkey);
975 paylen =
976 ips_proto_build_connect_message(proto, msg, ipsaddr,
977 message_type, payload);
978 /* Rewrite packet length since this subopcode has an eager payload */
979 tot_paywords += paylen >> 2;
980 p_hdr->lrh[2] = __cpu_to_be16(tot_paywords + SIZE_OF_CRC);
981
982 #if 0 /* MARKDEBBAGE - disabled this as it slows down connect at scale */
983 /* On request message, always set the kpf flag. If reply, only set it
984 * if we know that the recvthread is running */
985 if (message_type == OPCODE_CONNECT_REQUEST ||
986 ipsaddr->flags & SESS_FLAG_HAS_RCVTHREAD)
987 pkt_flags |= INFINIPATH_KPF_INTR;
988 #endif
989 break;
990
991 case OPCODE_DISCONNECT_REQUEST:
992 case OPCODE_DISCONNECT_REPLY:
993 paylen =
994 ips_proto_build_connect_message(proto, msg, ipsaddr,
995 message_type, payload);
996 tot_paywords += paylen >> 2;
997 p_hdr->hdr_dlen = (epinfo->ep_hdrq_msg_size>>2) -
998 IPS_HEADER_QUEUE_IWORDS - IPS_HEADER_QUEUE_HWORDS;
999 p_hdr->lrh[2] = __cpu_to_be16(tot_paywords + SIZE_OF_CRC);
1000 break;
1001
1002 case OPCODE_TIDS_RELEASE:
1003 case OPCODE_TIDS_RELEASE_CONFIRM:
1004 case OPCODE_TIDS_GRANT_ACK:
1005 case OPCODE_TIDS_GRANT:
1006 paylen = ips_protoexp_build_ctrl_message(proto->protoexp, ipsaddr,
1007 p_hdr->data, &pkt_flags, message_type, payload);
1008 if (paylen < 0) {
1009 *discard_msg = 1;
1010 break;
1011 }
1012 tot_paywords += paylen >> 2;
1013 p_hdr->lrh[2] = __cpu_to_be16(tot_paywords + SIZE_OF_CRC);
1014 break;
1015
1016 default:
1017 break;
1018 }
1019
1020 p_hdr->iph.ver_context_tid_offset = __cpu_to_le32(
1021 (IPS_PROTO_VERSION << INFINIPATH_I_VERS_SHIFT) +
1022 (epr->epr_pkt_context << INFINIPATH_I_CONTEXT_SHIFT) +
1023 (IPATH_EAGER_TID_ID << INFINIPATH_I_TID_SHIFT));
1024 p_hdr->iph.pkt_flags = __cpu_to_le16(pkt_flags);
1025
1026 ips_kdeth_cksum(p_hdr); // Generate KDETH checksum
1027
1028 /* Require 4-byte alignment always */
1029 psmi_assert(!(paylen & 0x3));
1030 return paylen;
1031 }
1032
1033 psm_error_t ips_proto_timer_ctrlq_callback(struct psmi_timer *, uint64_t);
1034
1035 psm_error_t __recvpath
1036 ips_proto_send_ctrl_message(struct ips_flow *flow, uint8_t message_type,
1037 uint32_t *msg_queue_mask, void *payload)
1038 {
1039 struct ips_proto_ctrl_message msg;
1040 psm_error_t err = PSM_EP_NO_RESOURCES;
1041 ptl_arg_t *args = (ptl_arg_t *) payload;
1042 ips_epaddr_t *ipsaddr = flow->ipsaddr;
1043 struct ips_proto *proto = ipsaddr->proto;
1044 struct ips_ctrlq *ctrlq = &proto->ctrlq[IPS_FLOWID2INDEX(flow->flowid)&0x3];
1045 struct ips_ctrlq_elem *cqe = ctrlq->ctrlq_cqe;
1046 uint32_t cksum = 0;
1047 int paylen;
1048 uint8_t discard_msg = 0;
1049
1050 /* Drain queue if non-empty */
1051 if (cqe[ctrlq->ctrlq_tail].ipsaddr)
1052 ips_proto_timer_ctrlq_callback(&ctrlq->ctrlq_timer, 0ULL);
1053
1054 if (!cqe[ctrlq->ctrlq_tail].ipsaddr) {
1055 paylen = _build_ctrl_message(proto, &msg, ipsaddr, message_type,
1056 flow, payload, &discard_msg);
1057
1058 if_pt (!discard_msg) {
1059 /* If enabled checksum control message */
1060 ips_do_cksum(proto, &msg.pbc_hdr.hdr, payload, paylen, &cksum);
1061
1062 /* Error check messages are serialized with respect to the underlying
1063 * transfer mechanism.
1064 */
1065 if ((message_type == OPCODE_ERR_CHK) ||
1066 (message_type == OPCODE_ERR_CHK_GEN) ||
1067 (message_type == OPCODE_ERR_CHK_BAD)) {
1068 switch(flow->transfer) {
1069 case PSM_TRANSFER_PIO:
1070 case PSM_TRANSFER_LAST:
1071 err = ips_spio_transfer_frame(proto->spioc, flow, &msg.pbc_hdr.hdr,
1072 payload, paylen, PSMI_TRUE,
1073 (proto->flags & IPS_PROTO_FLAG_CKSUM),
1074 cksum);
1075 break;
1076 case PSM_TRANSFER_DMA:
1077 err = ips_dma_transfer_frame(proto, flow, &msg.pbc_hdr, payload,
1078 paylen, cksum);
1079 break;
1080 }
1081 }
1082 else
1083 if (proto->flags & IPS_PROTO_FLAG_CTRL_SDMA)
1084 err = ips_dma_transfer_frame(proto, flow, &msg.pbc_hdr, payload,
1085 paylen, cksum);
1086 else
1087 err = ips_spio_transfer_frame(proto->spioc, flow, &msg.pbc_hdr.hdr,
1088 payload, paylen, PSMI_TRUE,
1089 (proto->flags & IPS_PROTO_FLAG_CKSUM),
1090 cksum);
1091
1092 if (err == PSM_OK)
1093 ips_epaddr_stats_send(ipsaddr, message_type);
1094 }
1095 else
1096 err = PSM_OK; /* Ctrl message is discarded. May want to add stats */
1097
1098 _IPATH_VDBG("transfer_frame of opcode=0x%x,remote_lid=%d,"
1099 "src=%p,len=%d returns %d\n", (int) msg.pbc_hdr.hdr.sub_opcode,
1100 __be16_to_cpu(msg.pbc_hdr.hdr.lrh[1]), payload, paylen, err);
1101 }
1102 if (err != PSM_EP_NO_RESOURCES)
1103 return err;
1104 if (proto->flags & IPS_PROTO_FLAG_CTRL_SDMA)
1105 proto->stats.writev_busy_cnt++;
1106 else
1107 proto->stats.pio_busy_cnt++;
1108
1109 if (!(proto->ctrl_msg_queue_never_enqueue & proto->message_type_to_index[message_type])) {
1110
1111 if ((*msg_queue_mask) & proto->message_type_to_index[message_type]) {
1112 /* This type of control message is already queued, skip it */
1113 err = PSM_OK;
1114 } else if (cqe[ctrlq->ctrlq_head].ipsaddr == NULL) {
1115 // entry is free
1116 *msg_queue_mask |= message_type2index(proto, message_type);
1117
1118 cqe[ctrlq->ctrlq_head].ipsaddr = ipsaddr;
1119 cqe[ctrlq->ctrlq_head].message_type = message_type;
1120 cqe[ctrlq->ctrlq_head].msg_queue_mask = msg_queue_mask;
1121 cqe[ctrlq->ctrlq_head].flow = flow;
1122
1123 if (args) {
1124 cqe[ctrlq->ctrlq_head].args[0].u64w0 = args[0].u64w0;
1125 cqe[ctrlq->ctrlq_head].args[1].u64w0 = args[1].u64w0;
1126 cqe[ctrlq->ctrlq_head].args[2].u64w0 = args[2].u64w0;
1127 }
1128
1129 ctrlq->ctrlq_head = (ctrlq->ctrlq_head + 1) % CTRL_MSG_QEUEUE_SIZE;
1130 //_IPATH_INFO("requesting ctrlq timer for msgtype=%d!\n", message_type);
1131 psmi_timer_request(proto->timerq, &ctrlq->ctrlq_timer,
1132 PSMI_TIMER_PRIO_0);
1133
1134 err = PSM_OK;
1135 } else {
1136 proto->ctrl_msg_queue_overflow++;
1137 }
1138 }
1139
1140 return err;
1141 }
1142
1143 psm_error_t __recvpath
1144 ips_proto_timer_ctrlq_callback(struct psmi_timer *timer, uint64_t t_cyc_expire)
1145 {
1146 struct ips_ctrlq *ctrlq = (struct ips_ctrlq *) timer->context;
1147 struct ips_proto *proto = ctrlq->ctrlq_proto;
1148 struct ips_proto_ctrl_message msg;
1149 struct ips_ctrlq_elem *cqe = ctrlq->ctrlq_cqe;
1150 struct ips_flow *flow;
1151 uint8_t msg_type;
1152 psm_error_t err;
1153 struct ptl_epaddr *ipsaddr;
1154 uint32_t cksum = 0;
1155 int paylen;
1156 uint8_t discard_msg = 0;
1157
1158 // service ctrl send queue first
1159 while (cqe[ctrlq->ctrlq_tail].ipsaddr) {
1160 msg_type = cqe[ctrlq->ctrlq_tail].message_type;
1161 ipsaddr = cqe[ctrlq->ctrlq_tail].ipsaddr;
1162 flow = cqe[ctrlq->ctrlq_tail].flow;
1163
1164 paylen = _build_ctrl_message(proto, &msg,
1165 ipsaddr, msg_type, flow,
1166 cqe[ctrlq->ctrlq_tail].args,
1167 &discard_msg);
1168
1169 psmi_assert_always(paylen == 0);
1170
1171 if_pt (!discard_msg) {
1172 /* If enabled checksum control message */
1173 ips_do_cksum(proto, &msg.pbc_hdr.hdr, NULL, 0, &cksum);
1174
1175 /* Error check messages are serialized with respect to the underlying
1176 * transfer mechanism.
1177 */
1178 if ((msg_type == OPCODE_ERR_CHK) ||
1179 (msg_type == OPCODE_ERR_CHK_GEN) ||
1180 (msg_type == OPCODE_ERR_CHK_BAD)) {
1181 switch(flow->transfer) {
1182 case PSM_TRANSFER_DMA:
1183 err = ips_dma_transfer_frame(proto,flow,&msg.pbc_hdr,0,0, cksum);
1184 break;
1185 case PSM_TRANSFER_PIO:
1186 default:
1187 err =
1188 ips_spio_transfer_frame(proto->spioc, flow, &msg.pbc_hdr.hdr,
1189 NULL, 0, PSMI_TRUE,
1190 (proto->flags & IPS_PROTO_FLAG_CKSUM),
1191 cksum);
1192 break;
1193 }
1194 }
1195 else
1196 if (proto->flags & IPS_PROTO_FLAG_CTRL_SDMA)
1197 err = ips_dma_transfer_frame(proto,flow,&msg.pbc_hdr,NULL,0,cksum);
1198 else
1199 err =
1200 ips_spio_transfer_frame(proto->spioc, flow, &msg.pbc_hdr.hdr,
1201 0, 0, PSMI_TRUE,
1202 (proto->flags & IPS_PROTO_FLAG_CKSUM),
1203 cksum);
1204 }
1205 else
1206 err = PSM_OK; /* Discard ctrl message */
1207
1208 if (err == PSM_OK) {
1209 ips_epaddr_stats_send(ipsaddr, msg_type);
1210 *cqe[ctrlq->ctrlq_tail].msg_queue_mask &=
1211 ~message_type2index(proto, cqe[ctrlq->ctrlq_tail].message_type);
1212 cqe[ctrlq->ctrlq_tail].ipsaddr = NULL;
1213 ctrlq->ctrlq_tail = (ctrlq->ctrlq_tail + 1) % CTRL_MSG_QEUEUE_SIZE;
1214 } else {
1215 psmi_assert(err == PSM_EP_NO_RESOURCES);
1216
1217 if (proto->flags & IPS_PROTO_FLAG_CTRL_SDMA)
1218 proto->stats.writev_busy_cnt++;
1219 else
1220 proto->stats.pio_busy_cnt++;
1221 /* re-request a timer expiration */
1222 psmi_timer_request(proto->timerq, &ctrlq->ctrlq_timer,
1223 PSMI_TIMER_PRIO_0);
1224 return PSM_OK;
1225 }
1226 }
1227
1228 return PSM_OK;
1229 }
1230
1231 void __sendpath
1232 ips_proto_flow_enqueue(struct ips_flow *flow, ips_scb_t *scb)
1233 {
1234 ips_epaddr_t *ipsaddr = flow->ipsaddr;
1235
1236 /* Don't support send to self */
1237 psmi_assert(flow->path->epr_dlid != flow->path->epr_slid);
1238
1239 ips_scb_prepare_flow_inner(scb, flow->epinfo, &ipsaddr->epr, flow);
1240 ips_do_cksum(ipsaddr->proto, &scb->ips_lrh,
1241 scb->payload, scb->payload_size, &scb->cksum);
1242
1243 STAILQ_INSERT_TAIL(&flow->scb_unacked, scb, nextq);
1244 flow->scb_num_pending++;
1245 flow->scb_num_unacked++;
1246
1247 /* Every ipsaddr has a pending head that points into the unacked queue.
1248 * If sends are already pending, process those first */
1249 if (SLIST_EMPTY(&flow->scb_pend))
1250 SLIST_FIRST(&flow->scb_pend) = scb;
1251 }
1252
1253 /*
1254 * This function attempts to flush the current list of pending
1255 * packets through PIO.
1256 *
1257 * Recoverable errors:
1258 * PSM_OK: Packet triggered through PIO.
1259 * PSM_EP_NO_RESOURCES: No PIO bufs available or cable pulled.
1260 *
1261 * Unrecoverable errors:
1262 * PSM_EP_NO_NETWORK: No network, no lid, ...
1263 * PSM_EP_DEVICE_FAILURE: Chip failures, rxe/txe parity, etc.
1264 */
1265 psm_error_t __sendpath
1266 ips_proto_flow_flush_pio(struct ips_flow *flow, int *nflushed)
1267 {
1268 struct ips_proto *proto = flow->ipsaddr->proto;
1269 struct ips_scb_pendlist *scb_pend = &flow->scb_pend;
1270 int num_sent = 0;
1271 uint64_t t_cyc;
1272 ips_scb_t *scb;
1273 psm_error_t err = PSM_OK;
1274
1275 /* Out of credits - ACKs/NAKs reclaim recredit or congested flow */
1276 if_pf ((!flow->credits) || (flow->flags & IPS_FLOW_FLAG_CONGESTED))
1277 return PSM_OK;
1278
1279 while (!SLIST_EMPTY(scb_pend) && flow->credits) {
1280 scb = SLIST_FIRST(scb_pend);
1281
1282 if ((err = ips_spio_transfer_frame(proto->spioc, flow, &scb->ips_lrh,
1283 scb->payload, scb->payload_size,
1284 PSMI_FALSE,
1285 (proto->flags & IPS_PROTO_FLAG_CKSUM) && (scb->tid == IPATH_EAGER_TID_ID),
1286 scb->cksum)) == PSM_OK)
1287 {
1288 t_cyc = get_cycles();
1289 scb->flags &= ~IPS_SEND_FLAG_PENDING;
1290 scb->ack_timeout = flow->path->epr_timeout_ack;
1291 scb->abs_timeout = flow->path->epr_timeout_ack + t_cyc;
1292 psmi_timer_request(proto->timerq, &flow->timer_ack,
1293 scb->abs_timeout);
1294 num_sent++;
1295 flow->scb_num_pending--;
1296 flow->credits--;
1297 SLIST_REMOVE_HEAD(scb_pend, next);
1298
1299 }
1300 else
1301 break;
1302 }
1303
1304 /* If out of flow credits re-schedule send timer */
1305 if (!SLIST_EMPTY(scb_pend)) {
1306 proto->stats.pio_busy_cnt++;
1307 psmi_timer_request(proto->timerq, &flow->timer_send,
1308 get_cycles() + proto->timeout_send);
1309 }
1310
1311 if (nflushed != NULL)
1312 *nflushed = num_sent;
1313
1314 return err;
1315 }
1316
1317 /*
1318 * Flush all packets currently marked as pending
1319 */
1320 static psm_error_t scb_dma_send(struct ips_proto *proto, struct ips_flow *flow,
1321 struct ips_scb_pendlist *slist, int num,
1322 int *num_sent);
1323
1324 #ifdef PSM_DEBUG
1325 #define PSM_DEBUG_CHECK_INFLIGHT_CNTR(proto) \
1326 do { \
1327 uint32_t cntr_inflight; \
1328 ipath_sdma_inflight(proto->ptl->context->ctrl, &cntr_inflight); \
1329 VALGRIND_MAKE_MEM_DEFINED(&cntr_inflight, sizeof(uint32_t)); \
1330 psmi_assert_always(cntr_inflight == \
1331 proto->iovec_cntr_next_inflight); \
1332 } while (0)
1333 #else
1334 #define PSM_DEBUG_CHECK_INFLIGHT_CNTR(proto)
1335 #endif
1336
1337 /*
1338 * Flush all packets queued up on a flow via send DMA.
1339 *
1340 * Recoverable errors:
1341 * PSM_OK: Able to flush entire pending queue for DMA.
1342 * PSM_OK_NO_PROGRESS: Flushed at least 1 but not all pending packets for DMA.
1343 * PSM_EP_NO_RESOURCES: No scb's available to handle unaligned packets
1344 * or writev returned a recoverable error (no mem for
1345 * descriptors, dma interrupted or no space left in dma
1346 * queue).
1347 *
1348 * Unrecoverable errors:
1349 * PSM_EP_DEVICE_FAILURE: Unexpected error calling writev(), chip failure,
1350 * rxe/txe parity error.
1351 * PSM_EP_NO_NETWORK: No network, no lid, ...
1352 */
1353 psm_error_t __sendpath
1354 ips_proto_flow_flush_dma(struct ips_flow *flow, int *nflushed)
1355 {
1356 struct ips_proto *proto = flow->ipsaddr->proto;
1357 struct ips_scb_pendlist *scb_pend = &flow->scb_pend;
1358 uint32_t cntr_init;
1359 ips_scb_t *scb;
1360 psm_error_t err = PSM_OK;
1361 int howmany = 0;
1362 int nsent = 0;
1363
1364 /* Out of credits - ACKs/NAKs reclaim recredit or congested flow */
1365 if_pf ((!flow->credits) || (flow->flags & IPS_FLOW_FLAG_CONGESTED)) {
1366 if (nflushed)
1367 *nflushed = 0;
1368 return PSM_EP_NO_RESOURCES;
1369 }
1370
1371 if (SLIST_EMPTY(scb_pend))
1372 goto success;
1373
1374 /*
1375 * Count how many are to be sent and fire dma.
1376 */
1377 #ifdef PSM_DEBUG
1378 SLIST_FOREACH(scb, scb_pend, next)
1379 howmany++;
1380 psmi_assert_always(howmany == flow->scb_num_pending);
1381 #else
1382 howmany = min(flow->scb_num_pending, flow->credits);
1383 #endif
1384
1385 howmany = min(howmany, proto->scb_max_sdma);
1386
1387 if (howmany == 0)
1388 goto success;
1389
1390 PSM_DEBUG_CHECK_INFLIGHT_CNTR(proto); /* Pre-check */
1391
1392 cntr_init = proto->iovec_cntr_next_inflight;
1393 err = scb_dma_send(proto, flow, scb_pend, howmany, &nsent);
1394 if (err != PSM_OK && err != PSM_EP_NO_RESOURCES &&
1395 err != PSM_OK_NO_PROGRESS)
1396 goto fail;
1397
1398 /* scb_dma_send shouldn't modify iovec_cntr_next_inflight */
1399 psmi_assert_always(cntr_init == proto->iovec_cntr_next_inflight);
1400
1401 if (nsent > 0) {
1402 uint64_t t_cyc = get_cycles();
1403 uint32_t new_inflight = proto->iovec_cntr_next_inflight + nsent;
1404 int i = 0;
1405
1406 /* We have to ensure that the inflight counter doesn't drift away too
1407 * far from the completion counter or else our wraparound arithmetic
1408 * in ips_proto_dma_wait_until will fail.
1409 */
1410 if ((int) new_inflight - (int) proto->iovec_cntr_last_completed < 0)
1411 ips_proto_dma_wait_until(proto,
1412 proto->iovec_cntr_last_completed + nsent);
1413
1414 flow->scb_num_pending -= nsent;
1415 flow->credits = max((int) flow->credits - nsent, 0);
1416
1417 SLIST_FOREACH(scb, scb_pend, next) {
1418 if (++i > nsent)
1419 break;
1420 scb->flags &= ~IPS_SEND_FLAG_PENDING;
1421 scb->ack_timeout = scb->nfrag*flow->path->epr_timeout_ack;
1422 scb->abs_timeout = scb->nfrag*flow->path->epr_timeout_ack + t_cyc;
1423 scb->dma_ctr = proto->iovec_cntr_next_inflight++;
1424 if (scb->tidsendc)
1425 ips_protoexp_scb_inflight(scb);
1426 }
1427 SLIST_FIRST(scb_pend) = scb;
1428 }
1429
1430 PSM_DEBUG_CHECK_INFLIGHT_CNTR(proto); /* Post Check */
1431
1432 if (SLIST_FIRST(scb_pend) != NULL) {
1433 psmi_assert(flow->scb_num_pending > 0);
1434
1435 switch(flow->protocol) {
1436 case PSM_PROTOCOL_TIDFLOW:
1437 /* For Tidflow we can cancel the ack timer if we have flow credits
1438 * available and schedule the send timer. If we are out of flow
1439 * credits then the ack timer is scheduled as we are waiting for
1440 * an ACK to reclaim credits. This is required since multiple
1441 * tidflows may be active concurrently.
1442 */
1443 if (flow->credits) {
1444 /* Cancel ack timer and reschedule send timer. Increment
1445 * writev_busy_cnt as this really is DMA buffer exhaustion.
1446 */
1447 psmi_timer_cancel(proto->timerq, &flow->timer_ack);
1448 psmi_timer_request(proto->timerq, &flow->timer_send,
1449 get_cycles() + (proto->timeout_send << 1));
1450 proto->stats.writev_busy_cnt++;
1451 }
1452 else {
1453 /* Re-instate ACK timer to reap flow credits */
1454 psmi_timer_request(proto->timerq, &flow->timer_ack,
1455 get_cycles() + (flow->path->epr_timeout_ack>>2));
1456 }
1457
1458 break;
1459 case PSM_PROTOCOL_GO_BACK_N:
1460 default:
1461 if (flow->credits) {
1462 /* Schedule send timer and increment writev_busy_cnt */
1463 psmi_timer_request(proto->timerq, &flow->timer_send,
1464 get_cycles() + (proto->timeout_send << 1));
1465 proto->stats.writev_busy_cnt++;
1466 }
1467 else {
1468 /* Schedule ACK timer to reap flow credits */
1469 psmi_timer_request(proto->timerq, &flow->timer_ack,
1470 get_cycles() + (flow->path->epr_timeout_ack>>2));
1471 }
1472 break;
1473 }
1474 }
1475 else {
1476 /* Schedule ack timer */
1477 psmi_timer_cancel(proto->timerq, &flow->timer_send);
1478 psmi_timer_request(proto->timerq, &flow->timer_ack,
1479 get_cycles() + flow->path->epr_timeout_ack);
1480 }
1481
1482 /* We overwrite error with its new meaning for flushing packets */
1483 if (nsent > 0)
1484 if (nsent < howmany)
1485 err = PSM_OK_NO_PROGRESS; /* partial flush */
1486 else
1487 err = PSM_OK; /* complete flush */
1488 else
1489 err = PSM_EP_NO_RESOURCES; /* no flush at all */
1490
1491 success:
1492 fail:
1493 if (nflushed)
1494 *nflushed = nsent;
1495
1496 return err;
1497 }
1498
1499 /*
1500 * Fault injection in dma sends. Since DMA through writev() is all-or-nothing,
1501 * we don't inject faults on a packet-per-packet basis since the code gets
1502 * quite complex. Instead, each call to flush_dma or transfer_frame is treated
1503 * as an "event" and faults are generated according to the IPS_FAULTINJ_DMASEND
1504 * setting.
1505 *
1506 * The effect is as if the event was successful but dropped on the wire
1507 * somewhere.
1508 */
1509 PSMI_ALWAYS_INLINE(
1510 int
1511 dma_do_fault())
1512 {
1513
1514 if_pf (PSMI_FAULTINJ_ENABLED()) {
1515 PSMI_FAULTINJ_STATIC_DECL(fi, "dmalost", 1, IPS_FAULTINJ_DMALOST);
1516 return psmi_faultinj_is_fault(fi);
1517 }
1518 else
1519 return 0;
1520 }
1521
1522 /* ips_dma_transfer_frame is used only for control messages, and is
1523 * not enabled by default, and not tested by QA; expected send
1524 * dma goes through scb_dma_send() */
1525 psm_error_t __sendpath
1526 ips_dma_transfer_frame(struct ips_proto *proto, struct ips_flow *flow,
1527 struct ips_pbc_header *pbc_hdr_i,
1528 void *payload, uint32_t paylen, uint32_t cksum)
1529 {
1530 struct iovec iovec;
1531 ssize_t ret;
1532 psm_error_t err;
1533 uint32_t have_cksum =
1534 ((proto->flags & IPS_PROTO_FLAG_CKSUM) &&
1535 (((__le32_to_cpu(pbc_hdr_i->hdr.iph.ver_context_tid_offset) >> INFINIPATH_I_TID_SHIFT) & INFINIPATH_I_TID_MASK) == IPATH_EAGER_TID_ID) && (pbc_hdr_i->hdr.mqhdr != MQ_MSG_DATA_BLK) && (pbc_hdr_i->hdr.mqhdr != MQ_MSG_DATA_REQ_BLK));
1536
1537 psmi_assert((paylen & 0x3) == 0); /* require 4-byte multiple */
1538 psmi_assert(((uintptr_t) payload & 0x3) == 0); /* require 4-byte alignment */
1539 psmi_assert(paylen < proto->epinfo.ep_mtu);
1540
1541 /* See comments above for fault injection */
1542 if_pf (dma_do_fault())
1543 return PSM_OK;
1544
1545 ips_proto_pbc_update(proto, flow, PSMI_TRUE, &pbc_hdr_i->pbc,
1546 sizeof(struct ips_message_header),
1547 payload, paylen +
1548 (have_cksum ? PSM_CRC_SIZE_IN_BYTES : 0));
1549
1550 /* If we have a payload, we need to copy it inline to a single element to
1551 * ensure that the driver copies it out completely as part of the writev
1552 * call since the payload can be stack-allocated memory.
1553 */
1554 if (paylen > 0) {
1555 uint32_t len = sizeof(struct ips_pbc_header) +
1556 paylen + (have_cksum ? PSM_CRC_SIZE_IN_BYTES : 0);
1557 struct ips_pbc_header *pbc_hdr = alloca(len);
1558
1559 if_pf (pbc_hdr == NULL) {
1560 err = psmi_handle_error(PSMI_EP_NORETURN, PSM_NO_MEMORY,
1561 "alloca for %d bytes failed in writev", len);
1562 goto fail;
1563 }
1564
1565 psmi_mq_mtucpy(pbc_hdr, pbc_hdr_i, sizeof(struct ips_pbc_header));
1566 psmi_mq_mtucpy(pbc_hdr+1, payload, paylen);
1567
1568 if (have_cksum) {
1569 uint32_t *ckptr = (uint32_t*) ((uint8_t*) pbc_hdr +
1570 (len - PSM_CRC_SIZE_IN_BYTES));
1571 *ckptr = cksum;
1572 ckptr++;
1573 *ckptr = cksum;
1574 }
1575
1576 iovec.iov_base = pbc_hdr;
1577 iovec.iov_len = len;
1578 ret = ipath_cmd_writev(proto->fd, &iovec, 1);
1579 }
1580 else {
1581 uint32_t len = sizeof(struct ips_pbc_header) +
1582 (have_cksum ? PSM_CRC_SIZE_IN_BYTES : 0);
1583 struct ips_pbc_header *pbc_hdr = have_cksum ? alloca(len) : pbc_hdr_i;
1584
1585 if_pf (pbc_hdr == NULL) {
1586 err = psmi_handle_error(PSMI_EP_NORETURN, PSM_NO_MEMORY,
1587 "alloca for %d bytes failed in writev", len);
1588 goto fail;
1589 }
1590
1591 if (have_cksum) {
1592 uint32_t *ckptr = (uint32_t*) (pbc_hdr + 1);
1593 psmi_mq_mtucpy(pbc_hdr, pbc_hdr_i, sizeof(struct ips_pbc_header));
1594 *ckptr = cksum;
1595 ckptr++;
1596 *ckptr = cksum;
1597 }
1598
1599 iovec.iov_base = pbc_hdr;
1600 iovec.iov_len = len;
1601 ret = ipath_cmd_writev(proto->fd, &iovec, 1);
1602 }
1603
1604 if (ret > 0) {
1605 /* Even though we won't care about a completion in this frame send, we
1606 * still increment the iovec packet counter */
1607 proto->iovec_cntr_next_inflight += ret;
1608 err = PSM_OK;
1609 psmi_assert_always(ret == 1);
1610 }
1611 else {
1612 /*
1613 * ret == 0: Driver did not queue packet. Try later.
1614 * ENOMEM: No kernel memory to queue request, try later? *
1615 * ECOMM: Link may have gone down
1616 * EINTR: Got interrupt while in writev
1617 */
1618 if (ret == 0 || errno == ENOMEM || errno == ECOMM || errno == EINTR)
1619 err = PSM_EP_NO_RESOURCES;
1620 else
1621 err = psmi_handle_error(proto->ep, PSM_EP_DEVICE_FAILURE,
1622 "Unhandled error in writev(): %s (fd=%d,iovec=%p,len=%d)",
1623 strerror(errno), proto->fd, &iovec, 1);
1624 }
1625
1626 fail:
1627 return err;
1628 }
1629
1630 /*
1631 * Caller still expects num_sent to always be correctly set in case of an
1632 * error.
1633 *
1634 * Recoverable errors:
1635 * PSM_OK: At least one packet was successfully queued up for DMA.
1636 * PSM_EP_NO_RESOURCES: No scb's available to handle unaligned packets
1637 * or writev returned a recoverable error (no mem for
1638 * descriptors, dma interrupted or no space left in dma
1639 * queue).
1640 * PSM_OK_NO_PROGRESS: Cable pulled.
1641 *
1642 * Unrecoverable errors:
1643 * PSM_EP_DEVICE_FAILURE: Error calling ipath_sdma_inflight() or unexpected
1644 * error in calling writev(), or chip failure, rxe/txe
1645 * parity error.
1646 * PSM_EP_NO_NETWORK: No network, no lid, ...
1647 */
1648 static
1649 psm_error_t __sendpath
1650 scb_dma_send(struct ips_proto *proto, struct ips_flow *flow,
1651 struct ips_scb_pendlist *slist, int num, int *num_sent)
1652 {
1653 ssize_t ret;
1654 struct ips_scb *scb = SLIST_FIRST(slist);
1655 unsigned int vec_idx = 0, scb_idx = 0, scb_sent = 0;
1656 unsigned int max_elem;
1657 struct iovec *iovec;
1658 psm_error_t err = PSM_OK;
1659 uint32_t cksum;
1660
1661 psmi_assert(num > 0);
1662 psmi_assert(scb != NULL);
1663
1664 /* See comments above for fault injection */
1665 if_pf (dma_do_fault())
1666 goto fail;
1667
1668 max_elem = 3*num;
1669 iovec = alloca(sizeof(struct iovec) * max_elem);
1670
1671 if_pf (iovec == NULL) {
1672 err = psmi_handle_error(PSMI_EP_NORETURN, PSM_NO_MEMORY,
1673 "alloca for %d bytes failed in writev",
1674 (int)(sizeof(struct iovec) * max_elem));
1675 goto fail;
1676 }
1677
1678 writev_again:
1679 vec_idx = 0;
1680
1681 SLIST_FOREACH(scb, slist, next) {
1682 /* Can't exceed posix max writev count */
1683 if (vec_idx + (int) !!(scb->payload_size > 0) >= UIO_MAXIOV)
1684 break;
1685
1686 psmi_assert(vec_idx < max_elem);
1687 psmi_assert_always((scb->payload_size & 0x3) == 0);
1688
1689 /* Checksum all eager packets */
1690 cksum = ((proto->flags & IPS_PROTO_FLAG_CKSUM) &&
1691 (scb->tid == IPATH_EAGER_TID_ID) &&
1692 (scb->ips_lrh.mqhdr != MQ_MSG_DATA_BLK) &&
1693 (scb->ips_lrh.mqhdr != MQ_MSG_DATA_REQ_BLK));
1694
1695 ips_proto_pbc_update(proto, flow, PSMI_FALSE, &scb->pbc,
1696 sizeof(struct ips_message_header),
1697 scb->payload,
1698 scb->payload_size +
1699 (cksum ? PSM_CRC_SIZE_IN_BYTES : 0));
1700
1701 iovec[vec_idx].iov_base = &scb->pbc;
1702 iovec[vec_idx].iov_len = sizeof(struct ips_message_header) +
1703 sizeof(union ipath_pbc);
1704 vec_idx++;
1705
1706 if (scb->payload_size > 0) {
1707 /*
1708 * Payloads must be 4-byte aligned. If not, we need a bounce
1709 * buffer for them. This should be rare, but may be a performance
1710 * penalty, so we log it as a stat in case we need to narrow in
1711 * on a performance problem.
1712 *
1713 * If checksum is enabled use a bounce buffer.
1714 */
1715 if ((((uintptr_t) scb->payload) & 0x3) || cksum) {
1716 void *buf = scb->payload;
1717 uint32_t len = scb->payload_size;
1718
1719 if (scb->nfrag > 1) {
1720 err = psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
1721 "buffer alignment for sdma error");
1722 goto fail;
1723 }
1724
1725 /* Only allocate buffer if current buffer is a user buffer */
1726 if (!((scb->payload >= scb->scbc->sbuf_buf_base) &&
1727 (scb->payload <= scb->scbc->sbuf_buf_last))){
1728
1729 if (!ips_scbctrl_bufalloc(scb)) {
1730 err = PSM_EP_NO_RESOURCES;
1731 if (--vec_idx == 0) /* Remove header, nothing to send */
1732 goto fail;
1733 else /* send what we have so far, but no more */
1734 break;
1735 }
1736
1737 /* Only need to copy if bounce buffer is used. */
1738 psmi_mq_mtucpy(scb->payload, buf, len);
1739 scb->payload_size = len;
1740 }
1741
1742 /* If checksum then update checksum */
1743 if (cksum) {
1744 uint32_t *ckptr = (uint32_t*) ((uint8_t*) scb->payload + len);
1745 *ckptr = scb->cksum;
1746 ckptr++;
1747 *ckptr = scb->cksum;
1748 }
1749
1750 if (((uintptr_t) buf) & 0x3)
1751 proto->stats.send_dma_misaligns++;
1752 }
1753
1754 iovec[vec_idx].iov_base = scb->payload;
1755 iovec[vec_idx].iov_len = scb->payload_size +
1756 (cksum ? PSM_CRC_SIZE_IN_BYTES : 0);
1757 vec_idx++;
1758
1759 _IPATH_VDBG("seqno=%d hdr=%p,%d payload=%p,%d\n",
1760 scb->seq_num.psn,
1761 iovec[vec_idx-2].iov_base, (int) iovec[vec_idx-2].iov_len,
1762 iovec[vec_idx-1].iov_base, (int) iovec[vec_idx-1].iov_len);
1763
1764 /*
1765 * if there are multiple frag payload, set the right frag size.
1766 */
1767 if (scb->nfrag > 1) {
1768 scb->pbc.fill1 = __cpu_to_le16(scb->frag_size);
1769
1770 /* give tidinfo to qib driver */
1771 if (scb->tidsendc) {
1772 iovec[vec_idx].iov_base = scb->tsess;
1773 iovec[vec_idx].iov_len = scb->tsess_length;
1774 vec_idx++;
1775 }
1776 }
1777 }
1778 else {
1779 /* If checksum enabled need to send checksum at end of header
1780 * as we have no payload.
1781 */
1782 if (cksum) {
1783 char *pbc_hdr = alloca(iovec[vec_idx-1].iov_len +
1784 PSM_CRC_SIZE_IN_BYTES);
1785 uint32_t *ckptr = (uint32_t*)
1786 ((uint8_t*) pbc_hdr + iovec[vec_idx-1].iov_len);
1787
1788 psmi_mq_mtucpy(pbc_hdr, iovec[vec_idx-1].iov_base,iovec[vec_idx-1].iov_len);
1789 *ckptr = scb->cksum;
1790 ckptr++;
1791 *ckptr = scb->cksum;
1792
1793 iovec[vec_idx-1].iov_base = pbc_hdr;
1794 iovec[vec_idx-1].iov_len += PSM_CRC_SIZE_IN_BYTES;
1795
1796 }
1797
1798 _IPATH_VDBG("hdr=%p,%d\n",
1799 iovec[vec_idx-1].iov_base, (int) iovec[vec_idx-1].iov_len);
1800 }
1801
1802 /* Can bound the number to send by 'num' */
1803 if (++scb_idx == num)
1804 break;
1805 }
1806 psmi_assert(vec_idx > 0);
1807 ret = ipath_cmd_writev(proto->fd, iovec, vec_idx);
1808
1809 /*
1810 * Successfully wrote entire vector
1811 */
1812 if (ret == scb_idx) {
1813 scb_sent += ret;
1814 /* scbs are left if we didn't want to send less and didn't have
1815 * to break out of scbctrl_bufalloc */
1816 if (scb != NULL && scb_idx < num && err == PSM_OK)
1817 goto writev_again;
1818 }
1819 else {
1820 if (ret < 0) {
1821 uint32_t cntr_fini;
1822
1823 /* ENOMEM: No kernel memory to queue request, try later?
1824 * ECOMM: Link may have gone down
1825 * EINTR: Got interrupt while in writev
1826 */
1827 if (errno == ENOMEM || errno == ECOMM || errno == EINTR) {
1828 err = psmi_context_check_status(
1829 (const psmi_context_t *) &proto->ep->context);
1830 if (err == PSM_OK)
1831 err = PSM_EP_NO_RESOURCES;
1832 }
1833 else {
1834 err = psmi_handle_error(proto->ep, PSM_EP_DEVICE_FAILURE,
1835 "Unexpected error in writev(): %s (errno=%d) "
1836 "(fd=%d,iovec=%p,len=%d)", strerror(errno), errno,
1837 proto->fd, iovec, vec_idx);
1838 goto fail;
1839 }
1840 /* Find out the latest packet that we were able to put in flight */
1841 if (ipath_sdma_inflight(proto->ptl->context->ctrl, &cntr_fini) < 0)
1842 {
1843 err = psmi_handle_error(proto->ep, PSM_EP_DEVICE_FAILURE,
1844 "Unable to retrieve inflight sdma counter: %s",
1845 strerror(errno));
1846 goto fail;
1847 }
1848
1849 /* Re-write ret to actual inflight count */
1850 scb_sent += cntr_fini - proto->iovec_cntr_next_inflight;
1851 }
1852 else {
1853 /* No need for inflight system call, we can infer it's value from
1854 * writev's return value */
1855 scb_sent += ret;
1856 }
1857 }
1858
1859 fail:
1860 *num_sent = scb_sent;
1861 psmi_assert(*num_sent <= num && *num_sent >= 0);
1862 return err;
1863 }
1864
1865 /*
1866 * Because we only lazily reap send dma completions, it's possible that we
1867 * receive a packet's remote acknowledgement before seeing that packet's local
1868 * completion. As part of processing ack packets and releasing scbs, we issue
1869 * a wait for the local completion if the scb is marked as having been sent via
1870 * send dma.
1871 */
1872 psm_error_t __sendpath
1873 ips_proto_dma_wait_until(struct ips_proto *proto, uint32_t dma_cntr)
1874 {
1875 psm_error_t err = PSM_OK;
1876 int spin_cnt = 0;
1877 int did_yield = 0;
1878
1879 PSM_DEBUG_CHECK_INFLIGHT_CNTR(proto);
1880
1881 if ((int) proto->iovec_cntr_last_completed - (int) dma_cntr >= 0)
1882 return PSM_OK;
1883
1884 PSMI_PROFILE_BLOCK();
1885
1886 while ((int) proto->iovec_cntr_last_completed - (int) dma_cntr < 0)
1887 {
1888 if (spin_cnt++ == proto->ep->yield_spin_cnt) {
1889 /* Have to yield holding the PSM lock, mostly because we don't
1890 * support another thread changing internal state at this point in
1891 * the code.
1892 */
1893 did_yield = 1;
1894 sched_yield();
1895 }
1896
1897 /* Not there yet in completion count. Update our view of
1898 * last_completed. */
1899 if (ipath_sdma_complete(proto->ptl->context->ctrl,
1900 &proto->iovec_cntr_last_completed) == -1)
1901 {
1902 err = psmi_handle_error(proto->ep, PSM_EP_DEVICE_FAILURE,
1903 "unable to retrieve completion sdma counter: %s",
1904 strerror(errno));
1905 break;
1906 }
1907 }
1908
1909 if (did_yield)
1910 proto->stats.writev_compl_delay++;
1911
1912 PSMI_PROFILE_UNBLOCK();
1913
1914 return err;
1915 }
1916
1917 #define ERRCHK_NOT_SERIALIZED 1
1918
1919 psm_error_t
1920 ips_proto_timer_ack_callback(struct psmi_timer *current_timer, uint64_t current)
1921 {
1922 struct ips_flow *flow = (struct ips_flow *) current_timer->context;
1923 ips_epaddr_t *ipsaddr = flow->ipsaddr;
1924 struct ips_proto *proto = ipsaddr->proto;
1925 uint64_t t_cyc_next = get_cycles();
1926 ips_scb_t *scb;
1927
1928 if (STAILQ_EMPTY(&flow->scb_unacked))
1929 return PSM_OK;
1930
1931 scb = STAILQ_FIRST(&flow->scb_unacked);
1932
1933 if (current >= scb->abs_timeout) {
1934 int done_local;
1935
1936 #if ERRCHK_NOT_SERIALIZED
1937 /* We have to ensure that the send is at least locally complete before
1938 * sending an error check or else earlier data can get to the
1939 * destination *after* we pio this err_chk.
1940 */
1941 if (flow->transfer == PSM_TRANSFER_DMA) {
1942 uint32_t dma_cntr;
1943 uint32_t scb_cntr =
1944 STAILQ_LAST(&flow->scb_unacked, ips_scb, nextq)->dma_ctr;
1945 done_local =
1946 (ipath_sdma_complete(proto->ptl->context->ctrl, &dma_cntr) > 0 &&
1947 ((int) dma_cntr - (int) scb_cntr >= 0));
1948 if (!done_local)
1949 proto->stats.writev_compl_eagain++;
1950 }
1951 else
1952 done_local = 1; /* Always done for PIO flows */
1953 #else
1954 done_local = 1; /* Otherwise always done */
1955 #endif
1956
1957 scb->ack_timeout =
1958 min(scb->ack_timeout * flow->path->epr_timeout_ack_factor,
1959 flow->path->epr_timeout_ack_max);
1960 scb->abs_timeout = t_cyc_next + scb->ack_timeout;
1961
1962 if (done_local) {
1963 _IPATH_VDBG("sending err_chk flow=%d with first=%d,last=%d\n",
1964 flow->flowid, STAILQ_FIRST(&flow->scb_unacked)->seq_num.psn,
1965 STAILQ_LAST(&flow->scb_unacked, ips_scb, nextq)->seq_num.psn);
1966
1967 if (flow->protocol == PSM_PROTOCOL_TIDFLOW)
1968 ips_proto_send_ctrl_message(flow,
1969 OPCODE_ERR_CHK_GEN,
1970 &scb->tidsendc->ctrl_msg_queued,
1971 NULL);
1972 else
1973 ips_proto_send_ctrl_message(flow,
1974 OPCODE_ERR_CHK,
1975 &flow->ipsaddr->ctrl_msg_queued,
1976 NULL);
1977 }
1978
1979 t_cyc_next = get_cycles() + scb->ack_timeout;
1980 }
1981 else
1982 t_cyc_next += (scb->abs_timeout - current);
1983
1984 psmi_timer_request(proto->timerq, current_timer, t_cyc_next);
1985
1986 return PSM_OK;
1987 }
1988
1989 psm_error_t
1990 ips_proto_timer_send_callback(struct psmi_timer *current_timer, uint64_t current)
1991 {
1992 struct ips_flow *flow = (struct ips_flow *) current_timer->context;
1993
1994 /* If flow is marked as congested adjust injection rate - see process nak
1995 * when a congestion NAK is received.
1996 */
1997 if_pf (flow->flags & IPS_FLOW_FLAG_CONGESTED) {
1998 struct ips_proto *proto = flow->ipsaddr->proto;
1999
2000 /* Clear congestion flag and decrease injection rate */
2001 flow->flags &= ~IPS_FLOW_FLAG_CONGESTED;
2002 if ((flow->path->epr_ccti +
2003 proto->cace[flow->path->epr_sl].ccti_increase) <=
2004 proto->ccti_limit)
2005 ips_cca_adjust_rate(flow->path,
2006 proto->cace[flow->path->epr_sl].ccti_increase);
2007 }
2008
2009 flow->fn.xfer.flush(flow, NULL);
2010 return PSM_OK;
2011 }
2012
2013 psm_error_t
2014 ips_cca_adjust_rate(ips_path_rec_t *path_rec, int cct_increment)
2015 {
2016 struct ips_proto *proto = path_rec->proto;
2017 uint16_t prev_ipd, prev_divisor;
2018
2019 /* Increment/decrement ccti for path */
2020 psmi_assert_always(path_rec->epr_ccti >= path_rec->epr_ccti_min);
2021 path_rec->epr_ccti += cct_increment;
2022
2023 /* Determine new active IPD. */
2024 prev_ipd = path_rec->epr_active_ipd;
2025 prev_divisor = path_rec->epr_cca_divisor;
2026 if ((path_rec->epr_static_ipd) &&
2027 ((path_rec->epr_static_ipd + 1) >
2028 (proto->cct[path_rec->epr_ccti] & CCA_IPD_MASK))) {
2029 path_rec->epr_active_ipd = path_rec->epr_static_ipd + 1;
2030 path_rec->epr_cca_divisor = 0;
2031 }
2032 else {
2033 path_rec->epr_active_ipd = proto->cct[path_rec->epr_ccti] & CCA_IPD_MASK;
2034 path_rec->epr_cca_divisor =
2035 proto->cct[path_rec->epr_ccti] >> CCA_DIVISOR_SHIFT;
2036 }
2037
2038 _IPATH_CCADBG("CCA: %s injection rate to <%x.%x> from <%x.%x>\n", (cct_increment > 0) ? "Decreasing" : "Increasing", path_rec->epr_cca_divisor, path_rec->epr_active_ipd, prev_divisor, prev_ipd);
2039
2040 /* Reschedule CCA timer if this path is still marked as congested */
2041 if (path_rec->epr_ccti > path_rec->epr_ccti_min) {
2042 psmi_timer_request(proto->timerq,
2043 &path_rec->epr_timer_cca,
2044 get_cycles() +
2045 proto->cace[path_rec->epr_sl].ccti_timer_cycles);
2046 }
2047
2048 return PSM_OK;
2049 }
2050
2051 psm_error_t
2052 ips_cca_timer_callback(struct psmi_timer *current_timer, uint64_t current)
2053 {
2054 ips_path_rec_t *path_rec = (ips_path_rec_t *) current_timer->context;
2055
2056 /* Increase injection rate for flow. Decrement CCTI */
2057 if (path_rec->epr_ccti > path_rec->epr_ccti_min)
2058 return ips_cca_adjust_rate(path_rec, -1);
2059 else
2060 return PSM_OK;
2061 }