Debian Codegraph @

ptl_ips/ips_proto_expected.c

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "psm_user.h"
36 #include "ipserror.h"
37 #include "ips_proto.h"
38 #include "ips_proto_internal.h"
39
40 /*
41 * Expected tid operations are carried out over "sessions". One session is a
42 * collection of N tids where N is determined by the expected message window
43 * size (-W option or PSM_MQ_RNDV_IPATH_WINDOW). Since naks can cause
44 * retransmissions, each session has an session index (_desc_idx) and a
45 * generation count (_desc_genc) to be able to identify if retransmitted
46 * packets reference the correct session.
47 *
48 * index and generation count are each 4 bytes encoded in one ptl_arg. They
49 * could be compressed further but we have the header space, so we don't
50 * bother.
51 */
52 #define _desc_idx u32w0
53 #define _desc_genc u32w1
54
55 /*
56 * Easy switch to (say) _IPATH_INFO if debugging in the expected protocol is
57 * needed
58 */
59 #define _IPATH_EXP _IPATH_VDBG
60
61 /*
62 * Timer callbacks. When we need work to be done out of the receive process
63 * loop, we schedule work on timers to be done at a later time.
64 */
65 static psm_error_t
66 ips_tid_pendsend_timer_callback(struct psmi_timer *timer, uint64_t current);
67
68 static psm_error_t
69 ips_tid_pendtids_timer_callback(struct psmi_timer *timer, uint64_t current);
70
71 static psm_error_t
72 ips_tid_release_timer_callback(struct psmi_timer *timer, uint64_t current);
73
74 static psm_error_t
75 ips_tid_grant_timer_callback(struct psmi_timer *timer, uint64_t current);
76
77 static psm_error_t
78 ips_tid_send_handle_tidreq(struct ips_protoexp *protoexp, psm_mq_req_t req,
79 uint32_t msglen, int flags, ptl_epaddr_t *ipsaddr,
80 psmi_seqnum_t flowgenseq,
81 ips_tid_session_list *tid_list,
82 uint32_t tid_list_size);
83
84 static void
85 ips_tid_scbavail_callback(struct ips_scbctrl *scbc, void *context);
86
87 static void
88 ips_tid_flowavail_callback(struct ips_tfctrl *tfctrl, void *context);
89
90 static void
91 ips_tid_mpool_tidrecv_callback(void *context);
92
93 /* Defined at the ptl-level (breaks abstractions but needed for shared vs
94 * non-shared contexts */
95 extern int ips_ptl_recvq_isempty(const struct ptl *ptl);
96
97 static psm_error_t ips_tid_recv_free(struct ips_tid_recv_desc *tidrecvc);
98
99 psm_error_t
100 ips_protoexp_init(const psmi_context_t *context,
101 const struct ips_proto *proto,
102 uint32_t protoexp_flags,
103 int num_of_send_bufs,
104 int num_of_send_desc,
105 struct ips_protoexp **protoexp_o)
106 {
107 struct ips_protoexp *protoexp = NULL;
108 uint32_t tidmtu_max;
109 psm_error_t err = PSM_OK;
110
111 protoexp = (struct ips_protoexp *)
112 psmi_calloc(context->ep, UNDEFINED, 1, sizeof(struct ips_protoexp));
113 if (protoexp == NULL) {
114 err = PSM_NO_MEMORY;
115 goto fail;
116 }
117 *protoexp_o = protoexp;
118
119 protoexp->ptl = (const struct ptl *) proto->ptl;
120 protoexp->proto = (struct ips_proto *) proto;
121 protoexp->timerq = proto->timerq;
122 protoexp->tid_flags = protoexp_flags;
123 protoexp->tidflow_seed = (unsigned int) getpid();
124
125 /* Must be initialized already */
126 /* Comment out because of Klockwork scanning critical error. CQ 11/16/2012
127 psmi_assert_always(proto->ep != NULL && proto->ep->mq != NULL &&
128 proto->ep->mq->rreq_pool != NULL &&
129 proto->ep->mq->sreq_pool != NULL);
130 */
131 psmi_assert_always(proto->timerq != NULL);
132 /* Make sure pbc is at the right place before the message header */
133 psmi_assert_always(sizeof(union ipath_pbc) == (size_t)
134 (offsetof(struct ips_scb, ips_lrh) - offsetof(struct ips_scb, pbc)));
135
136 /* These request pools are managed by the MQ component */
137 protoexp->tid_sreq_pool = proto->ep->mq->sreq_pool;
138 protoexp->tid_rreq_pool = proto->ep->mq->rreq_pool;
139
140 if (proto->flags & IPS_PROTO_FLAG_MQ_EXPECTED_SDMA) {
141 protoexp->tid_ep_flow = EP_FLOW_GO_BACK_N_DMA;
142 protoexp->tid_xfer_type = PSM_TRANSFER_DMA;
143 }
144 else {
145 protoexp->tid_ep_flow = EP_FLOW_GO_BACK_N_PIO;
146 protoexp->tid_xfer_type = PSM_TRANSFER_PIO;
147 }
148
149 /* Initialze tid flow control. */
150 {
151 const struct ipath_user_info *user_info = &context->user_info;
152 const struct ipath_base_info *base_info = &context->base_info;
153 uint32_t num_flow, start_flow, end_flow;
154 uint32_t has_hw_hdrsupp = (context->runtime_flags & IPATH_RUNTIME_HDRSUPP);
155
156 if (!user_info->spu_subcontext_cnt || !has_hw_hdrsupp) {
157 /* If no context sharing enabled can use full tidflow table for
158 * all HCAs.
159 */
160 start_flow = 0;
161 num_flow = INFINIPATH_TF_NFLOWS;
162 }
163 else {
164 /* Context sharing on QLE73XX requires hardware tidflow table to be
165 * shared as well.
166 */
167 num_flow = (uint32_t) (INFINIPATH_TF_NFLOWS / user_info->spu_subcontext_cnt);
168 start_flow = base_info->spi_subcontext * num_flow;
169 }
170
171 end_flow = start_flow + num_flow;
172
173 if ((err = ips_tf_init(context, &protoexp->tfctrl,
174 start_flow, end_flow,
175 ips_tid_flowavail_callback, protoexp)))
176 goto fail;
177 }
178
179 /* Fix the fragsize to be a power of two (usually 2048) */
180 protoexp->tid_send_fragsize = context->base_info.spi_tid_maxsize;
181 if (proto->flags & IPS_PROTO_FLAG_MQ_EXPECTED_SDMA)
182 tidmtu_max = proto->epinfo.ep_mtu;
183 else
184 tidmtu_max = proto->epinfo.ep_piosize;
185
186 while (protoexp->tid_send_fragsize > tidmtu_max)
187 protoexp->tid_send_fragsize /= 2;
188
189 if ((err = ips_tid_init(&protoexp->tidc, context)))
190 goto fail;
191
192 {
193 uint32_t bounce_size, num_bounce_bufs;
194
195 if ((protoexp->tid_xfer_type == PSM_TRANSFER_DMA) ||
196 (protoexp->proto->flags & IPS_PROTO_FLAG_CKSUM)) {
197 num_bounce_bufs = max(8, num_of_send_bufs >> 2);
198 bounce_size = protoexp->tid_send_fragsize;
199 }
200 else {
201 /* no bufs, we only need the buffers to handle misalignment on the
202 * sender when using send dma. */
203 num_bounce_bufs = 0;
204 bounce_size = 0;
205 }
206 if ((err = ips_scbctrl_init(context, num_of_send_desc, num_bounce_bufs,
207 0, bounce_size, ips_tid_scbavail_callback,
208 protoexp, &protoexp->tid_scbc_rv)))
209 goto fail;
210 }
211
212 {
213 /* Determine interval to generate headers (relevant only when header
214 * suppression is enabled) else headers will always be generated.
215 *
216 * The PSM_EXPECTED_HEADERS environment variable can specify the
217 * packet interval to generate headers at. Else a header packet is
218 * generated every
219 * min(PSM_DEFAULT_EXPECTED_HEADER, window_size/tid_send_fragsize).
220 * Note: A header is always generated for the last packet in the flow.
221 */
222
223 union psmi_envvar_val env_exp_hdr;
224 uint32_t defval =
225 min(PSM_DEFAULT_EXPECTED_HEADER,
226 proto->mq->ipath_window_rv/protoexp->tid_send_fragsize);
227
228 psmi_getenv("PSM_EXPECTED_HEADERS",
229 "Interval to generate expected protocol headers",
230 PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT_FLAGS,
231 (union psmi_envvar_val) defval, &env_exp_hdr);
232
233 protoexp->hdr_pkt_interval = env_exp_hdr.e_uint;
234 /* Account for flow credits - Should try to have atleast 4 headers
235 * generated per window.
236 */
237 protoexp->hdr_pkt_interval =
238 max(min(protoexp->hdr_pkt_interval, proto->flow_credits >> 2), 1);
239
240 if (protoexp->hdr_pkt_interval != env_exp_hdr.e_uint) {
241 _IPATH_VDBG("Overriding PSM_EXPECTED_HEADERS=%u to be '%u'\n",
242 env_exp_hdr.e_uint, protoexp->hdr_pkt_interval);
243 }
244
245 }
246
247 /* Send descriptors.
248 *
249 * There can be up to 2^32 of these send descriptors. We conservatively
250 * allocate 256 but large node configurations can allocate up to sdesc_num
251 * of these (they are about 2k each).
252 * We impose a theoretical limit of 2^30.
253 */
254 {
255 struct psmi_rlimit_mpool rlim = TID_SENDSESSIONS_LIMITS;
256 uint32_t maxsz, chunksz;
257
258 if ((err = psmi_parse_mpool_env(protoexp->proto->mq, 1,
259 &rlim, &maxsz, &chunksz)))
260 goto fail;
261
262 protoexp->tid_desc_send_pool =
263 psmi_mpool_create(sizeof(struct ips_tid_send_desc), chunksz, maxsz,
264 0, DESCRIPTORS, NULL, NULL);
265
266 if (protoexp->tid_desc_send_pool == NULL) {
267 err = psmi_handle_error(proto->ep, PSM_NO_MEMORY,
268 "Couldn't allocate tid descriptor memory pool");
269 goto fail;
270 }
271 }
272
273 /* Receive descriptors.
274 *
275 * There can only be 256 of these because the field to identify the receive
276 * descriptor is only 8 bits. This currently isn't a problem because we
277 * only have 512 tids and each descriptor consumes ~32 tids per tid window.
278 * This means only roughly 16 descriptors are ever used.
279 */
280
281 {
282 struct psmi_rlimit_mpool rlim = TID_RECVSESSIONS_LIMITS;
283 uint32_t maxsz, chunksz;
284
285 if ((err = psmi_parse_mpool_env(protoexp->proto->mq, 1,
286 &rlim, &maxsz, &chunksz)))
287 goto fail;
288
289 protoexp->tid_desc_recv_pool =
290 psmi_mpool_create(sizeof(struct ips_tid_recv_desc), chunksz, maxsz,
291 0, DESCRIPTORS, ips_tid_mpool_tidrecv_callback,
292 protoexp);
293
294 if (protoexp->tid_desc_recv_pool == NULL) {
295 err = psmi_handle_error(proto->ep, PSM_NO_MEMORY,
296 "Couldn't allocate tid descriptor memory pool");
297 goto fail;
298 }
299 }
300
301 /* This pool can never be smaller than the max number of rreqs that can be
302 * allocated. */
303 {
304 uint32_t rreq_per_chunk, rreq_max;
305
306 psmi_assert_always(protoexp->proto->mq->rreq_pool != NULL);
307
308 psmi_mpool_get_obj_info(protoexp->proto->mq->rreq_pool,
309 &rreq_per_chunk,
310 &rreq_max);
311
312 protoexp->tid_getreq_pool =
313 psmi_mpool_create(sizeof(struct ips_tid_get_request),
314 rreq_per_chunk, rreq_max, 0, DESCRIPTORS, NULL, NULL);
315
316 if (protoexp->tid_getreq_pool == NULL) {
317 err = psmi_handle_error(proto->ep, PSM_NO_MEMORY,
318 "Couldn't allocate getreq descriptor memory pool");
319 goto fail;
320 }
321 }
322
323 /*
324 * Parse the tid timeout settings from the environment.
325 * <min_timeout>:<max_timeout>:<interrupt_iters>
326 *
327 */
328 {
329 int tvals[3];
330 char *tid_to;
331 union psmi_envvar_val env_to;
332
333 if (context->runtime_flags & PSMI_RUNTIME_RCVTHREAD) {
334 tvals[0] = 200;
335 tvals[1] = 1000;
336 tvals[2] = 2;
337 tid_to = "200:1000:2";
338 }
339 else {
340 /* This has always been the behavior ips < 2.1 */
341 tid_to = "100:100:3";
342 tvals[0] = 100;
343 tvals[1] = 100;
344 tvals[2] = 3;
345 }
346
347 if (!psmi_getenv("PSM_TID_TIMEOUT",
348 "Tid timeout control <min:max:intr_count>",
349 PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_STR,
350 (union psmi_envvar_val) tid_to,
351 &env_to)) {
352 /* not using default values */
353 tid_to = env_to.e_str;
354 psmi_parse_str_tuples(tid_to, 3, tvals);
355 }
356 protoexp->tid_to_cyc_min = us_2_cycles((uint64_t) tvals[0]);
357 protoexp->tid_to_cyc_max = us_2_cycles((uint64_t) tvals[1]);
358 protoexp->tid_to_intr = tvals[2];
359 _IPATH_PRDBG("Tid control message settings: timeout min=%dus/max=%dus, "
360 "interrupt when trying attempt #%d\n",
361 tvals[0], tvals[1], tvals[2]);
362 }
363
364 /*
365 * Make sure that the rendezvous window size settings are not larger than
366 * the largest packet we can put on the wire.
367 */
368 {
369 uint32_t winsize = protoexp->proto->mq->ipath_window_rv;
370
371 if (winsize < ips_tid_page_size(&protoexp->tidc)) {
372 _IPATH_INFO("Overriding request for rndv window size %d "
373 "to minimum supported value %d bytes\n",
374 winsize, ips_tid_page_size(&protoexp->tidc));
375 protoexp->proto->mq->ipath_window_rv =
376 ips_tid_page_size(&protoexp->tidc);
377 }
378 else { /* Figure out maximum supportable value assuming we can
379 * send a maxmium payload of 2048 bytes */
380 int maxtids = 0;
381
382 while (PSMI_ALIGNUP((sizeof(ips_tid_session_list) +
383 ((maxtids+1) * sizeof(ips_tid_session_member))), 4)
384 < IPS_PROTOEXP_MIN_MTU)
385 {
386 maxtids++;
387 }
388
389 /* Assume worse-case alignment when deriving the amount of tids,
390 * need one tid for bad page-alignment and another for spillover
391 * into last page */
392 winsize = (maxtids-2) * ips_tid_page_size(&protoexp->tidc);
393
394 if (protoexp->proto->mq->ipath_window_rv > winsize) {
395 _IPATH_INFO("Overriding request for rndv window size %d "
396 "to maximum supported value %d bytes\n",
397 protoexp->proto->mq->ipath_window_rv,
398 winsize);
399 protoexp->proto->mq->ipath_window_rv = winsize;
400 }
401 }
402 }
403
404 /*
405 * Allow setting of PSM_TID_MIN_EXPSEND, the minimum amount of expected
406 * send packets we send before checking the receive queue.
407 */
408 {
409 union psmi_envvar_val env_mincnt;
410
411 psmi_getenv("PSM_TID_MIN_EXPSEND",
412 "Min expsend pkt cnt before recv",
413 PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_UINT,
414 (union psmi_envvar_val) 3, &env_mincnt);
415 protoexp->tid_min_expsend_cnt = env_mincnt.e_uint;
416 }
417
418 /* Timers to handle requeueing of work out of the receive path */
419 psmi_timer_entry_init(&protoexp->timer_send,
420 ips_tid_pendsend_timer_callback, protoexp);
421 STAILQ_INIT(&protoexp->pend_sendq);
422 psmi_timer_entry_init(&protoexp->timer_getreqs,
423 ips_tid_pendtids_timer_callback, protoexp);
424 STAILQ_INIT(&protoexp->pend_getreqsq);
425
426 protoexp->tid_page_offset_mask =
427 (uint32_t) context->base_info.spi_tid_maxsize - 1;
428 protoexp->tid_page_mask =
429 ~((uint64_t) context->base_info.spi_tid_maxsize - 1);
430
431 if (protoexp->tid_flags & IPS_PROTOEXP_FLAG_TID_DEBUG) {
432 protoexp->tid_info = (struct ips_tidinfo *)
433 psmi_calloc(context->ep, UNDEFINED, IPS_TID_MAX_TIDS,
434 sizeof (struct ips_tidinfo));
435 if (protoexp->tid_info == NULL) {
436 err = PSM_NO_MEMORY;
437 goto fail;
438 }
439 }
440 else
441 protoexp->tid_info = NULL;
442
443 psmi_assert(err == PSM_OK);
444 return err;
445
446 fail:
447 if (protoexp != NULL && protoexp->tid_getreq_pool != NULL)
448 psmi_mpool_destroy(protoexp->tid_getreq_pool);
449 if (protoexp != NULL && protoexp->tid_desc_recv_pool != NULL)
450 psmi_mpool_destroy(protoexp->tid_desc_recv_pool);
451 if (protoexp != NULL && protoexp->tid_desc_send_pool != NULL)
452 psmi_mpool_destroy(protoexp->tid_desc_send_pool);
453 if (protoexp != NULL)
454 ips_scbctrl_fini(&protoexp->tid_scbc_rv);
455 if (protoexp != NULL)
456 psmi_free(protoexp);
457 return err;
458 }
459
460 psm_error_t
461 ips_protoexp_fini(struct ips_protoexp *protoexp)
462 {
463 psm_error_t err = PSM_OK;
464
465 psmi_mpool_destroy(protoexp->tid_getreq_pool);
466 psmi_mpool_destroy(protoexp->tid_desc_recv_pool);
467 psmi_mpool_destroy(protoexp->tid_desc_send_pool);
468
469 if ((err = ips_scbctrl_fini(&protoexp->tid_scbc_rv)))
470 goto fail;
471
472 if ((err = ips_tid_fini(&protoexp->tidc)))
473 goto fail;
474
475 if ((err = ips_tf_fini(&protoexp->tfctrl)))
476 goto fail;
477
478 _IPATH_PRDBG("Tid control resends: tid_grant=%lld,tid_release=%lld,"
479 "request_intr=%lld\n",
480 (long long) protoexp->tid_grant_resends,
481 (long long) protoexp->tid_release_resends,
482 (long long) protoexp->tid_intr_reqs);
483
484 if (protoexp->tid_flags & IPS_PROTOEXP_FLAG_TID_DEBUG)
485 psmi_free(protoexp->tid_info);
486
487 psmi_free(protoexp);
488
489 fail:
490 return err;
491 }
492
493 /* New scbs now available. If we have pending sends because we were out of
494 * scbs, put the pendq on the timerq so it can be processed. */
495 static
496 void
497 ips_tid_scbavail_callback(struct ips_scbctrl *scbc, void *context)
498 {
499 struct ips_protoexp *protoexp = (struct ips_protoexp *) context;
500
501 if (!STAILQ_EMPTY(&protoexp->pend_sendq))
502 psmi_timer_request(protoexp->timerq,
503 &protoexp->timer_send, PSMI_TIMER_PRIO_1);
504 return;
505 }
506
507 /* New Tid Flows are available. If there are pending get requests put the
508 * get timer on the timerq so it can be processed. */
509 static
510 void
511 ips_tid_flowavail_callback(struct ips_tfctrl *tfctrl, void *context)
512 {
513 struct ips_protoexp *protoexp = (struct ips_protoexp *) context;
514
515 if (!STAILQ_EMPTY(&protoexp->pend_getreqsq))
516 psmi_timer_request(protoexp->timerq,
517 &protoexp->timer_getreqs, PSMI_TIMER_PRIO_1);
518 return;
519 }
520
521 /*
522 * The tid get request is always issued from within the receive progress loop,
523 * which is why we always enqueue the request instead of issuing it directly.
524 * Eventually, if we expose tid_get to users, we will want to differentiate
525 * when the request comes from the receive progress loop from cases where the
526 * tid_get is issued directly from user code.
527 *
528 */
529 psm_error_t
530 ips_protoexp_tid_get_from_token(
531 struct ips_protoexp *protoexp,
532 void *buf,
533 uint32_t length,
534 psm_epaddr_t epaddr,
535 uint32_t remote_tok,
536 uint32_t flags,
537 ips_tid_completion_callback_t callback,
538 void *context)
539 {
540 struct ips_tid_get_request *getreq;
541 int count, fragsize;
542
543 getreq = (struct ips_tid_get_request *)
544 psmi_mpool_get(protoexp->tid_getreq_pool);
545
546 /* We can't *really* run out of these here because we always allocate as
547 * much as available receive reqs */
548 if_pf (getreq == NULL)
549 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
550 "Ran out of 'getreq' descriptors");
551
552 getreq->tidgr_protoexp = protoexp;
553 getreq->tidgr_epaddr = epaddr;
554 getreq->tidgr_lbuf = buf;
555 getreq->tidgr_length = length;
556 getreq->tidgr_sendtoken = remote_tok;
557 getreq->tidgr_ucontext = context;
558 getreq->tidgr_callback = callback;
559 getreq->tidgr_offset = 0;
560 getreq->tidgr_bytesdone = 0;
561 getreq->tidgr_desc_seqno= 0;
562 getreq->tidgr_flags = flags;
563
564 /* nsconn is the # of slave channels. */
565 /* fragsize is the bytes each channel should transfer. */
566 count = epaddr->mctxt_master->mctxt_nsconn;
567 fragsize = (length+count)/(count+1);
568 if (fragsize < 4096) fragsize = 4096;
569 getreq->tidgr_rndv_winsz= min(fragsize, epaddr->ep->mq->ipath_window_rv);
570
571 STAILQ_INSERT_TAIL(&protoexp->pend_getreqsq, getreq, tidgr_next);
572 if (ips_tid_num_available(&protoexp->tidc) >=
573 ips_tid_num_required(&protoexp->tidc, (void *) NULL,
574 getreq->tidgr_rndv_winsz))
575 ips_tid_pendtids_timer_callback(&protoexp->timer_getreqs, 0);
576 else
577 psmi_timer_request(protoexp->timerq, &protoexp->timer_getreqs,
578 PSMI_TIMER_PRIO_1);
579 return PSM_OK;
580 }
581
582 /* List of perf events */
583 #define _ips_logeventid_tid_send_reqs 0 /* out of tid send descriptors */
584
585 #define ips_logevent_id(event) _ips_logeventid_ ## event
586 #define ips_logevent(proto, event,ptr) ips_logevent_inner(proto, ips_logevent_id(event), ptr)
587
588 static
589 void
590 ips_logevent_inner(struct ips_proto *proto, int eventid, void *context)
591 {
592 uint64_t t_now = get_cycles();
593
594 switch (eventid) {
595 case ips_logevent_id(tid_send_reqs): {
596 ips_epaddr_t *ipsaddr = (ips_epaddr_t *) context;
597 proto->psmi_logevent_tid_send_reqs.count++;
598
599 if (t_now >= proto->psmi_logevent_tid_send_reqs.next_warning) {
600 psmi_handle_error(PSMI_EP_LOGEVENT, PSM_OK,
601 "Non-fatal temporary exhaustion of send tid dma descriptors "
602 "(elapsed=%.3fs, source LID=0x%x/context=%d, count=%lld)",
603 (double) cycles_to_nanosecs(t_now - ipsaddr->proto->t_init) / 1.0e9,
604 (int) psm_epid_nid(ipsaddr->epaddr->epid),
605 (int) psm_epid_context(ipsaddr->epaddr->epid),
606 (long long) proto->psmi_logevent_tid_send_reqs.count);
607 proto->psmi_logevent_tid_send_reqs.next_warning = t_now +
608 sec_2_cycles(proto->psmi_logevent_tid_send_reqs.interval_secs);
609 }
610 }
611 break;
612
613 default:
614 break;
615 }
616
617 return;
618 }
619
620 /*
621 * Expected Protocol.
622 *
623 * We're granted tids (as part of a tid get request) and expected to fulfill
624 * the request by associating the request's sendtoken to a tid send descriptor.
625 *
626 * It's possible to be out of tid send descriptors when somehow all allocated
627 * descriptors can't complete all of their sends. For example, the targets of
628 * the sends may be busy in computation loops and not processing incoming
629 * packets.
630 */
631
632 void __fastpath
633 ips_protoexp_tid_grant(const struct ips_recvhdrq_event *rcv_ev)
634 {
635 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
636 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
637 ips_tid_session_list *tid_list;
638 ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr;
639 uint32_t paylen, msglen;
640 uint32_t reqidx;
641 psmi_seqnum_t flowgenseq;
642 psm_error_t err = PSM_OK;
643 psm_mq_req_t req;
644 ptl_arg_t args[3];
645 uint8_t index, seqno;
646
647 paylen = ips_recvhdrq_event_paylen(rcv_ev);
648 tid_list = (ips_tid_session_list *) ips_recvhdrq_event_payload(rcv_ev);
649 reqidx = p_hdr->data[0].u32w0;
650 msglen = p_hdr->data[0].u32w1;
651 flowgenseq.val = p_hdr->data[1].u32w0;
652
653 /* Increment grant received stats for endpoint */
654 ipsaddr->stats.tids_grant_recv++;
655 index = tid_list->tsess_seqno % sizeof(req->tid_grant);
656 seqno = tid_list->tsess_seqno / sizeof(req->tid_grant);
657
658 req = psmi_mpool_find_obj_by_index(protoexp->tid_sreq_pool, reqidx);
659
660 if (req) {
661 _IPATH_VDBG("req=%p (%d) wait=%s req_seqno=%d pkt_len=%d, seqno=%d, msglen=%d\n",
662 req, reqidx, req->type & MQE_TYPE_WAITING ? "yes" : "no",
663 req->recv_msgoff, paylen, tid_list->tsess_seqno, msglen);
664 }
665
666 /* We use recv_msgoff to track the latest receive sequence number */
667
668 if (req == NULL) {
669 /* Not found, bogus req, ack it anyway */
670 }
671 else if (seqno < req->tid_grant[index]) {
672 /* dupe, ack it */
673 }
674 else if (seqno > req->tid_grant[index]) {
675 /* lost tidreq, wait for rexmit */
676 /* XXX count this to see if it's worth handling instead of dropping */
677 goto no_ack;
678 }
679 else {
680 req->tid_grant[index]++;
681 /* Safe to keep updating every time */
682 req->send_msglen = msglen;
683 if ((err = ips_tid_send_handle_tidreq(protoexp, req, msglen, 0, ipsaddr, flowgenseq, tid_list, paylen)) != PSM_OK)
684 {
685 ips_logevent(rcv_ev->proto, tid_send_reqs, ipsaddr);
686 /* Out of send reqs, wait for rexmit */
687 goto no_ack;
688 }
689 req->recv_msgoff = tid_list->tsess_seqno + 1;
690 rcv_ev->proto->psmi_logevent_tid_send_reqs.next_warning = 0;
691 }
692
693 /* At this point we can ack the request */
694 args[0] = tid_list->tsess_descid;
695
696 ips_proto_send_ctrl_message(&ipsaddr->flows[protoexp->tid_ep_flow],
697 OPCODE_TIDS_GRANT_ACK,
698 &ipsaddr->ctrl_msg_queued, args);
699
700 no_ack:
701 return;
702 }
703
704 void __fastpath
705 ips_protoexp_tid_grant_ack(const struct ips_recvhdrq_event *rcv_ev)
706 {
707 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
708 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
709 struct ips_tid_recv_desc *tidrecvc;
710 ptl_arg_t desc_id = p_hdr->data[0];
711 ptl_arg_t desc_tidrecvc;
712
713 tidrecvc = (struct ips_tid_recv_desc *)
714 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
715 desc_id._desc_idx);
716
717 if (tidrecvc == NULL) /* dupe or gone, drop it */
718 return;
719
720 psmi_mpool_get_obj_index_gen_count(tidrecvc,
721 &desc_tidrecvc._desc_idx,
722 &desc_tidrecvc._desc_genc);
723
724 _IPATH_VDBG("desc_req:id=%d,gen=%d desc_tidc:id=%d,gen=%d\n",
725 desc_id._desc_idx, desc_id._desc_genc,
726 desc_tidrecvc._desc_idx, desc_tidrecvc._desc_genc);
727
728 if (desc_tidrecvc.u64 == desc_id.u64 &&
729 tidrecvc->state == TIDRECVC_STATE_GRANT)
730 {
731 psmi_timer_cancel(protoexp->timerq, &tidrecvc->timer_tidreq);
732 tidrecvc->state = TIDRECVC_STATE_GRANT_ACK;
733 }
734 return;
735 }
736
737 void
738 __fastpath
739 ips_protoexp_recv_unaligned_data(struct ips_recvhdrq_event *rcv_ev)
740 {
741
742 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
743 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
744 struct ptl_epaddr *ipsaddr = rcv_ev->ipsaddr;
745 uint32_t tid_recv_sessid;
746 struct ips_tid_recv_desc *tidrecvc;
747 ptl_arg_t desc_id = rcv_ev->p_hdr->data[0];
748 int i;
749 uint8_t *byte_index = (uint8_t *) &p_hdr->data[1].u32w0;
750 uint8_t *buffer;
751
752 if (!ips_proto_is_expected_or_nak(rcv_ev)) goto process_ack;
753
754 psmi_assert(p_hdr->flags & (IPS_SEND_FLAG_UNALIGNED_DATA | IPS_SEND_FLAG_ACK_REQ));
755
756 tid_recv_sessid = desc_id._desc_idx;
757 tidrecvc =
758 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
759 tid_recv_sessid);
760
761 if_pf (tidrecvc == NULL) {
762 _IPATH_ERROR("No tidrecv session with index %d\n",
763 tid_recv_sessid);
764 goto process_ack;
765 }
766
767 if_pf (psmi_mpool_get_obj_gen_count(tidrecvc) != desc_id._desc_genc) {
768 _IPATH_ERROR("Expected packet to tid session %d, now %d instead "
769 "of %d; skipping\n", tid_recv_sessid,
770 psmi_mpool_get_obj_gen_count(tidrecvc),
771 desc_id._desc_genc);
772 goto process_ack; /* skip */
773 }
774
775 psmi_assert(p_hdr->hdr_dlen ==
776 (tidrecvc->tid_list.tsess_unaligned_start + tidrecvc->tid_list.tsess_unaligned_end));
777
778 /* Cancel tid grant timer (if still active) */
779 if (tidrecvc->num_recv_hdrs++ == 0)
780 psmi_timer_cancel(protoexp->timerq, &tidrecvc->timer_tidreq);
781
782 buffer = tidrecvc->buffer;
783 for (i = 0; i < tidrecvc->tid_list.tsess_unaligned_start; i++)
784 *buffer++ = *byte_index++;
785
786 buffer =
787 (uint8_t *) tidrecvc->buffer + tidrecvc->recv_msglen -
788 tidrecvc->tid_list.tsess_unaligned_end;
789 byte_index = (uint8_t *)&p_hdr->data[1].u32w1;
790
791 for (i = 0; i < tidrecvc->tid_list.tsess_unaligned_end; i++)
792 *buffer++ = *byte_index++;
793
794 /* If packet has checksum for window cache it */
795 if (p_hdr->flags & IPS_SEND_FLAG_HAS_CKSUM) {
796 uint32_t *cksum = (uint32_t*) ips_recvhdrq_event_payload(rcv_ev);
797
798 psmi_assert_always(protoexp->proto->flags & IPS_PROTO_FLAG_CKSUM);
799 psmi_assert_always(ips_recvhdrq_event_payload(rcv_ev));
800 psmi_assert_always(ips_recvhdrq_event_paylen(rcv_ev));
801 tidrecvc->cksum = *cksum;
802 }
803
804 process_ack:
805 ips_proto_process_ack(rcv_ev);
806 /* May require ACK for this packet. */
807 if (p_hdr->flags & IPS_SEND_FLAG_ACK_REQ)
808 ips_proto_send_ack((struct ips_recvhdrq *) rcv_ev->recvq,
809 &ipsaddr->flows[ips_proto_flowid(p_hdr)]);
810
811 return;
812 }
813
814 void
815 __fastpath
816 ips_protoexp_data(struct ips_recvhdrq_event *rcv_ev)
817 {
818 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
819 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
820 uint32_t tid_recv_sessid;
821 struct ips_tid_recv_desc *tidrecvc;
822 ptl_arg_t desc_id = rcv_ev->p_hdr->data[0];
823 ptl_arg_t send_descid = rcv_ev->p_hdr->data[1];
824 uint32_t paylen;
825 psmi_seqnum_t sequence_num, expected_sequence_num;
826 uint32_t has_hw_hdrsupp = (protoexp->ptl->context->runtime_flags & IPATH_RUNTIME_HDRSUPP);
827 ptl_arg_t args[3];
828
829 paylen = ips_recvhdrq_event_paylen(rcv_ev);
830 tid_recv_sessid = desc_id._desc_idx;
831 tidrecvc =
832 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
833 tid_recv_sessid);
834
835 if_pf (tidrecvc == NULL) {
836 _IPATH_ERROR("No tidrecv session with index %d\n",
837 tid_recv_sessid);
838 return;
839 }
840
841 if_pf (rcv_ev->ptype == RCVHQ_RCV_TYPE_EAGER && paylen != 0) {
842 _IPATH_ERROR("Expected packet, but eager index is set; skipping\n");
843 return;
844 }
845
846 if_pf (psmi_mpool_get_obj_gen_count(tidrecvc) != desc_id._desc_genc) {
847 _IPATH_ERROR("Expected packet to tid session %d, now %d instead "
848 "of %d; skipping\n", tid_recv_sessid,
849 psmi_mpool_get_obj_gen_count(tidrecvc),
850 desc_id._desc_genc);
851 return; /* skip */
852 }
853
854 sequence_num.val = __be32_to_cpu(p_hdr->bth[2]);
855 expected_sequence_num = tidrecvc->tidflow_genseq;
856
857 /* On QLE73XX this is only called if data was fully received or the ACK
858 * interval was reached else the gen/seq error handlers are called
859 * from ips_proto_recv.
860 */
861 if (has_hw_hdrsupp) {
862
863 /* Drop packet if generation number does not match */
864 if (expected_sequence_num.gen != sequence_num.gen)
865 return;
866
867 /* Increment the expected sequence number taking into account the number
868 * of headers that were suppressed.
869 */
870 expected_sequence_num.seq += (protoexp->hdr_pkt_interval - 1);
871
872 /* Special case for last packet as may be lesser than interval. */
873 if (p_hdr->flags & IPS_SEND_FLAG_EXPECTED_DONE)
874 expected_sequence_num = sequence_num;
875
876 /* TIDFLOW will restart in the if block below */
877 if_pf (sequence_num.psn != expected_sequence_num.psn) {
878 _IPATH_EPDBG("Expected: Packet PSN %d received and were expecting %d. Restarting flow.\n", sequence_num.psn, expected_sequence_num.psn);
879 }
880
881 }
882
883 /* IBTA CCA handling for expected flow. */
884 if (rcv_ev->is_congested & IPS_RECV_EVENT_FECN) {
885 /* Mark flow to generate BECN in control packet */
886 tidrecvc->ipsaddr->tidgr_flow.flags |= IPS_FLOW_FLAG_GEN_BECN;
887 /* Update stats for congestion encountered */
888 rcv_ev->ipsaddr->stats.congestion_pkts++;
889 /* Clear FECN event */
890 rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN;
891 }
892
893 if_pf (sequence_num.psn != expected_sequence_num.psn) {
894 psmi_assert(sequence_num.flow == tidrecvc->tidflow_idx);
895 psmi_assert(sequence_num.flow == tidrecvc->tidflow_genseq.flow);
896
897 /* Generation mismatch */
898 if (sequence_num.gen != tidrecvc->tidflow_genseq.gen)
899 return ips_protoexp_handle_tf_generr(rcv_ev);
900
901 /* Sequence mismatch error */
902 return ips_protoexp_handle_tf_seqerr(rcv_ev);
903 }
904 else {
905
906 /* Update the shadow tidflow_genseq */
907 tidrecvc->tidflow_genseq.seq = sequence_num.seq + 1;
908
909 /* On QLE71XX/QLE72XX update tidflow table in software */
910 if (!has_hw_hdrsupp)
911 ipath_tidflow_set_entry(tidrecvc->context->ctrl,
912 tidrecvc->tidflow_idx,
913 tidrecvc->tidflow_genseq.gen,
914 tidrecvc->tidflow_genseq.seq);
915
916 /* Reset the swapped generation count as we received a valid packet */
917 tidrecvc->tidflow_nswap_gen = 0;
918 }
919
920 /* Do some sanity checking */
921 psmi_assert_always(((__be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 20) & 3) == 0);
922 psmi_assert_always(tidrecvc->state != TIDRECVC_STATE_DONE);
923
924 /* If first packet received cancel tid grant timer */
925 if (tidrecvc->num_recv_hdrs++ == 0)
926 psmi_timer_cancel(protoexp->timerq, &tidrecvc->timer_tidreq);
927
928 /* If last packet we can close the tidflow.
929 * We can deallocate tidflow even if the unaligned data has not been
930 * received. The TID_RELEASE message will deallocate the receive
931 * descriptor.
932 *
933 * Note: If we were out of tidflows this will invoke the callback to
934 * schedule pending transfers.
935 */
936
937 if (p_hdr->flags & IPS_SEND_FLAG_EXPECTED_DONE) {
938
939 psm_error_t ret = PSM_OK;
940
941 /* Acquire lock before updating state (ERR_CHK_GEN also tests for
942 * state before responding.
943 */
944
945 ips_ptladdr_lock(rcv_ev->ipsaddr);
946
947 /* Mark receive as done */
948 tidrecvc->state = TIDRECVC_STATE_DONE;
949
950 ret = ips_tf_deallocate(&protoexp->tfctrl,
951 tidrecvc->tidflow_idx);
952 psmi_assert_always (ret == PSM_OK);
953
954 /* Release lock */
955 ips_ptladdr_unlock(rcv_ev->ipsaddr);
956 }
957
958 /* Respond with an ACK if sender requested one or incoming flow faced
959 * congestion. The ACK in this case will have the BECN bit set.
960 */
961 if ((p_hdr->flags & IPS_SEND_FLAG_ACK_REQ) ||
962 (tidrecvc->ipsaddr->tidgr_flow.flags & IPS_FLOW_FLAG_GEN_BECN)) {
963
964 /* Ack sender with descriptor index */
965 args[0] = send_descid;
966 args[1] = tidrecvc->tid_list.tsess_descid;
967
968 ips_proto_send_ctrl_message(&tidrecvc->ipsaddr->tidgr_flow,
969 OPCODE_ACK,
970 &tidrecvc->ctrl_msg_queued, args);
971 }
972
973 return;
974 }
975
976 #ifndef PSM_DEBUG
977 # define ips_dump_tids(tid_list,msg,...)
978 #else
979 static
980 void
981 ips_dump_tids(ips_tid_session_list *tid_list, const char *msg, ...)
982 {
983 char buf[256];
984 size_t off = 0;
985 int i, num_tids = tid_list->tsess_tidcount;
986
987 va_list argptr;
988 va_start(argptr, msg);
989 off += vsnprintf(buf, sizeof buf - off, msg, argptr);
990 va_end(argptr);
991
992 for (i = 0; i < num_tids && off < (sizeof buf - 1); i++)
993 off += snprintf(buf + off, sizeof buf - off, "%d%s",
994 (int) tid_list->tsess_list[i].tid,
995 i < num_tids-1 ? "," : "");
996
997 _IPATH_VDBG("%s\n", buf);
998 return;
999 }
1000 #endif
1001
1002 static
1003 void
1004 ips_expsend_tiderr(struct ips_tid_send_desc *tidsendc)
1005 {
1006 char buf[256];
1007 size_t off = 0;
1008 int i;
1009
1010 off += snprintf(buf + off, sizeof buf - off,
1011 "Remaining bytes: %d Member id %d is not in tid_session_id=%d :", tidsendc->remaining_bytes, tidsendc->tid_idx,
1012 tidsendc->tid_list.tsess_descid._desc_idx);
1013
1014 for (i = 0; i < tidsendc->tid_list.tsess_tidcount+1; i++)
1015 off += snprintf(buf + off, sizeof buf - off, "%d,",
1016 tidsendc->tid_list.tsess_list[i].tid);
1017 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
1018 "Trying to use tid idx %d and there are %d members: %s\n",
1019 tidsendc->tid_idx, tidsendc->tid_list.tsess_tidcount, buf);
1020 return;
1021 }
1022
1023 void
1024 ips_protoexp_scb_inflight(ips_scb_t *scb)
1025 {
1026 if (scb->tidsendc)
1027 scb->tidsendc->iovec_cntr_last = scb->dma_ctr;
1028 return;
1029 }
1030
1031 static
1032 void __fastpath
1033 ips_tid_send_tid_release_msg(struct ips_tid_send_desc *tidsendc)
1034 {
1035 psm_error_t err;
1036 struct ips_protoexp *protoexp = tidsendc->protoexp;
1037 psm_mq_req_t req = tidsendc->mqreq;
1038 ptl_arg_t desc_id[3] = {};
1039 uint64_t t_cyc;
1040
1041 desc_id[0] = tidsendc->tid_list.tsess_descid;
1042 desc_id[1] = tidsendc->descid;
1043 desc_id[2].u32w0 = tidsendc->release_cnt;
1044
1045 err = ips_proto_send_ctrl_message(&tidsendc->ipsaddr->
1046 flows[protoexp->tid_ep_flow],
1047 OPCODE_TIDS_RELEASE,
1048 &tidsendc->ctrl_msg_queued,
1049 desc_id);
1050
1051 if (err != PSM_EP_NO_RESOURCES) {
1052 tidsendc->release_cnt++;
1053 t_cyc = get_cycles() + protoexp->tid_to_cyc_min;
1054 }
1055 else
1056 t_cyc = get_cycles() + protoexp->proto->timeout_send;
1057
1058 psmi_timer_request_always(protoexp->timerq, &tidsendc->timer_tidrelease,
1059 t_cyc);
1060
1061 req->send_msgoff += tidsendc->length;
1062
1063 _IPATH_VDBG("[rndv][send] tid chunk of size %d done %d/%d for req=%p%s\n",
1064 tidsendc->length, req->send_msgoff, req->send_msglen, req,
1065 req->send_msgoff == req->send_msglen ? " (complete)" : "");
1066
1067 if (req->send_msgoff == req->send_msglen)
1068 psmi_mq_handle_rts_complete(req);
1069 }
1070
1071 static
1072 int __fastpath
1073 ips_tid_send_completion_unaligned_callback(void * param, uint32_t nbytes)
1074 {
1075 struct ips_tid_send_desc *tidsendc = (struct ips_tid_send_desc *) param;
1076
1077 /* Decrement completion counter and complete if unaligned data sent */
1078 tidsendc->completion_counter--;
1079
1080 psmi_assert(tidsendc->completion_counter >= 0);
1081
1082 if (tidsendc->completion_counter == 0)
1083 ips_tid_send_tid_release_msg(tidsendc);
1084
1085 return IPS_RECVHDRQ_CONTINUE;
1086 }
1087
1088 static
1089 int __fastpath
1090 ips_tid_send_completion_callback(void * param, uint32_t nbytes)
1091 {
1092 struct ips_tid_send_desc *tidsendc = (struct ips_tid_send_desc *) param;
1093 struct ips_protoexp *protoexp = tidsendc->protoexp;
1094
1095 if (protoexp->tid_xfer_type == PSM_TRANSFER_DMA)
1096 ips_proto_dma_wait_until(protoexp->proto, tidsendc->iovec_cntr_last);
1097
1098 if (tidsendc->bounce_buf) psmi_free(tidsendc->bounce_buf);
1099
1100 /* Decrement completion counter and complete if unaligned data sent */
1101 tidsendc->completion_counter--;
1102
1103 psmi_assert(tidsendc->completion_counter >= 0);
1104
1105 if (tidsendc->completion_counter == 0)
1106 ips_tid_send_tid_release_msg(tidsendc);
1107
1108 return IPS_RECVHDRQ_CONTINUE;
1109 }
1110
1111 static
1112 psm_error_t __fastpath
1113 ips_tid_release_timer_callback(struct psmi_timer *timer, uint64_t current)
1114 {
1115 struct ips_tid_send_desc *tidsendc =
1116 (struct ips_tid_send_desc *) timer->context;
1117 struct ips_protoexp *protoexp = tidsendc->protoexp;
1118 uint64_t t_cyc;
1119 psm_error_t err;
1120 ptl_arg_t desc_id[3] = {};
1121
1122 /* 0 contain's the receiver's desc_id, 1 contains the sender's desc_id */
1123 desc_id[0] = tidsendc->tid_list.tsess_descid;
1124 desc_id[1] = tidsendc->descid;
1125 desc_id[2].u32w0 = tidsendc->release_cnt;
1126
1127 err = ips_proto_send_ctrl_message(&tidsendc->ipsaddr->
1128 flows[protoexp->tid_ep_flow],
1129 OPCODE_TIDS_RELEASE,
1130 &tidsendc->ctrl_msg_queued,
1131 desc_id);
1132
1133 if (err == PSM_EP_NO_RESOURCES) {
1134 t_cyc = get_cycles() + protoexp->proto->timeout_send;
1135 }
1136 else {
1137 tidsendc->release_cnt++;
1138 protoexp->tid_release_resends++;
1139 t_cyc = get_cycles() +
1140 min(tidsendc->release_cnt * protoexp->tid_to_cyc_min,
1141 protoexp->tid_to_cyc_max);
1142 }
1143
1144 psmi_timer_request_always(protoexp->timerq,
1145 &tidsendc->timer_tidrelease,
1146 t_cyc);
1147
1148 return PSM_OK;
1149 }
1150
1151 static
1152 psm_error_t __fastpath
1153 ips_tid_grant_timer_callback(struct psmi_timer *timer, uint64_t current)
1154 {
1155 struct ips_tid_recv_desc *tidrecvc =
1156 (struct ips_tid_recv_desc *) timer->context;
1157 struct ips_protoexp *protoexp = tidrecvc->protoexp;
1158 ips_epaddr_t *ipsaddr = tidrecvc->ipsaddr;
1159 psm_error_t err;
1160 uint64_t t_cyc;
1161
1162 err = ips_proto_send_ctrl_message(&ipsaddr->flows[protoexp->tid_ep_flow],
1163 OPCODE_TIDS_GRANT,
1164 &tidrecvc->ctrl_msg_queued,
1165 &tidrecvc->tid_list);
1166
1167 if (err == PSM_EP_NO_RESOURCES) {
1168 t_cyc = get_cycles() + protoexp->proto->timeout_send;
1169 }
1170 else {
1171 tidrecvc->grant_cnt++;
1172 protoexp->tid_grant_resends++;
1173 t_cyc = get_cycles() +
1174 min(tidrecvc->grant_cnt * protoexp->tid_to_cyc_min,
1175 protoexp->tid_to_cyc_max);
1176 }
1177
1178 psmi_timer_request_always(protoexp->timerq, timer, t_cyc);
1179
1180 return PSM_OK;
1181 }
1182
1183 static
1184 __fastpath
1185 psm_error_t
1186 ips_tid_send_handle_tidreq(struct ips_protoexp *protoexp,
1187 psm_mq_req_t req, uint32_t msglen,
1188 int flags, ptl_epaddr_t *ipsaddr,
1189 psmi_seqnum_t flowgenseq,
1190 ips_tid_session_list *tid_list,
1191 uint32_t tid_list_size)
1192 {
1193 struct ips_tid_send_desc *tidsendc;
1194 req->send_msglen = msglen;
1195
1196 psmi_assert(tid_list_size >= sizeof(ips_tid_session_list));
1197 psmi_assert(tid_list_size <= 2096);
1198
1199 tidsendc = (struct ips_tid_send_desc *)
1200 psmi_mpool_get(protoexp->tid_desc_send_pool);
1201 if (tidsendc == NULL)
1202 return PSM_EP_NO_RESOURCES;
1203
1204 tidsendc->protoexp = protoexp;
1205
1206 /* Uniquely identify this send descriptor in space and time */
1207 tidsendc->descid._desc_idx = psmi_mpool_get_obj_index(tidsendc);
1208 tidsendc->descid._desc_genc = psmi_mpool_get_obj_gen_count(tidsendc);
1209
1210 psmi_mq_mtucpy(&tidsendc->tid_list, tid_list, tid_list_size);
1211 tid_list = &tidsendc->tid_list;
1212
1213 tidsendc->length = tid_list->tsess_length;
1214 tidsendc->ipsaddr = ipsaddr;
1215 tidsendc->mqreq = req;
1216 tidsendc->bounce_buf = NULL;
1217 tidsendc->buffer =
1218 (void *)((uintptr_t)req->buf + tid_list->tsess_srcoff);
1219 tidsendc->tid_idx = 0;
1220 tidsendc->is_complete= 0;
1221 tidsendc->release_cnt= 0;
1222
1223 /* Initialize tidflow for window. Use path requested by remote endpoint */
1224 ips_flow_init(&tidsendc->tidflow, NULL, ipsaddr, protoexp->tid_xfer_type,
1225 PSM_PROTOCOL_TIDFLOW, IPS_PATH_LOW_PRIORITY, 0);
1226
1227 tidsendc->tidflow.xmit_seq_num = flowgenseq;
1228 tidsendc->tidflow.xmit_ack_num = flowgenseq;
1229 tidsendc->tidflow.xmit_ack_num.seq--; /* last acked */
1230 tidsendc->ctrl_msg_queued = 0;
1231 tidsendc->completion_counter = 1;
1232
1233 /* If unaligned data will need to send a separate packet containing
1234 * unaligned data.
1235 */
1236 if ((tidsendc->tid_list.tsess_unaligned_start) ||
1237 (tidsendc->tid_list.tsess_unaligned_end) ||
1238 (protoexp->proto->flags & IPS_PROTO_FLAG_CKSUM))
1239 tidsendc->completion_counter += 1;
1240
1241 if (tid_list->tsess_tidcount == 0) {
1242 _IPATH_VDBG("no tids used, alloc eager tid\n");
1243 tid_list->tsess_list[0].tid = IPATH_EAGER_TID_ID;
1244 tid_list->tsess_list[0].length = 0;
1245 tid_list->tsess_list[0].offset = 0;
1246 }
1247
1248 tidsendc->frame_send = 0;
1249 tidsendc->remaining_bytes = tid_list->tsess_length;
1250 tidsendc->remaining_bytes_in_page =
1251 tid_list->tsess_list[0].length;
1252 tidsendc->offset = tid_list->tsess_list[0].offset;
1253 tidsendc->unaligned_sent = 0;
1254
1255 psmi_timer_entry_init(&tidsendc->timer_tidrelease,
1256 ips_tid_release_timer_callback, tidsendc);
1257
1258 _IPATH_EXP("alloc tidsend=%4d tidrecv=%4d srcoff=%6d length=%6d,s=%d,e=%d\n",
1259 tidsendc->descid._desc_idx, tid_list->tsess_descid._desc_idx,
1260 tid_list->tsess_srcoff, tid_list->tsess_length,
1261 tid_list->tsess_unaligned_start,
1262 tid_list->tsess_unaligned_end
1263 );
1264
1265 /* We have no tids, we're expected to stuff everything in user
1266 * header words, so mark it as an eager packet */
1267 if (tid_list->tsess_tidcount > 0) {
1268 ips_dump_tids(&tidsendc->tid_list,
1269 "Received %d tids: ", tidsendc->tid_list.tsess_tidcount);
1270 }
1271
1272 /* Add as a pending op and ring up the timer */
1273 STAILQ_INSERT_TAIL(&protoexp->pend_sendq, tidsendc, next);
1274 psmi_timer_request(protoexp->timerq, &protoexp->timer_send, PSMI_TIMER_PRIO_1);
1275
1276 /* Consider breaking out of progress engine here */
1277 return PSM_OK;
1278 }
1279
1280 void __fastpath
1281 ips_protoexp_tid_release_ack(const struct ips_recvhdrq_event *rcv_ev)
1282 {
1283 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
1284 struct ips_tid_send_desc *tidsendc;
1285 ptl_arg_t desc_id = rcv_ev->p_hdr->data[1];
1286
1287 tidsendc = (struct ips_tid_send_desc *)
1288 psmi_mpool_find_obj_by_index(protoexp->tid_desc_send_pool,
1289 desc_id._desc_idx);
1290 _IPATH_VDBG("desc_id=%d (%p)\n", desc_id._desc_idx, tidsendc);
1291 if (tidsendc == NULL) {
1292 _IPATH_ERROR("OPCODE_TIDS_RELEASE_CONFIRM ERROR: Index %d is out of range\n",
1293 desc_id._desc_idx);
1294 }
1295 else {
1296 ptl_arg_t desc_tidsendc;
1297 psmi_mpool_get_obj_index_gen_count(tidsendc,
1298 &desc_tidsendc._desc_idx,
1299 &desc_tidsendc._desc_genc);
1300
1301 _IPATH_VDBG("desc_req:id=%d,gen=%d desc_sendc:id=%d,gen=%d\n",
1302 desc_id._desc_idx, desc_id._desc_genc,
1303 desc_tidsendc._desc_idx, desc_tidsendc._desc_genc);
1304
1305 /* See if the reference is still live and valid */
1306 if (desc_tidsendc.u64 == desc_id.u64) {
1307 psmi_timer_cancel(protoexp->timerq, &tidsendc->timer_tidrelease);
1308 psmi_timer_cancel(rcv_ev->proto->timerq,
1309 &tidsendc->tidflow.timer_send);
1310 psmi_timer_cancel(rcv_ev->proto->timerq,
1311 &tidsendc->tidflow.timer_ack);
1312 psmi_mpool_put(tidsendc);
1313 }
1314 }
1315 return;
1316 }
1317
1318 static
1319 psm_error_t __fastpath
1320 ips_scb_send_unaligned_data(ips_scb_t *scb)
1321 {
1322 struct ips_tid_send_desc *tidsendc = scb->tidsendc;
1323 struct ips_protoexp *protoexp = tidsendc->protoexp;
1324 uint8_t *bufptr = tidsendc->buffer;
1325 int frame_extra, i;
1326 uint8_t *packptr;
1327 uint8_t *unptr_beg = bufptr;
1328 uint8_t *unptr_end = bufptr + tidsendc->length -
1329 tidsendc->tid_list.tsess_unaligned_end;
1330 struct ips_flow *flow = &tidsendc->ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO];
1331
1332 psmi_assert(tidsendc->tid_idx == 0);
1333
1334 /* arg[0] is recv descriptor id */
1335 scb->ips_lrh.data[0] = tidsendc->tid_list.tsess_descid;
1336
1337 if (protoexp->proto->flags & IPS_PROTO_FLAG_CKSUM) {
1338 uint32_t cksum = 0xffffffff;
1339
1340 if (!ips_scbctrl_bufalloc(scb)) {
1341 ips_scbctrl_free(scb);
1342 return PSM_EP_NO_RESOURCES;
1343 }
1344
1345 cksum = ips_crc_calculate(tidsendc->length,
1346 (uint8_t*) tidsendc->buffer, cksum);
1347 *(uint32_t*) ips_scb_buffer(scb) = cksum;
1348 ips_scb_length(scb) = sizeof(cksum);
1349 scb->flags |= IPS_SEND_FLAG_HAS_CKSUM;
1350 }
1351
1352 // Make sure not to over read unaligned buffer
1353 packptr = (uint8_t *)&scb->ips_lrh.data[1].u32w0;
1354 for (i = 0; i < tidsendc->tid_list.tsess_unaligned_start; i++)
1355 packptr[i] = unptr_beg[i];
1356
1357 packptr = (uint8_t *)&scb->ips_lrh.data[1].u32w1;
1358 for (i = 0; i < tidsendc->tid_list.tsess_unaligned_end; i++)
1359 packptr[i] = unptr_end[i];
1360
1361 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_EXPTID_UNALIGNED;
1362 ips_scb_hdr_dlen(scb) = tidsendc->tid_list.tsess_unaligned_start +
1363 tidsendc->tid_list.tsess_unaligned_end;
1364
1365 ips_scb_cb(scb) = ips_tid_send_completion_unaligned_callback;
1366 ips_scb_cb_param(scb) = tidsendc;
1367 scb->flags |= IPS_SEND_FLAG_UNALIGNED_DATA | IPS_SEND_FLAG_ACK_REQ;
1368
1369 bufptr += tidsendc->tid_list.tsess_unaligned_start;
1370 frame_extra = tidsendc->tid_list.tsess_unaligned_start +
1371 tidsendc->tid_list.tsess_unaligned_end;
1372
1373
1374 tidsendc->remaining_bytes -= frame_extra;
1375
1376 tidsendc->buffer = bufptr;
1377
1378 /* Enqueue scb on the flow and flush */
1379 flow->fn.xfer.enqueue(flow, scb);
1380 flow->fn.xfer.flush(flow, NULL);
1381
1382 return PSM_OK;
1383 }
1384
1385 static
1386 ips_scb_t * __fastpath
1387 ips_scb_prepare_tid_sendctrl(struct ips_flow *flow,
1388 struct ips_tid_send_desc *tidsendc)
1389 {
1390 struct ips_protoexp *protoexp = tidsendc->protoexp;
1391 uint8_t *bufptr = tidsendc->buffer;
1392 uint16_t frame_len, frag_size, nfrag;
1393 int payload_size, idx;
1394 ips_scb_t *scb;
1395
1396 if ((scb = ips_scbctrl_alloc(&protoexp->tid_scbc_rv, 1, 0, 0)) == NULL)
1397 return NULL;
1398
1399 /*
1400 * Expected sends require 4-byte alignment, so we stuff whatever
1401 * misalignment in the header's available user bytes.
1402 *
1403 * In the current interface, misalignment can only occur at the
1404 * start or end of the packet, so we handle it as a special packet
1405 * before the first packet can be sent off.
1406 *
1407 * If checksum is enabled we send the checksum for the send window
1408 * wiithin/as an unaligned packet as well.
1409 */
1410
1411 if (tidsendc->length &&
1412 (tidsendc->tid_list.tsess_unaligned_start ||
1413 tidsendc->tid_list.tsess_unaligned_end ||
1414 (protoexp->proto->flags & IPS_PROTO_FLAG_CKSUM)) &&
1415 !(tidsendc->unaligned_sent)) {
1416
1417 /* Send unaligned data separately over ipsaddr->flow. Completion over
1418 * both flows is synchronized to generate TIDS_RELEASE. The receive will
1419 * only finish when tid release is received. */
1420 scb->tidsendc = tidsendc;
1421 if (ips_scb_send_unaligned_data(scb) != PSM_OK)
1422 return NULL;
1423
1424 /* Sent unaligned data */
1425 tidsendc->unaligned_sent = 1;
1426
1427
1428 /* Buffer may have been updated (unaligned start) */
1429 bufptr = tidsendc->buffer;
1430
1431 /* Try to obtain another scb after sending unaligned data */
1432 if ((scb = ips_scbctrl_alloc(&protoexp->tid_scbc_rv, 1, 0, 0)) == NULL)
1433 return NULL;
1434 }
1435
1436 if ((uintptr_t)bufptr & 0x3) {
1437 bufptr = psmi_malloc(protoexp->proto->ep,
1438 UNDEFINED, tidsendc->remaining_bytes);
1439 if (!bufptr) {
1440 ips_scbctrl_free(scb);
1441 return NULL;
1442 }
1443
1444 memcpy(bufptr, tidsendc->buffer, tidsendc->remaining_bytes);
1445 tidsendc->buffer = tidsendc->bounce_buf = bufptr;
1446 }
1447
1448 idx = tidsendc->tid_idx;
1449 scb->tidsendc = tidsendc;
1450 SLIST_NEXT(scb,next) = NULL;
1451
1452 scb->ips_lrh.sub_opcode = OPCODE_SEQ_MQ_EXPTID;
1453 scb->ips_lrh.data[0] = tidsendc->tid_list.tsess_descid;
1454 scb->ips_lrh.data[1] = tidsendc->descid;
1455 scb->tid = tidsendc->tid_list.tsess_list[idx].tid;
1456 scb->tsess = (void *)&tidsendc->tid_list.tsess_list[idx];
1457 scb->offset = tidsendc->offset;
1458 scb->payload = (void *) bufptr;
1459
1460 /*
1461 * Loop over the tid session list, count the frag number and payload size.
1462 * The payload size is limited by the pbc.length field which is 16 bits in
1463 * DWORD, including both message header and payload. This translates to
1464 * less than 256K payload. So 128K is used.
1465 */
1466 nfrag = 0;
1467 payload_size = 0;
1468 frag_size = min(protoexp->tid_send_fragsize, flow->path->epr_mtu);
1469 frame_len = min(tidsendc->remaining_bytes_in_page, frag_size);
1470 while (1) {
1471 nfrag++;
1472 payload_size += frame_len;
1473
1474 /* adjust counter and pointers */
1475 tidsendc->remaining_bytes -= frame_len;
1476 tidsendc->remaining_bytes_in_page -= frame_len;
1477 tidsendc->offset += frame_len;
1478
1479 if (!tidsendc->remaining_bytes_in_page) {
1480 /* Done with this page, move on to the next tid */
1481 tidsendc->tid_idx++;
1482 tidsendc->remaining_bytes_in_page =
1483 tidsendc->tid_list.tsess_list[tidsendc->tid_idx].length;
1484 tidsendc->offset =
1485 tidsendc->tid_list.tsess_list[tidsendc->tid_idx].offset;
1486
1487 /* The payload size is limited by the pbc.length field which
1488 * is 16 bits in DWORD, including both message header and
1489 * payload. This translates to less than 256K payload. So 128K
1490 * is used. */
1491 /* break when current page is done */
1492 if (payload_size > 131072) break;
1493 }
1494
1495 #if 0
1496 if (1) {
1497 #else
1498 if (flow->transfer == PSM_TRANSFER_PIO) {
1499 #endif
1500 break; /* turn on to use single frag-size packet */
1501 }
1502
1503 if (!tidsendc->remaining_bytes) break;
1504 frame_len = min(tidsendc->remaining_bytes_in_page, frag_size);
1505 }
1506 scb->nfrag = nfrag;
1507 scb->frag_size = frag_size;
1508 scb->payload_size = payload_size;
1509 scb->tsess_length = sizeof(ips_tid_session_member) *
1510 (tidsendc->tid_idx - idx);
1511
1512 /* Keep track of latest buffer location so we restart at the
1513 * right location, if we don't complete the transfer */
1514 tidsendc->buffer = bufptr + payload_size;
1515
1516 /* If last packet, we want a completion notification */
1517 if (!tidsendc->remaining_bytes) {
1518 scb->flags = (IPS_SEND_FLAG_ACK_REQ | IPS_SEND_FLAG_EXPECTED_DONE);
1519 scb->callback = ips_tid_send_completion_callback;
1520 scb->cb_param = tidsendc;
1521
1522 tidsendc->is_complete = 1;
1523 } else {
1524 scb->flags = IPS_SEND_FLAG_HDR_SUPPRESS;
1525 scb->callback = NULL;
1526 scb->cb_param = NULL;
1527 }
1528
1529 #if 0
1530 if (1) {
1531 #else
1532 if (flow->transfer == PSM_TRANSFER_PIO) {
1533 #endif
1534 /* turn on to use single frag-size packet */
1535 /* Do not suppress header every hdr_pkt_interval or the last packet */
1536 if ((++tidsendc->frame_send % protoexp->hdr_pkt_interval) == 0) {
1537 scb->flags &= ~IPS_SEND_FLAG_HDR_SUPPRESS;
1538 scb->flags |= IPS_SEND_FLAG_ACK_REQ; /* Request an ACK */
1539 }
1540 }
1541
1542 return scb;
1543 }
1544
1545 /*
1546 * Returns:
1547 *
1548 * PSM_OK: scb was allocated for at least one frame, the packet may be queued
1549 * or actually sent.
1550 *
1551 * PSM_OK_NO_PROGRESS: Reached a limit on the maximum number of sends we allow
1552 * to be enqueued before polling receive queue.
1553 *
1554 * PSM_EP_NO_RESOURCES: No scbs, available, a callback will be issued when more
1555 * scbs become available.
1556 *
1557 * PSM_TIMEOUT: PIO-busy or DMA-busy, stop trying to send for now.
1558 *
1559 */
1560
1561 psm_error_t __fastpath
1562 ips_tid_send_exp(struct ips_tid_send_desc *tidsendc)
1563 {
1564 ips_scb_t *scb = NULL;
1565 psm_error_t err = PSM_OK, err_f;
1566 struct ips_protoexp *protoexp = tidsendc->protoexp;
1567 struct ips_proto *proto = protoexp->proto;
1568 struct ips_flow *flow = &tidsendc->tidflow;
1569
1570 /*
1571 * We aggressively try to grab as many scbs as possible, enqueue them to a
1572 * flow and flush them when either we're out of scbs our we've completely
1573 * filled the send request.
1574 */
1575 while (!tidsendc->is_complete)
1576 {
1577 if_pf (tidsendc->tid_list.tsess_tidcount &&
1578 (tidsendc->tid_idx >= tidsendc->tid_list.tsess_tidcount ||
1579 tidsendc->tid_idx < 0) )
1580 ips_expsend_tiderr(tidsendc);
1581
1582 if ((scb = ips_scb_prepare_tid_sendctrl(flow, tidsendc)) == NULL) {
1583 proto->stats.scb_exp_unavail_cnt++;
1584 err = PSM_EP_NO_RESOURCES;
1585 break;
1586 }
1587 else {
1588 flow->fn.xfer.enqueue(flow, scb);
1589 }
1590 }
1591
1592 if (!SLIST_EMPTY(&flow->scb_pend)) { /* Something to flush */
1593 int num_sent;
1594 err_f = flow->fn.xfer.flush(flow, &num_sent);
1595
1596 if (err != PSM_EP_NO_RESOURCES) {
1597 /* PSM_EP_NO_RESOURCES is reserved for out-of-scbs */
1598 if (err_f == PSM_EP_NO_RESOURCES)
1599 err = PSM_TIMEOUT; /* force a resend reschedule */
1600 else if (err_f == PSM_OK && num_sent > 0 &&
1601 !ips_ptl_recvq_isempty(protoexp->ptl))
1602 err = PSM_OK_NO_PROGRESS; /* force a rcvhdrq service */
1603 }
1604 }
1605
1606 return err;
1607 }
1608
1609 static
1610 psm_error_t __recvpath
1611 ips_tid_pendsend_timer_callback(struct psmi_timer *timer, uint64_t current)
1612 {
1613 struct ips_protoexp *protoexp = (struct ips_protoexp *) timer->context;
1614 struct ips_tid_send_pend *phead = &protoexp->pend_sendq;
1615 struct ips_tid_send_desc *tidsendc;
1616 psm_error_t err = PSM_OK;
1617
1618 while (!STAILQ_EMPTY(phead)) {
1619 tidsendc = STAILQ_FIRST(phead);
1620
1621 err = ips_tid_send_exp(tidsendc);
1622
1623 if (tidsendc->is_complete)
1624 STAILQ_REMOVE_HEAD(phead, next);
1625
1626 if (err == PSM_OK) {
1627 /* Was able to complete the send, keep going */
1628
1629 #if 0
1630 _IPATH_EXP("tidsess=%6d tid=%4d @ %3d size=%4d offset=%4d, next=%p\n",
1631 tidsendc->descid.u32w0,
1632 tidsendc->tid_list.tsess_list[tidsendc->tid_idx].tid,
1633 tidsendc->tid_idx,
1634 tidsendc->length,
1635 tidsendc->length - tidsendc->remaining_bytes,
1636 STAILQ_FIRST(phead)
1637 );
1638 #endif
1639 }
1640 else if (err == PSM_EP_NO_RESOURCES) {
1641 /* No more sendbufs available, sendbuf callback will requeue this
1642 * timer */
1643 break;
1644 }
1645 else if (err == PSM_TIMEOUT) {
1646 /* Always a case of try later:
1647 * On PIO flow, means no send pio bufs available
1648 * On DMA flow, means kernel can't queue request or would have to block
1649 */
1650 psmi_timer_request(protoexp->proto->timerq,
1651 &protoexp->timer_send,
1652 get_cycles() + protoexp->proto->timeout_send);
1653 break;
1654 }
1655 else {
1656 /* Forced to reschedule later so we can check receive queue */
1657 psmi_assert(err == PSM_OK_NO_PROGRESS);
1658 psmi_timer_request(protoexp->proto->timerq,
1659 &protoexp->timer_send, PSMI_TIMER_PRIO_1);
1660 break;
1661 }
1662 }
1663
1664 return PSM_OK;
1665 }
1666
1667 // Right now, in the kernel we are allowing for virtually non-contiguous pages,
1668 // in a single call, and we are therefore locking one page at a time, but since
1669 // the intended use of this routine is for a single group of
1670 // virtually contiguous pages, that should change to improve
1671 // performance. That means possibly changing the calling MPI code.
1672 // Doing so gets rid of some of the loop stuff here, and in the driver,
1673 // and allows for a single call to the core VM code in the kernel,
1674 // rather than one per page, definitely improving performance.
1675
1676 static
1677 psm_error_t __fastpath
1678 ips_tid_recv_alloc_frag(struct ips_protoexp *protoexp,
1679 void *buf, uint32_t buflen,
1680 ips_tid_session_list *tid_list,
1681 uint64_t *ts_map)
1682 {
1683 uint16_t unalignment;
1684 uint32_t remaining_buffer_size = buflen;
1685 uint32_t num_tids;
1686 uint32_t num_tids_avail = ips_tid_num_available(&protoexp->tidc);
1687 uint16_t tidids[IPS_TID_MAX_TIDS];
1688 void *bufmap;
1689 uint8_t *bufptr = (uint8_t *) buf;
1690 const uint32_t page_size = ips_tid_page_size(&protoexp->tidc);
1691 const uint32_t page_offset_mask = protoexp->tid_page_offset_mask;
1692 int i;
1693 psm_error_t err = PSM_OK;
1694
1695 /*
1696 * The following remaining_buffer_size calculation
1697 * does not work with buflen<4 and byte aligned
1698 * buf, it can get negative value.
1699 * In function ips_tid_pendtids_timer_callback(),
1700 * we try to avoid nbytes_this(which is buflen)
1701 * to be a few bytes.
1702 */
1703 if (buflen < 4) {
1704 tid_list->tsess_unaligned_start = buflen;
1705 tid_list->tsess_unaligned_end = 0;
1706 remaining_buffer_size = 0;
1707 } else {
1708 tid_list->tsess_unaligned_start = unalignment =
1709 ((uintptr_t) buf & 3) ? (4 - ((uintptr_t) buf & 3)) : 0;
1710 remaining_buffer_size -= unalignment;
1711 bufptr += unalignment;
1712
1713 tid_list->tsess_unaligned_end = unalignment =
1714 remaining_buffer_size & 3;
1715 remaining_buffer_size -= unalignment;
1716 }
1717
1718 bufmap = bufptr;
1719 psmi_assert_always(ips_tid_num_required(&protoexp->tidc, bufmap,
1720 remaining_buffer_size) <= num_tids_avail);
1721
1722 tid_list->tsess_list[0].tid = 0;
1723 tid_list->tsess_list[0].offset = 0;
1724 tid_list->tsess_list[0].length = 0;
1725
1726 for (i = 0, num_tids = 0; remaining_buffer_size && i < num_tids_avail; i++) {
1727 uint32_t page_off = (uintptr_t) bufptr & page_offset_mask;
1728 uint32_t page_len = min(remaining_buffer_size, page_size - page_off);
1729 tid_list->tsess_list[i].offset = page_off;
1730 tid_list->tsess_list[i].length = page_len;
1731 bufptr += page_len;
1732 remaining_buffer_size -= page_len;
1733 tidids[i] = 0; /* Ensure tidids[i] is never seen as uninitialized */
1734 num_tids++;
1735 }
1736 psmi_assert_always(remaining_buffer_size == 0);
1737
1738 if (num_tids &&
1739 (err = ips_tid_acquire(&protoexp->tidc,
1740 (void *) ((uintptr_t) bufmap &
1741 (uintptr_t) protoexp->tid_page_mask),
1742 num_tids, ts_map, tidids)))
1743 goto fail;
1744
1745 tid_list->tsess_tidcount = num_tids;
1746 for (i = 0; i < num_tids; i++)
1747 tid_list->tsess_list[i].tid = tidids[i];
1748
1749 ips_dump_tids(tid_list, "Registered %d tids: ", num_tids);
1750
1751 fail:
1752 return err;
1753 }
1754
1755 static
1756 void
1757 ips_tid_mpool_tidrecv_callback(void *context)
1758 {
1759 struct ips_protoexp *protoexp = (struct ips_protoexp *) context;
1760
1761 if (!STAILQ_EMPTY(&protoexp->pend_getreqsq))
1762 psmi_timer_request(protoexp->proto->timerq,
1763 &protoexp->timer_getreqs, PSMI_TIMER_PRIO_1);
1764
1765 return;
1766 }
1767
1768 static
1769 __fastpath
1770 struct ips_tid_recv_desc *
1771 ips_tid_recv_alloc(struct ips_protoexp *protoexp, ips_epaddr_t *ipsaddr,
1772 const struct ips_tid_get_request *getreq, uint32_t nbytes_this)
1773 {
1774 struct ips_tid_recv_desc *tidrecvc;
1775 psm_error_t err = PSM_OK;
1776
1777 tidrecvc = (struct ips_tid_recv_desc *)
1778 psmi_mpool_get(protoexp->tid_desc_recv_pool);
1779 if (tidrecvc == NULL)
1780 return NULL;
1781
1782 tidrecvc->context = &protoexp->proto->ep->context;
1783 tidrecvc->protoexp = protoexp;
1784 tidrecvc->ipsaddr = ipsaddr;
1785 tidrecvc->state = TIDRECVC_STATE_GRANT;
1786 tidrecvc->buffer =
1787 (void *)((uintptr_t) getreq->tidgr_lbuf + getreq->tidgr_offset);
1788 tidrecvc->num_recv_hdrs = 0;
1789 tidrecvc->recv_msglen = nbytes_this;
1790 tidrecvc->tid_list.tsess_tidcount = 0;
1791 tidrecvc->getreq = (struct ips_tid_get_request *) getreq;
1792 tidrecvc->grant_cnt = 0;
1793 tidrecvc->recv_framecnt = 0;
1794 tidrecvc->flags = 0;
1795 tidrecvc->tidflow_active_gen = IPS_TF_INVALID_GENERATION;
1796 tidrecvc->ctrl_msg_queued = 0;
1797 tidrecvc->cksum = 0xb5b5b5b5;
1798 tidrecvc->stats.nSeqErr = 0;
1799 tidrecvc->stats.nGenErr = 0;
1800 tidrecvc->stats.nReXmit = 0;
1801 tidrecvc->stats.nErrChkReceived = 0;
1802
1803 if ((err = ips_tf_allocate(&protoexp->tfctrl,
1804 &tidrecvc->tidflow_idx,
1805 &tidrecvc->tidflow_active_gen))){
1806 /* Unable to get a tidflow for expected protocol. */
1807 psmi_mpool_put(tidrecvc);
1808 /* XXX log this event */
1809 return NULL;
1810 }
1811
1812 tidrecvc->tidflow_genseq.flow = tidrecvc->tidflow_idx;
1813 tidrecvc->tidflow_genseq.gen = tidrecvc->tidflow_active_gen;
1814 tidrecvc->tidflow_genseq.seq = rand_r(&protoexp->tidflow_seed) & 0x3ff;
1815
1816 ipath_tidflow_set_entry(tidrecvc->context->ctrl,
1817 tidrecvc->tidflow_genseq.flow,
1818 tidrecvc->tidflow_genseq.gen,
1819 tidrecvc->tidflow_genseq.seq);
1820
1821 tidrecvc->tidflow_nswap_gen = 0;
1822 tidrecvc->tid_list.tsess_type = IPS_TID_SESSTYPE_MEMBER_LIST;
1823 tidrecvc->tid_list.tsess_tidcount = 0;
1824 tidrecvc->tid_list.tsess_tidlist_length = 0;
1825 tidrecvc->tid_list.tsess_unaligned_start = 0;
1826 tidrecvc->tid_list.tsess_unaligned_end = 0;
1827
1828 tidrecvc->tid_list.tsess_descid._desc_idx =
1829 psmi_mpool_get_obj_index(tidrecvc);
1830 tidrecvc->tid_list.tsess_descid._desc_genc =
1831 psmi_mpool_get_obj_gen_count(tidrecvc);
1832
1833 tidrecvc->tid_list.tsess_seqno = getreq->tidgr_desc_seqno;
1834 tidrecvc->tid_list.tsess_srcoff = getreq->tidgr_offset;
1835 tidrecvc->tid_list.tsess_length = nbytes_this;
1836
1837 psmi_timer_entry_init(&tidrecvc->timer_tidreq,
1838 ips_tid_grant_timer_callback, tidrecvc);
1839
1840 if (nbytes_this > 0) {
1841 if ((err = ips_tid_recv_alloc_frag(protoexp, tidrecvc->buffer,
1842 nbytes_this, &tidrecvc->tid_list, tidrecvc->ts_map))) {
1843 tidrecvc->tid_list.tsess_tidcount = 0;
1844 ips_tf_deallocate(&protoexp->tfctrl, tidrecvc->tidflow_idx);
1845 psmi_mpool_put(tidrecvc);
1846 /* XXX log me !!! */
1847 return NULL;
1848 }
1849 if (protoexp->tid_flags & IPS_PROTOEXP_FLAG_TID_DEBUG)
1850 {
1851 int num_tids = tidrecvc->tid_list.tsess_tidcount;
1852 int tid, i;
1853 for (i = 0; i < num_tids; i++) {
1854 tid = tidrecvc->tid_list.tsess_list[i].tid;
1855 psmi_assert(protoexp->tid_info[tid].state == TIDSTATE_FREE);
1856 protoexp->tid_info[tid].tid = tid;
1857 protoexp->tid_info[tid].state = TIDSTATE_USED;
1858 protoexp->tid_info[tid].tidrecvc = tidrecvc;
1859 }
1860 }
1861 }
1862
1863 /* This gets sent out as a control message, so we need to force 4-byte IB
1864 * alignment */
1865 tidrecvc->tid_list.tsess_tidlist_length = (uint16_t)
1866 PSMI_ALIGNUP((sizeof(ips_tid_session_list) +
1867 (tidrecvc->tid_list.tsess_tidcount *
1868 sizeof(ips_tid_session_member))), 4);
1869
1870 _IPATH_EXP("alloc tidrecv=%d, ntid=%d, paylen=%d\n",
1871 tidrecvc->tid_list.tsess_descid._desc_idx,
1872 tidrecvc->tid_list.tsess_tidcount,
1873 tidrecvc->tid_list.tsess_tidlist_length);
1874
1875 return tidrecvc;
1876 }
1877
1878 static
1879 psm_error_t __recvpath
1880 ips_tid_pendtids_timer_callback(struct psmi_timer *timer, uint64_t current)
1881 {
1882 struct ips_protoexp *protoexp = (struct ips_protoexp *) timer->context;
1883 struct ips_tid_get_pend *phead = &protoexp->pend_getreqsq;
1884 struct ips_tid_get_request *getreq;
1885 struct ips_tid_recv_desc *tidrecvc;
1886 uint32_t nbytes_this, leftover;
1887 uint64_t t_cyc;
1888 uintptr_t bufptr;
1889 psm_epaddr_t epaddr;
1890 ptl_epaddr_t *ipsaddr;
1891 psm_error_t err = PSM_OK;
1892
1893 while (!STAILQ_EMPTY(phead)) {
1894 getreq = STAILQ_FIRST(phead);
1895 epaddr = getreq->tidgr_epaddr;
1896
1897 next_epaddr:
1898 ipsaddr = epaddr->ptladdr;
1899 protoexp = ipsaddr->proto->protoexp;
1900 nbytes_this = min(getreq->tidgr_length - getreq->tidgr_offset,
1901 getreq->tidgr_rndv_winsz);
1902 /*
1903 * if the leftover is less than half window size,
1904 * we reduce nbytes_this by half, we want to avoid
1905 * to send a few bytes in a tid transaction.
1906 */
1907 leftover = getreq->tidgr_length -
1908 (getreq->tidgr_offset + nbytes_this);
1909 if (leftover && leftover < getreq->tidgr_rndv_winsz/2) {
1910 nbytes_this /= 2;
1911 }
1912
1913 bufptr = (uintptr_t) getreq->tidgr_lbuf + getreq->tidgr_offset;
1914
1915 if ((ips_tid_num_required(&protoexp->tidc, (void *) bufptr, nbytes_this) > ips_tid_num_available(&protoexp->tidc)) ||
1916 !ips_tf_available(&protoexp->tfctrl)) {
1917 /* We're out of tids/tidflow, tid release will requeue the callback */
1918 ;
1919 }
1920 else if ((tidrecvc = ips_tid_recv_alloc(protoexp, ipsaddr,
1921 getreq, nbytes_this)) != NULL) {
1922
1923 err = ips_proto_send_ctrl_message(&ipsaddr->
1924 flows[protoexp->tid_ep_flow],
1925 OPCODE_TIDS_GRANT,
1926 &tidrecvc->ctrl_msg_queued,
1927 &tidrecvc->tid_list);
1928
1929 if (err != PSM_EP_NO_RESOURCES) {
1930 tidrecvc->grant_cnt++;
1931 t_cyc = get_cycles() + protoexp->tid_to_cyc_min;
1932 }
1933 else
1934 t_cyc = get_cycles() + protoexp->proto->timeout_send;
1935
1936 psmi_timer_request_always(protoexp->timerq,
1937 &tidrecvc->timer_tidreq, t_cyc);
1938
1939 getreq->tidgr_offset += nbytes_this;
1940 _IPATH_VDBG("GRANT tididx=%d.%d srcoff=%d nbytes=%d/%d\n",
1941 tidrecvc->tid_list.tsess_descid._desc_idx,
1942 getreq->tidgr_desc_seqno,
1943 getreq->tidgr_offset, nbytes_this, getreq->tidgr_length);
1944
1945 getreq->tidgr_desc_seqno++;
1946 if (getreq->tidgr_offset == getreq->tidgr_length) {
1947 getreq->tidgr_protoexp = NULL;
1948 getreq->tidgr_epaddr = NULL;
1949 STAILQ_REMOVE_HEAD(phead, tidgr_next);
1950 continue;
1951 }
1952 epaddr = epaddr->mctxt_next;
1953 goto next_epaddr;
1954 }
1955 else {
1956 /* out of tidrecv desc. The not-empty tidrecv mpool callback will
1957 * cause us to requeue the getreq on the active timer queue */
1958 ;
1959 }
1960
1961 epaddr = epaddr->mctxt_next;
1962 if (epaddr != getreq->tidgr_epaddr) goto next_epaddr;
1963 break;
1964 }
1965 return PSM_OK; /* XXX err-broken */
1966 }
1967
1968 static
1969 psm_error_t __fastpath
1970 ips_tid_recv_free(struct ips_tid_recv_desc *tidrecvc)
1971 {
1972 struct ips_tid_get_request *getreq = tidrecvc->getreq;
1973 struct ips_protoexp *protoexp = tidrecvc->protoexp;
1974 int tidcount = tidrecvc->tid_list.tsess_tidcount;
1975 psm_error_t err = PSM_OK;
1976
1977 psmi_assert(getreq != NULL);
1978
1979 /* If checksum is enabled, make sure we have valid data for window */
1980 if (protoexp->proto->flags & IPS_PROTO_FLAG_CKSUM) {
1981 uint32_t cksum = ips_crc_calculate(tidrecvc->recv_msglen,
1982 (uint8_t*) tidrecvc->buffer,
1983 0xffffffff);
1984 if (tidrecvc->cksum != cksum) {
1985 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
1986 "ErrPkt: Checksum mismatch. Expected: 0x%08x, Received: 0x%08x Source LID: %i. Rendezvous stats: nSeqErr: %d, nGenErr: %d, nReXmits: %d, nErrChkGen: %d. Aborting! \n", tidrecvc->cksum, cksum, __be16_to_cpu(tidrecvc->ipsaddr->tidgr_flow.path->epr_dlid), tidrecvc->stats.nSeqErr, tidrecvc->stats.nGenErr, tidrecvc->stats.nReXmit, tidrecvc->stats.nErrChkReceived);
1987 ips_proto_dump_data(tidrecvc->buffer, tidrecvc->recv_msglen);
1988
1989 /* TODO: In order to recover from this we need to restart the rendezvous
1990 * window again. This requires modifying the sender to not complete the
1991 * send locally till TID_RELEASE_CONFIRM is released - currently it
1992 * locally completes before sending the TID_RELEASE message.
1993 */
1994 }
1995 }
1996
1997 psmi_assert_always(tidrecvc->state == TIDRECVC_STATE_DONE);
1998
1999 if (tidcount > 0) {
2000 if (protoexp->tid_flags & IPS_PROTOEXP_FLAG_TID_DEBUG)
2001 {
2002 int num_tids = tidrecvc->tid_list.tsess_tidcount;
2003 int tid, i;
2004 for (i = 0; i < num_tids; i++) {
2005 tid = tidrecvc->tid_list.tsess_list[i].tid;
2006 psmi_assert(protoexp->tid_info[tid].state == TIDSTATE_USED);
2007 psmi_assert(protoexp->tid_info[tid].tidrecvc == tidrecvc);
2008 protoexp->tid_info[tid].state = TIDSTATE_FREE;
2009 }
2010 }
2011
2012 ips_dump_tids(&tidrecvc->tid_list, "Deregistered %d tids: ",
2013 tidrecvc->tid_list.tsess_tidcount);
2014
2015 if ((err = ips_tid_release(&tidrecvc->protoexp->tidc,
2016 tidrecvc->ts_map, tidcount)))
2017 goto fail;
2018
2019 }
2020
2021 getreq->tidgr_bytesdone += tidrecvc->recv_msglen;
2022
2023 _IPATH_EXP("req=%p bytes=%d/%d\n",
2024 getreq->tidgr_ucontext,
2025 getreq->tidgr_bytesdone,
2026 getreq->tidgr_length);
2027
2028 tidrecvc->state = TIDRECVC_STATE_FREE;
2029 psmi_mpool_put(tidrecvc);
2030
2031 if (getreq->tidgr_bytesdone == getreq->tidgr_length) {
2032 if (getreq->tidgr_callback)
2033 getreq->tidgr_callback(getreq->tidgr_ucontext);
2034 psmi_mpool_put(getreq);
2035 }
2036
2037 /* We just released some tids. If requests are waiting on tids to be
2038 * freed, queue up the timer */
2039 if (tidcount > 0) {
2040 if (getreq->tidgr_offset < getreq->tidgr_length) {
2041 #if 0
2042 psmi_timer_request(getreq->tidgr_protoexp->timerq,
2043 &getreq->tidgr_protoexp->timer_getreqs,
2044 PSMI_TIMER_PRIO_1);
2045 #endif
2046 ips_tid_pendtids_timer_callback(
2047 &getreq->tidgr_protoexp->timer_getreqs, 0);
2048 }
2049
2050 if (!STAILQ_EMPTY(&protoexp->pend_getreqsq)) {
2051 psmi_timer_request(protoexp->timerq,
2052 &protoexp->timer_getreqs,
2053 PSMI_TIMER_PRIO_1);
2054 }
2055 }
2056
2057 fail:
2058 return err;
2059 }
2060
2061 int
2062 __fastpath
2063 ips_protoexp_tid_release(const struct ips_recvhdrq_event *rcv_ev)
2064 {
2065 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
2066 struct ips_tid_recv_desc *tidrecvc;
2067 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
2068 ptl_arg_t desc_id = p_hdr->data[0];
2069 ptl_arg_t args[3];
2070 int rc = IPS_RECVHDRQ_CONTINUE;
2071
2072 args[0] = p_hdr->data[0];
2073 args[1] = p_hdr->data[1];
2074
2075 tidrecvc = (struct ips_tid_recv_desc *)
2076 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
2077 desc_id._desc_idx);
2078
2079 if (tidrecvc == NULL)
2080 _IPATH_ERROR("OPCODE_TIDS_RELEASE: ERROR: Index %d is out of range\n",
2081 desc_id._desc_idx);
2082 else {
2083 ptl_arg_t desc_tidrecvc;
2084 psmi_mpool_get_obj_index_gen_count(tidrecvc,
2085 &desc_tidrecvc._desc_idx,
2086 &desc_tidrecvc._desc_genc);
2087
2088 _IPATH_VDBG("desc_req:id=%d,gen=%d desc_tidc:id=%d,gen=%d\n",
2089 desc_id._desc_idx, desc_id._desc_genc,
2090 desc_tidrecvc._desc_idx, desc_tidrecvc._desc_genc);
2091
2092 /* See if the reference is still live and valid */
2093 if (desc_tidrecvc.u64 == desc_id.u64)
2094 ips_tid_recv_free(tidrecvc);
2095 }
2096
2097 /* Unconditionally echo back the confirmation. If the release is a dupe
2098 * because a previous confirmation was lost, it still needs to be released
2099 * at the other end. */
2100 ips_proto_send_ctrl_message(&rcv_ev->ipsaddr->flows[protoexp->tid_ep_flow],
2101 OPCODE_TIDS_RELEASE_CONFIRM,
2102 &rcv_ev->ipsaddr->ctrl_msg_queued,
2103 args);
2104 return rc;
2105 }
2106
2107 int __fastpath
2108 ips_protoexp_build_ctrl_message(struct ips_protoexp *protoexp,
2109 struct ptl_epaddr *ipsaddr,
2110 ptl_arg_t *pargs,
2111 uint16_t *pkt_flags, uint8_t opcode,
2112 void *payload)
2113 {
2114 switch (opcode) {
2115 case OPCODE_TIDS_GRANT:
2116 {
2117 ips_tid_session_list *tid_list = (ips_tid_session_list *) payload;
2118 uint32_t desc_idx = tid_list->tsess_descid._desc_idx;
2119 struct ips_tid_recv_desc *tidrecvc = (struct ips_tid_recv_desc *)
2120 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
2121 desc_idx);
2122 if (tidrecvc == NULL) return -1;
2123
2124 pargs[0].u32w0 = tidrecvc->getreq->tidgr_sendtoken;
2125 pargs[0].u32w1 = tidrecvc->getreq->tidgr_length;
2126 pargs[1].u32w0 = tidrecvc->tidflow_genseq.val;
2127
2128 if (tidrecvc->grant_cnt >= protoexp->tid_to_intr &&
2129 ipsaddr->flags & SESS_FLAG_HAS_RCVTHREAD &&
2130 !(tidrecvc->getreq->tidgr_flags & IPS_PROTOEXP_TIDGET_PEERWAIT))
2131 {
2132
2133 *pkt_flags |= INFINIPATH_KPF_INTR;
2134 protoexp->tid_intr_reqs++;
2135 }
2136 return tid_list->tsess_tidlist_length;
2137 break;
2138 }
2139
2140 case OPCODE_TIDS_RELEASE:
2141 case OPCODE_TIDS_RELEASE_CONFIRM:
2142 case OPCODE_TIDS_GRANT_ACK:
2143 {
2144 ptl_arg_t *args = (ptl_arg_t *) payload;
2145 pargs[0].u64w0 = args[0].u64w0;
2146 pargs[1].u64w0 = args[1].u64w0;
2147 if (opcode == OPCODE_TIDS_RELEASE) {
2148 uint32_t release_cnt = args[2].u32w0;
2149 if (release_cnt >= protoexp->tid_to_intr &&
2150 ipsaddr->flags & SESS_FLAG_HAS_RCVTHREAD)
2151 {
2152 *pkt_flags |= INFINIPATH_KPF_INTR;
2153 protoexp->tid_intr_reqs++;
2154 }
2155 }
2156 return 0;
2157 }
2158 default:
2159 return 0;
2160 }
2161 }
2162
2163 void
2164 __fastpath
2165 ips_protoexp_handle_tiderr(const struct ips_recvhdrq_event *rcv_ev)
2166 {
2167 struct ips_tid_recv_desc *tidrecvc;
2168 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
2169 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
2170
2171 ptl_arg_t desc_id = p_hdr->data[0];
2172 ptl_arg_t desc_tidrecvc;
2173 int tid = IPS_HDR_TID(p_hdr);
2174
2175 /* Expected sends not enabled */
2176 if (protoexp == NULL)
2177 return;
2178
2179 /* Not doing extra tid debugging or not really a tiderr */
2180 if (!(protoexp->tid_flags & IPS_PROTOEXP_FLAG_TID_DEBUG) ||
2181 !(rcv_ev->error_flags & INFINIPATH_RHF_H_TIDERR))
2182 return;
2183
2184 if (tid >= IPS_TID_MAX_TIDS || rcv_ev->ptype != RCVHQ_RCV_TYPE_EXPECTED) {
2185 _IPATH_ERROR("Unexpected tid value %d or ptype %d is not expected "
2186 "in tid debugging\n", tid, rcv_ev->ptype);
2187 return;
2188 }
2189
2190 tidrecvc = (struct ips_tid_recv_desc *)
2191 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
2192 desc_id._desc_idx);
2193
2194 if (tidrecvc != NULL)
2195 psmi_mpool_get_obj_index_gen_count(tidrecvc,
2196 &desc_tidrecvc._desc_idx,
2197 &desc_tidrecvc._desc_genc);
2198
2199 if (protoexp->tid_info[tid].state != TIDSTATE_USED) {
2200 char buf[128];
2201 char *s = "invalid (not even in table)";
2202 if (tidrecvc != NULL) {
2203 if (desc_tidrecvc._desc_idx == desc_id._desc_idx) {
2204 if (desc_tidrecvc._desc_genc == desc_id._desc_genc)
2205 s = "valid";
2206 else {
2207 snprintf(buf, sizeof buf - 1, "valid session, but wrong "
2208 "generation (gen=%d,received=%d)",
2209 desc_tidrecvc._desc_genc, desc_id._desc_genc);
2210 buf[sizeof buf - 1] = '\0';
2211 s = buf;
2212 }
2213 }
2214 else {
2215 snprintf(buf, sizeof buf - 1, "invalid session %d",
2216 desc_id._desc_idx);
2217 buf[sizeof buf - 1] = '\0';
2218 s = buf;
2219 }
2220
2221 if (protoexp->tid_info[tid].tidrecvc != tidrecvc) {
2222 _IPATH_ERROR("tid %d not a known member of tidsess %d\n", tid,
2223 desc_id._desc_idx);
2224 }
2225 }
2226
2227 _IPATH_ERROR("tid %d is marked unused (session=%d): %s\n", tid,
2228 desc_id._desc_idx, s);
2229 }
2230 return;
2231 }
2232
2233 void
2234 __fastpath
2235 ips_protoexp_handle_data_err(const struct ips_recvhdrq_event *rcv_ev)
2236 {
2237 struct ips_tid_recv_desc *tidrecvc;
2238 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
2239 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
2240 int hdr_err = rcv_ev->error_flags & INFINIPATH_RHF_H_IHDRERR;
2241 uint8_t op_code = __be32_to_cpu(p_hdr->bth[0]) >> 24 & 0xFF;
2242 char pktmsg[128];
2243 char errmsg[256];
2244
2245 ips_proto_get_rhf_errstring(rcv_ev->error_flags, pktmsg, sizeof(pktmsg));
2246
2247 snprintf(errmsg, sizeof(errmsg),
2248 "%s pkt type opcode 0x%x at hd=0x%x %s\n",
2249 (rcv_ev->ptype == RCVHQ_RCV_TYPE_EAGER) ? "Eager" :
2250 (rcv_ev->ptype == RCVHQ_RCV_TYPE_EXPECTED) ? "Expected" :
2251 (rcv_ev->ptype == RCVHQ_RCV_TYPE_NON_KD) ? "Non-kd" :
2252 "<Error>",
2253 op_code, rcv_ev->recvq->state->hdrq_head, pktmsg);
2254
2255 if (!hdr_err) {
2256 uint32_t tid_recv_sessid;
2257 ptl_arg_t desc_id = p_hdr->data[0];
2258 psmi_seqnum_t sequence_num;
2259 uint32_t cur_flowgenseq, tfgen, tfseq;
2260 uint16_t kdeth_cksum;
2261
2262 /* See if the KDETH checksum validates */
2263 kdeth_cksum =
2264 (uint16_t) IPATH_LRH_BTH +
2265 (uint16_t) (__be16_to_cpu(p_hdr->lrh[2])) -
2266 (uint16_t) ((__le32_to_cpu(p_hdr->iph.ver_context_tid_offset)>>16) &
2267 LOWER_16_BITS) -
2268 (uint16_t) (__le32_to_cpu(p_hdr->iph.ver_context_tid_offset) &
2269 LOWER_16_BITS) -
2270 (uint16_t) __le16_to_cpu(p_hdr->iph.pkt_flags);
2271
2272 if (kdeth_cksum != __le16_to_cpu(p_hdr->iph.chksum)) {
2273 _IPATH_EPDBG("Data Error Pkt With Invalid KDETH Checksum: Computed: 0x%04x, IPH_CKSUM: 0x%04x %s", kdeth_cksum, __le16_to_cpu(p_hdr->iph.chksum), errmsg);
2274 return;
2275 }
2276
2277 tid_recv_sessid = desc_id._desc_idx;
2278 tidrecvc =
2279 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
2280 tid_recv_sessid);
2281
2282 if_pf (tidrecvc == NULL) {
2283 _IPATH_EPDBG("Data Error Pkt and Invalid Recv Handle: %s", errmsg);
2284 return;
2285 }
2286
2287 if_pf (psmi_mpool_get_obj_gen_count(tidrecvc) != desc_id._desc_genc) {
2288 /* Print this at very verbose level. Noisy links can have a few of
2289 * these! */
2290 _IPATH_VDBG("Data Error Pkt and Recv Generation Mismatch: %s", errmsg);
2291 return; /* skip */
2292 }
2293
2294 if (tidrecvc->state == TIDRECVC_STATE_DONE) {
2295 _IPATH_EPDBG("Data Error Pkt for a Completed Rendezvous: %s", errmsg);
2296 return; /* skip */
2297 }
2298
2299 /* See if CRC error for a previous packet */
2300 cur_flowgenseq = ipath_tidflow_get(tidrecvc->context->ctrl,
2301 tidrecvc->tidflow_idx);
2302 tfgen = ipath_tidflow_get_genval(cur_flowgenseq);
2303 tfseq = ipath_tidflow_get_seqnum(cur_flowgenseq);
2304
2305 sequence_num.val = __be32_to_cpu(p_hdr->bth[2]);
2306
2307 if ((sequence_num.gen == tfgen) && (sequence_num.seq < tfseq)) {
2308 /* Try to recover the flow by restarting from previous known good
2309 * sequence (possible if the packet with CRC error is after the "known
2310 * good PSN" else we can't restart the flow.
2311 */
2312 if (tidrecvc->tidflow_genseq.seq < sequence_num.seq)
2313 return ips_protoexp_handle_tf_seqerr(rcv_ev);
2314 else
2315 _IPATH_EPDBG("ErrPkt: CRC Error for packet %d.%d. Currently at %d.%d. %s.\n", sequence_num.gen, sequence_num.seq, tfgen, tfseq, errmsg);
2316 }
2317 else {
2318 /* Print this at very verbose level */
2319 _IPATH_VDBG("Data Error Packet. GenMismatch: %s. Tidrecvc: %p. Pkt Gen.Seq: %d.%d, TF Gen.Seq: %d.%d. %s\n", (sequence_num.gen != tfgen) ? "Yes" : "No", tidrecvc, sequence_num.gen, sequence_num.seq, tfgen, tfseq, errmsg);
2320 }
2321
2322 }
2323 else {
2324 _IPATH_VDBG("HDR_ERROR: %s\n", errmsg);
2325 }
2326
2327 }
2328
2329 psm_error_t
2330 __fastpath
2331 ips_protoexp_flow_newgen(struct ips_tid_recv_desc *tidrecvc)
2332 {
2333 psmi_assert_always(tidrecvc->state != TIDRECVC_STATE_DONE);
2334 ips_tfgen_allocate(&tidrecvc->protoexp->tfctrl,
2335 tidrecvc->tidflow_idx,
2336 &tidrecvc->tidflow_active_gen);
2337
2338 /* Update tidflow table with new generation number */
2339 tidrecvc->tidflow_genseq.gen = tidrecvc->tidflow_active_gen;
2340 ipath_tidflow_set_entry(tidrecvc->context->ctrl,
2341 tidrecvc->tidflow_genseq.flow,
2342 tidrecvc->tidflow_genseq.gen,
2343 tidrecvc->tidflow_genseq.seq);
2344
2345 /* Increment swapped generation count for tidflow */
2346 tidrecvc->tidflow_nswap_gen++;
2347 return PSM_OK;
2348 }
2349
2350 void
2351 __fastpath
2352 ips_protoexp_handle_tf_seqerr(const struct ips_recvhdrq_event *rcv_ev)
2353 {
2354 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
2355 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
2356 struct ips_tid_recv_desc *tidrecvc;
2357 ptl_arg_t desc_id = rcv_ev->p_hdr->hdr_data[0];
2358 ptl_arg_t send_descid = rcv_ev->p_hdr->hdr_data[1];
2359 ptl_arg_t desc_tidrecvc;
2360 psmi_seqnum_t sequence_num;
2361 ptl_arg_t args[3] = {};
2362 psm_error_t err;
2363
2364 psmi_assert_always(protoexp != NULL);
2365
2366 desc_tidrecvc.u64 = 0;
2367 tidrecvc = (struct ips_tid_recv_desc *)
2368 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
2369 desc_id._desc_idx);
2370
2371 if (tidrecvc != NULL)
2372 psmi_mpool_get_obj_index_gen_count(tidrecvc,
2373 &desc_tidrecvc._desc_idx,
2374 &desc_tidrecvc._desc_genc);
2375
2376 if (tidrecvc && desc_tidrecvc.u64 == desc_id.u64) {
2377
2378 /* Update stats for sequence errors */
2379 tidrecvc->stats.nSeqErr++;
2380
2381 if (tidrecvc->state != TIDRECVC_STATE_DONE) {
2382
2383 sequence_num.val = __be32_to_cpu(p_hdr->bth[2]);
2384
2385 /* Only care about sequence error for currently active generation */
2386 if (tidrecvc->tidflow_active_gen == sequence_num.gen) {
2387
2388 /* For a sequence error we restart from where the last header
2389 * was successfully delivered for us since this is the last
2390 * known good state for this flow. The PSM version of the flow
2391 * sequence is the "safe" sequence number to restart at.
2392 */
2393
2394 /* If a "large" number of swapped generation we are loosing packets
2395 * for this flow. Request throttling of tidflow by generating a
2396 * BECN. With header suppression we will miss some FECN packet
2397 * on QLE73XX hence keeping track of swapped generation is another
2398 * mechanism to do congestion control for tidflows.
2399 *
2400 * For mismatched sender/receiver/link speeds we can get into a
2401 * deadly embrace where minimal progress is made due to generation
2402 * mismatch errors. This can occur if we wrap around the generation
2403 * count without making progress. Hence in cases where the swapped
2404 * generation count is > 254 stop sending BECN (and the NAK) so the
2405 * send -> receiver pipeline is flushed with an error check and things
2406 * can sync up. This should be an extremely rare event.
2407 */
2408
2409 if_pf (tidrecvc->tidflow_nswap_gen >= 254)
2410 goto fail; /* Do not send NAK. Let error check kick in. */
2411
2412 if_pf ((tidrecvc->tidflow_nswap_gen > 4) &&
2413 (protoexp->proto->flags & IPS_PROTO_FLAG_CCA)) {
2414 _IPATH_CCADBG("Generating BECN. Number of swapped generations: %d.\n", tidrecvc->tidflow_nswap_gen);
2415 /* Mark flow to generate BECN in control packet */
2416 tidrecvc->ipsaddr->tidgr_flow.flags |= IPS_FLOW_FLAG_GEN_BECN;
2417
2418 /* Update stats for congestion encountered */
2419 if (rcv_ev->ipsaddr)
2420 rcv_ev->ipsaddr->stats.congestion_pkts++;
2421 }
2422
2423 /* Swap generation for the flow. */
2424 err = ips_protoexp_flow_newgen(tidrecvc);
2425 if (err != PSM_OK)
2426 goto fail;
2427
2428 /* NAK the tid flow. Note: We can generate the latest NAK for this flow
2429 * based on the tidrecvc->tidflow_{active|passive}_gen fields. */
2430 args[0] = send_descid;
2431 args[1] = tidrecvc->tid_list.tsess_descid;
2432 args[2].u16w0 = sequence_num.gen; /* Older Gen to NAK */
2433
2434 ips_proto_send_ctrl_message(&tidrecvc->ipsaddr->tidgr_flow,
2435 OPCODE_NAK,
2436 &tidrecvc->ctrl_msg_queued, args);
2437
2438 /* Update stats for retransmit */
2439 tidrecvc->stats.nReXmit++;
2440 }
2441 } /* tidrecvc->state != DONE */
2442 }
2443
2444 fail:
2445 return;
2446 }
2447
2448 void
2449 __fastpath
2450 ips_protoexp_handle_tf_generr(const struct ips_recvhdrq_event *rcv_ev)
2451 {
2452 struct ips_protoexp *protoexp = rcv_ev->proto->protoexp;
2453 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
2454 int tid = IPS_HDR_TID(p_hdr);
2455 struct ips_tid_recv_desc *tidrecvc;
2456 psmi_assert(rcv_ev->p_hdr->data != NULL);
2457 ptl_arg_t desc_id = rcv_ev->p_hdr->data[0];
2458 ptl_arg_t desc_tidrecvc;
2459
2460 if (tid >= IPS_TID_MAX_TIDS || rcv_ev->ptype != RCVHQ_RCV_TYPE_EXPECTED) {
2461 _IPATH_ERROR("Unexpected tid value %d or ptype %d is not expected "
2462 "in tid debugging\n", tid, rcv_ev->ptype);
2463 return;
2464 }
2465
2466 /* For a generation error our NAK crossed on the wire or this is a stale
2467 * packet. Error recovery should sync things up again. Just drop this
2468 * packet.
2469 */
2470 desc_tidrecvc.u64 = 0;
2471 tidrecvc = (struct ips_tid_recv_desc *)
2472 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
2473 desc_id._desc_idx);
2474
2475 if (tidrecvc != NULL) {
2476 psmi_mpool_get_obj_index_gen_count(tidrecvc,
2477 &desc_tidrecvc._desc_idx,
2478 &desc_tidrecvc._desc_genc);
2479 if (desc_tidrecvc.u64 == desc_id.u64) {
2480 tidrecvc->stats.nGenErr++; /* Update stats for generation errors */
2481
2482 /* TODO_CCA: If packet faced congestion we may want to generate a CN
2483 * packet to rate control sender.
2484 */
2485 }
2486
2487 }
2488
2489 }