Debian Codegraph @

ptl_ips/ips_proto_recv.c

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "psm_user.h"
36 #include "ipserror.h"
37 #include "ips_proto.h"
38 #include "ips_proto_internal.h"
39
40 #define PSM_STRAY_WARN_INTERVAL_DEFAULT_SECS 30
41 static void ips_report_strays(struct ips_proto *proto);
42
43 #define INC_TIME_SPEND(timer)
44
45 #define _desc_idx u32w0
46 #define _desc_genc u32w1
47
48 psm_error_t
49 ips_proto_recv_init(struct ips_proto *proto)
50 {
51 uint32_t interval_secs;
52 union psmi_envvar_val env_stray;
53
54 psmi_getenv("PSM_STRAY_WARNINTERVAL",
55 "min secs between stray process warnings",
56 PSMI_ENVVAR_LEVEL_HIDDEN,
57 PSMI_ENVVAR_TYPE_UINT,
58 (union psmi_envvar_val) PSM_STRAY_WARN_INTERVAL_DEFAULT_SECS,
59 &env_stray);
60 interval_secs = env_stray.e_uint;
61 if (interval_secs > 0)
62 proto->stray_warn_interval = sec_2_cycles(interval_secs);
63 else
64 proto->stray_warn_interval = 0;
65
66 return PSM_OK;
67 }
68
69 psm_error_t
70 ips_proto_recv_fini(struct ips_proto *proto)
71 {
72 ips_report_strays(proto);
73 return PSM_OK;
74 }
75
76 #define cycles_to_sec_f(cycles) \
77 (((double)cycles_to_nanosecs(cycles)) / 1000000000.0)
78
79 struct ips_stray_epid {
80 psm_epid_t epid;
81 uint32_t err_check_bad_sent;
82 uint32_t ipv4_addr;
83 uint32_t pid;
84 uint32_t num_messages;
85 uint64_t t_warn_next;
86 uint64_t t_first;
87 uint64_t t_last;
88 };
89
90 static
91 void
92 ips_report_strays(struct ips_proto *proto)
93 {
94 struct ips_stray_epid *sepid;
95 struct psmi_eptab_iterator itor;
96 psmi_epid_itor_init(&itor, PSMI_EP_CROSSTALK);
97 double t_runtime = cycles_to_sec_f(proto->t_fini - proto->t_init);
98
99 while ((sepid = psmi_epid_itor_next(&itor))) {
100 char ipbuf[INET_ADDRSTRLEN], *ip = NULL;
101 char bufpid[32];
102 uint32_t lid = psm_epid_nid(sepid->epid);
103 double t_first = cycles_to_sec_f(sepid->t_first - proto->t_init);
104 double t_last = cycles_to_sec_f(sepid->t_last - proto->t_init);
105 if (sepid->ipv4_addr)
106 ip = (char *)
107 inet_ntop(AF_INET, &sepid->ipv4_addr, ipbuf, sizeof ipbuf);
108 if (!ip)
109 snprintf(ipbuf, sizeof ipbuf, "%d (%x)", lid, lid);
110
111 if (sepid->pid)
112 snprintf(bufpid, sizeof bufpid, "PID=%d", sepid->pid);
113 else
114 snprintf(bufpid, sizeof bufpid, "PID unknown");
115
116 _IPATH_INFO("Process %s on host %s=%s sent %d stray message(s) and "
117 "was told so %d time(s) (first stray message at %.1fs "
118 "(%d%%), last at %.1fs (%d%%) into application run)\n",
119 bufpid, ip ? "IP" : "LID", ipbuf, sepid->num_messages,
120 sepid->err_check_bad_sent, t_first,
121 (int) (t_first * 100.0 / t_runtime), t_last,
122 (int) (t_last * 100.0 / t_runtime));
123
124 psmi_epid_remove(PSMI_EP_CROSSTALK, sepid->epid);
125 psmi_free(sepid);
126 }
127 psmi_epid_itor_fini(&itor);
128 return;
129 }
130
131 /* New scbs now available. If we have pending sends because we were out of
132 * scbs, put the pendq on the timerq so it can be processed. */
133 void
134 ips_proto_rv_scbavail_callback(struct ips_scbctrl *scbc, void *context)
135 {
136 struct ips_proto *proto = (struct ips_proto *) context;
137 struct ips_pend_sreq *sreq = STAILQ_FIRST(&proto->pend_sends.pendq);
138 if (sreq != NULL)
139 psmi_timer_request(proto->timerq,
140 &proto->pend_sends.timer, PSMI_TIMER_PRIO_1);
141 return;
142 }
143
144 psm_error_t __recvpath
145 ips_proto_timer_pendq_callback(struct psmi_timer *timer, uint64_t current)
146 {
147 psm_error_t err = PSM_OK;
148 struct ips_pend_sends *pend_sends =
149 (struct ips_pend_sends *) timer->context;
150 struct ips_pendsendq *phead = &pend_sends->pendq;
151 struct ips_proto *proto = (struct ips_proto *) pend_sends->proto;
152 struct ips_pend_sreq *sreq;
153
154 while (!STAILQ_EMPTY(phead)) {
155 sreq = STAILQ_FIRST(phead);
156 switch (sreq->type) {
157 case IPS_PENDSEND_EAGER_REQ:
158 err = ips_proto_mq_push_eager_req(proto, sreq->req);
159 break;
160 case IPS_PENDSEND_EAGER_DATA:
161 err = ips_proto_mq_push_eager_data(proto, sreq->req);
162 break;
163
164 default:
165 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
166 "Unknown pendq state %d\n", sreq->type);
167 }
168
169 if (err == PSM_OK) {
170 STAILQ_REMOVE_HEAD(phead, next);
171 psmi_mpool_put(sreq);
172 }
173 else { /* out of scbs. wait for the next scb_avail callback */
174 //printf("!!!!! breaking out of pendq progress\n");
175 break;
176 }
177 }
178
179 return err;
180 }
181
182 static
183 int __recvpath
184 _process_mq(struct ips_recvhdrq_event *rcv_ev)
185 {
186 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
187 char *payload = ips_recvhdrq_event_payload(rcv_ev);
188 uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev);
189 uint32_t msglen = paylen;
190 uint16_t mode = p_hdr->mqhdr;
191 psm_mq_req_t req;
192 psmi_egrid_t egrid;
193 ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr;
194 psm_epaddr_t epaddr = ipsaddr->epaddr;
195 psm_mq_t mq = rcv_ev->proto->mq;
196 ptl_arg_t *args;
197 ptl_epaddr_flow_t flowid = ips_proto_flowid(p_hdr);
198 struct ips_flow *flow = &ipsaddr->flows[flowid];
199 int ret = IPS_RECVHDRQ_CONTINUE;
200
201 if (!ips_proto_is_expected_or_nak((struct ips_recvhdrq_event*) rcv_ev))
202 goto skip_ack_req;
203
204 _IPATH_VDBG("Rcvd ctrl packet %s length = %i, mode=%d, arg0=%llx arg1=%llx\n",
205 psmi_epaddr_get_name(epaddr->epid),
206 paylen, p_hdr->mqhdr,
207 (long long) p_hdr->data[0].u64, (long long) p_hdr->data[1].u64);
208
209 if (mode <= MQ_MSG_RTS_WAIT) {
210 ret = ips_proto_check_msg_order(epaddr, flow, p_hdr);
211 if (ret == 0) return IPS_RECVHDRQ_OOO;
212
213 if (mode <= MQ_MSG_LONG) {
214 egrid.egr_data = 0;
215 if (mode == MQ_MSG_SHORT) {
216 /* May have padded writes, account for it */
217 paylen -= p_hdr->hdr_dlen;
218 msglen = paylen;
219 }
220 else if (mode == MQ_MSG_TINY) {
221 payload = (void *) &p_hdr->data[1];
222 msglen = paylen = p_hdr->hdr_dlen;
223 }
224 else if (mode == MQ_MSG_LONG) {
225 msglen = p_hdr->data[1].u32w1;
226 if (ipsaddr->flags & SESS_FLAG_HAS_FLOWID) {
227 egrid.egr_data = p_hdr->data[1].u32w0;
228 _IPATH_VDBG("egrid-msglong is 0x%x\n", egrid.egr_data);
229 }
230 }
231
232 if (ret == 1)
233 psmi_mq_handle_envelope(
234 mq, mode, epaddr, p_hdr->data[0].u64, /* tag */
235 egrid, msglen, (void *) payload, paylen);
236 else
237 psmi_mq_handle_envelope_outoforder(
238 mq, mode, epaddr, flow->msg_ooo_seqnum,
239 p_hdr->data[0].u64, /* tag */
240 egrid, msglen, (void *) payload, paylen);
241 } else {
242 args = (ptl_arg_t *) p_hdr->data;
243 if (ret == 1)
244 ips_proto_mq_handle_rts_envelope(mq, mode, epaddr,
245 args[0].u64, args[1].u32w0, args[1].u32w1);
246 else
247 ips_proto_mq_handle_rts_envelope_outoforder(mq, mode,
248 epaddr, flow->msg_ooo_seqnum,
249 args[0].u64, args[1].u32w0, args[1].u32w1);
250 }
251
252 if (ret == 1) {
253 if (epaddr->mctxt_master->outoforder_c) {
254 psmi_mq_handle_outoforder_queue(epaddr->mctxt_master);
255 }
256 ret = IPS_RECVHDRQ_CONTINUE;
257 } else {
258 ret = IPS_RECVHDRQ_BREAK;
259 }
260 } else if (mode == MQ_MSG_DATA || mode == MQ_MSG_DATA_BLK) {
261 psm_mq_req_t req;
262
263 req = STAILQ_FIRST(&epaddr->mctxt_master->egrlong);
264 while (req) {
265 if (req->egrid.egr_data == p_hdr->data[1].u32w0) break;
266 req = STAILQ_NEXT(req, nextq);
267 }
268
269 /*
270 * Even with single context, since the header is sent via pio-flow,
271 * and data is sent via sdma-flow, data could be received first,
272 * thus causes req=NULL.
273 */
274 if (req == NULL) {
275 flow->msg_ooo_toggle = !flow->msg_ooo_toggle;
276 if (flow->msg_ooo_toggle) {
277 flow->recv_seq_num.pkt -= 1;
278 return IPS_RECVHDRQ_OOO;
279 }
280 } else {
281 flow->msg_ooo_toggle = 0;
282 }
283
284 psmi_mq_handle_data(req, epaddr, p_hdr->data[1].u32w0,
285 p_hdr->data[1].u32w1, payload, paylen);
286
287 /* If checksum is enabled, this matches what is done for tid-sdma */
288 /* if OOO and req is NULL, header is not received and we ignore chksum */
289 if (rcv_ev->proto->flags & IPS_PROTO_FLAG_CKSUM &&
290 mode == MQ_MSG_DATA_BLK &&
291 req && req->state == MQ_STATE_COMPLETE) {
292 uint32_t cksum = ips_crc_calculate(
293 req->recv_msglen - p_hdr->data[0].u32w1,
294 (uint8_t *)req->buf + p_hdr->data[0].u32w1,
295 0xffffffff);
296 if (p_hdr->data[0].u32w0 != cksum) {
297 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
298 "ErrPkt: Checksum mismatch. Expected: 0x%08x, Received: 0x%08x Source LID: %i. Aborting! \n", p_hdr->data[0].u32w0, cksum, __be16_to_cpu(flow->path->epr_dlid));
299 ips_proto_dump_data(req->buf, req->recv_msglen);
300 }
301 }
302
303 } else if (mode == MQ_MSG_DATA_REQ || mode == MQ_MSG_DATA_REQ_BLK) {
304 req = psmi_mpool_find_obj_by_index(mq->rreq_pool,
305 p_hdr->data[1].u32w0);
306 if (!req) goto skip_ack_req;
307 psmi_mq_handle_data(req, epaddr, p_hdr->data[1].u32w0,
308 p_hdr->data[1].u32w1, (void *) payload, paylen);
309
310 /* If checksum is enabled, this matches what is done for tid-sdma */
311 if (rcv_ev->proto->flags & IPS_PROTO_FLAG_CKSUM &&
312 mode == MQ_MSG_DATA_REQ_BLK &&
313 req->state == MQ_STATE_COMPLETE) {
314 uint32_t cksum = ips_crc_calculate(
315 req->recv_msglen, req->buf, 0xffffffff);
316 if (p_hdr->data[0].u32w0 != cksum) {
317 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
318 "ErrPkt: Checksum mismatch. Expected: 0x%08x, Received: 0x%08x Source LID: %i. Aborting! \n", p_hdr->data[0].u32w0, cksum, __be16_to_cpu(flow->path->epr_dlid));
319 ips_proto_dump_data(req->buf, req->recv_msglen);
320 }
321 }
322
323 } else if (mode == MQ_MSG_CTS_EGR) {
324 args = p_hdr->data;
325 ips_proto_mq_handle_cts(rcv_ev->proto, args);
326 } else {
327 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
328 "Unknown frame mode %x", mode);
329 }
330
331 if ((p_hdr->flags & IPS_SEND_FLAG_ACK_REQ) ||
332 (flow->flags & IPS_FLOW_FLAG_GEN_BECN))
333 ips_proto_send_ack((struct ips_recvhdrq *) rcv_ev->recvq, flow);
334
335 skip_ack_req:
336 ips_proto_process_ack(rcv_ev);
337
338 return ret; // skip
339 }
340
341 PSMI_INLINE(
342 int between(int first_seq, int last_seq, int seq))
343 {
344 if (last_seq >= first_seq) {
345 if (seq < first_seq || seq > last_seq) {
346 return 0;
347 }
348 } else {
349 if (seq > last_seq && seq < first_seq) {
350 return 0;
351 }
352 }
353 return 1;
354 }
355
356 PSMI_INLINE(
357 int pio_dma_ack_valid(struct ips_flow *flow, psmi_seqnum_t ack_seq_num,
358 uint32_t ack_window))
359 {
360 uint32_t first_pkt, last_pkt;
361 struct ips_scb_unackedq *unackedq = &flow->scb_unacked;
362
363 if (STAILQ_EMPTY(unackedq))
364 return 0;
365
366 first_pkt = flow->xmit_ack_num.pkt + 1;
367 last_pkt = STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num.pkt;
368 return between(first_pkt, last_pkt, ack_seq_num.pkt);
369 }
370
371 PSMI_INLINE(
372 struct ips_flow* get_tidflow(ips_epaddr_t *ipsaddr,
373 struct ips_message_header *p_hdr,
374 psmi_seqnum_t ack_seq_num,
375 uint32_t ack_window))
376 {
377 struct ips_flow *flow;
378 struct ips_protoexp *protoexp = ipsaddr->proto->protoexp;
379 struct ips_tid_send_desc *tidsendc;
380 ptl_arg_t desc_id = p_hdr->data[0];
381 ptl_arg_t desc_tidsendc;
382 uint32_t first_seq, last_seq;
383 struct ips_scb_unackedq *unackedq;
384
385 tidsendc = (struct ips_tid_send_desc*)
386 psmi_mpool_find_obj_by_index(protoexp->tid_desc_send_pool,
387 desc_id._desc_idx);
388 if (tidsendc == NULL) {
389 _IPATH_ERROR("OPCODE_ACK: Index %d is out of range in tidflow ack\n", desc_id._desc_idx);
390 return NULL;
391 }
392
393 /* Ensure generation matches */
394 psmi_mpool_get_obj_index_gen_count(tidsendc,
395 &desc_tidsendc._desc_idx,
396 &desc_tidsendc._desc_genc);
397 if (desc_tidsendc.u64 != desc_id.u64)
398 return NULL;
399
400 /* Ensure ack is within window */
401 flow = &tidsendc->tidflow;
402 unackedq = &flow->scb_unacked;
403
404 /* No unacked scbs */
405 if (STAILQ_EMPTY(unackedq))
406 return NULL;
407
408 first_seq = flow->xmit_ack_num.seq + 1;
409 last_seq = STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num.seq;
410 if (between(first_seq, last_seq, ack_seq_num.seq) == 0) {
411 return NULL;
412 }
413
414 /* Generation for ack should match */
415 if (STAILQ_FIRST(unackedq)->seq_num.gen != ack_seq_num.gen)
416 return NULL;
417
418 return flow;
419 }
420
421 /* NAK post process for tid flow */
422 void ips_tidflow_nak_post_process(struct ips_flow *flow,
423 struct ips_message_header *p_hdr)
424 {
425
426 ips_scb_t *scb;
427 struct ips_scb_unackedq *unackedq = &flow->scb_unacked;
428 #ifdef PSM_DEBUG
429 psmi_seqnum_t new_flowgenseq;
430
431 new_flowgenseq.val = p_hdr->data[1].u32w0;
432 /* Update any pending scb's to the new generation count.
433 * Note: flow->xmit_seq_num was updated to the new generation when the
434 * NAK was received.
435 */
436 psmi_assert(STAILQ_FIRST(unackedq)->seq_num.flow==new_flowgenseq.flow);
437 psmi_assert(STAILQ_FIRST(unackedq)->seq_num.gen != new_flowgenseq.gen);
438 psmi_assert(STAILQ_FIRST(unackedq)->
439 seq_num.seq-STAILQ_FIRST(unackedq)->nfrag+1 == new_flowgenseq.seq);
440 #endif
441
442 /* Update unacked scb's to use the new flowgenseq */
443 scb = STAILQ_FIRST(unackedq);
444 while (scb) {
445 scb->ips_lrh.bth[2] = __cpu_to_be32(flow->xmit_seq_num.psn);
446 flow->xmit_seq_num.seq += scb->nfrag;
447 scb->seq_num = flow->xmit_seq_num;
448 scb->seq_num.seq--;
449 scb = SLIST_NEXT(scb, next);
450 }
451
452 }
453
454 // process an incoming ack message. Separate function to allow
455 // for better optimization by compiler
456 void __recvpath
457 ips_proto_process_ack(struct ips_recvhdrq_event *rcv_ev)
458 {
459 ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr;
460 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
461 psmi_seqnum_t ack_seq_num, last_seq_num;
462 ips_scb_t *scb;
463 struct ips_proto *proto = ipsaddr->proto;
464 struct ips_flow *flow = NULL;
465 struct ips_scb_unackedq *unackedq;
466 struct ips_scb_pendlist *scb_pend;
467 psm_protocol_type_t protocol;
468 ptl_epaddr_flow_t flowid;
469
470 ips_ptladdr_lock(ipsaddr);
471
472 protocol = IPS_FLOWID_GET_PROTO(p_hdr->flowid);
473 flowid = IPS_FLOWID_GET_INDEX(p_hdr->flowid);
474 ack_seq_num.psn = p_hdr->ack_seq_num;
475
476 switch(protocol){
477 case PSM_PROTOCOL_GO_BACK_N:
478 flow = &ipsaddr->flows[flowid];
479 ack_seq_num.pkt -= 1;
480 if (!pio_dma_ack_valid(flow, ack_seq_num, proto->scb_max_inflight))
481 goto ret;
482 flow->xmit_ack_num = ack_seq_num;
483 break;
484 case PSM_PROTOCOL_TIDFLOW:
485 ack_seq_num.seq -= 1;
486 flow = get_tidflow(ipsaddr, p_hdr, ack_seq_num, proto->scb_max_inflight);
487 if (!flow) /* Invalid ack for flow */
488 goto ret;
489 flow->xmit_ack_num = ack_seq_num;
490 break;
491 default:
492 _IPATH_ERROR("OPCODE_ACK: Unknown flow type %d in ACK\n", flowid);
493 goto ret;
494 }
495
496 unackedq = &flow->scb_unacked;
497 scb_pend = &flow->scb_pend;
498 if (STAILQ_EMPTY(unackedq)) goto ret; // only for Klockwork scan.
499 last_seq_num = STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num;
500
501 INC_TIME_SPEND(TIME_SPEND_USER2);
502
503 /* For tidflow, we want to match all flow/gen/seq,
504 for gobackn, we only match pkt#, msg# is not known.
505 msg# is the message envelope number in the stream,
506 you don't know if the next packet has the old msg#
507 or starts a new msg#.
508 */
509 /* first release all xmit buffer that has been receveid */
510 while ((protocol==PSM_PROTOCOL_GO_BACK_N) ?
511 between(STAILQ_FIRST(unackedq)->seq_num.pkt,
512 last_seq_num.pkt, ack_seq_num.pkt) :
513 between(STAILQ_FIRST(unackedq)->seq_num.psn,
514 last_seq_num.psn, ack_seq_num.psn)
515 ) {
516
517 /* take it out of the xmit queue and .. */
518 scb = STAILQ_FIRST(unackedq);
519 STAILQ_REMOVE_HEAD(unackedq, nextq);
520 flow->scb_num_unacked--;
521 flow->credits++;
522
523 if (scb == SLIST_FIRST(scb_pend)) {
524 flow->scb_num_pending--;
525 SLIST_REMOVE_HEAD(scb_pend, next);
526 }
527
528 if (scb->flags & IPS_SEND_FLAG_WAIT_SDMA)
529 ips_proto_dma_wait_until(proto, scb->dma_ctr);
530
531 if (scb->callback)
532 (*scb->callback) (scb->cb_param, scb->payload_size-scb->extra_bytes);
533
534 if (!(scb->flags & IPS_SEND_FLAG_PERSISTENT))
535 ips_scbctrl_free(scb);
536
537 /* set all index pointer to NULL if all frames have been
538 * acked */
539 if (STAILQ_EMPTY(unackedq)) {
540 psmi_timer_cancel(proto->timerq, &flow->timer_ack);
541 psmi_timer_cancel(proto->timerq, &flow->timer_send);
542 SLIST_FIRST(scb_pend) = NULL;
543 psmi_assert(flow->scb_num_pending == 0);
544 /* Reset congestion window - all packets ACK'd */
545 flow->credits = flow->cwin = proto->flow_credits;
546 flow->ack_interval = max((flow->credits >> 2) - 1, 1);
547 flow->flags &= ~IPS_FLOW_FLAG_CONGESTED;
548 goto ret;
549 }
550 }
551
552 /* CCA: If flow is congested adjust rate */
553 if_pf (rcv_ev->is_congested & IPS_RECV_EVENT_BECN) {
554 if ((flow->path->epr_ccti +
555 proto->cace[flow->path->epr_sl].ccti_increase) <=
556 proto->ccti_limit) {
557 ips_cca_adjust_rate(flow->path,
558 proto->cace[flow->path->epr_sl].ccti_increase);
559 /* Clear congestion event */
560 rcv_ev->is_congested &= ~IPS_RECV_EVENT_BECN;
561 }
562 }
563 else {
564 /* Increase congestion window if flow is not congested */
565 if_pf (flow->cwin < proto->flow_credits) {
566 flow->credits +=
567 min(flow->cwin << 1, proto->flow_credits) - flow->cwin;
568 flow->cwin = min(flow->cwin << 1, proto->flow_credits);
569 flow->ack_interval = max((flow->credits >> 2) - 1, 1);
570 }
571 }
572
573 /* Reclaimed some credits - attempt to flush flow */
574 flow->fn.xfer.flush(flow, NULL);
575
576 /*
577 * If the next packet has not even been put on the wire, cancel the
578 * retransmission timer since we're still presumably waiting on free
579 * pio bufs
580 */
581 if (STAILQ_FIRST(unackedq)->abs_timeout == TIMEOUT_INFINITE)
582 psmi_timer_cancel(proto->timerq, &flow->timer_ack);
583
584 ret:
585 ips_ptladdr_unlock(ipsaddr);
586 return;
587 }
588
589 // process an incoming nack message. Separate function to allow
590 // for better optimization by compiler
591 static void
592 _process_nak(struct ips_recvhdrq_event *rcv_ev)
593 {
594 ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr;
595 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
596 psmi_seqnum_t ack_seq_num, last_seq_num;
597 ips_scb_t *scb;
598 struct ips_proto *proto = ipsaddr->proto;
599 struct ips_flow *flow = NULL;
600 struct ips_scb_unackedq *unackedq;
601 struct ips_scb_pendlist *scb_pend;
602 psm_protocol_type_t protocol;
603 ptl_epaddr_flow_t flowid;
604 int num_resent = 0;
605
606 ips_ptladdr_lock(ipsaddr);
607
608 protocol = IPS_FLOWID_GET_PROTO(p_hdr->flowid);
609 flowid = IPS_FLOWID_GET_INDEX(p_hdr->flowid);
610
611 INC_TIME_SPEND(TIME_SPEND_USER3);
612
613 ack_seq_num.psn = p_hdr->ack_seq_num;
614
615 switch(protocol){
616 case PSM_PROTOCOL_GO_BACK_N:
617 flow = &ipsaddr->flows[flowid];
618 if (!pio_dma_ack_valid(flow, ack_seq_num, proto->scb_max_inflight))
619 goto ret;
620 ack_seq_num.pkt--;
621 flow->xmit_ack_num = ack_seq_num;
622 break;
623 case PSM_PROTOCOL_TIDFLOW:
624 flow = get_tidflow(ipsaddr, p_hdr, ack_seq_num, proto->scb_max_inflight);
625 if (!flow)
626 goto ret; /* Invalid ack for flow */
627 ack_seq_num.seq--;
628 /* Update xmit seq num to the new flowgenseq */
629 flow->xmit_seq_num = (psmi_seqnum_t)p_hdr->data[1].u32w0;
630 flow->xmit_ack_num = flow->xmit_seq_num;
631 flow->xmit_ack_num.seq--;
632 break;
633 default:
634 _IPATH_ERROR("OPCODE_NAK: Unknown flow type %d in ACK\n", flowid);
635 goto ret;
636 }
637
638 unackedq = &flow->scb_unacked;
639 scb_pend = &flow->scb_pend;
640 if (STAILQ_EMPTY(unackedq)) goto ret; // only for Klockwork scan.
641 last_seq_num = STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num;
642
643 ipsaddr->stats.nak_recv++;
644
645 _IPATH_VDBG("got a nack %d on flow %d, "
646 "first is %d, last is %d\n", ack_seq_num.psn,
647 flowid, STAILQ_EMPTY(unackedq)?-1:STAILQ_FIRST(unackedq)->seq_num.psn,
648 STAILQ_EMPTY(unackedq)?-1:STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num.psn);
649
650 /* For tidflow, we want to match all flow/gen/seq,
651 for gobackn, we only match pkt#, msg# is not known.
652 msg# is the message envelope number in the stream,
653 you don't know if the next packet has the old msg#
654 or starts a new msg#.
655 */
656 /* first release all xmit buffer that has been receveid */
657 while ((protocol==PSM_PROTOCOL_GO_BACK_N) ?
658 between(STAILQ_FIRST(unackedq)->seq_num.pkt,
659 last_seq_num.pkt, ack_seq_num.pkt) :
660 between(STAILQ_FIRST(unackedq)->seq_num.psn,
661 last_seq_num.psn, ack_seq_num.psn)
662 ) {
663 /* take it out of the xmit queue and .. */
664 scb = STAILQ_FIRST(unackedq);
665 STAILQ_REMOVE_HEAD(unackedq, nextq);
666 flow->scb_num_unacked--;
667
668 if (scb->flags & IPS_SEND_FLAG_WAIT_SDMA)
669 ips_proto_dma_wait_until(proto, scb->dma_ctr);
670
671 if (scb->callback)
672 (*scb->callback) (scb->cb_param, scb->payload_size-scb->extra_bytes);
673
674 if (!(scb->flags & IPS_SEND_FLAG_PERSISTENT))
675 ips_scbctrl_free(scb);
676
677 /* set all index pointer to NULL if all frames has been acked */
678 if (STAILQ_EMPTY(unackedq)) {
679 psmi_timer_cancel(proto->timerq, &flow->timer_ack);
680 psmi_timer_cancel(proto->timerq, &flow->timer_send);
681 SLIST_FIRST(scb_pend) = NULL;
682 psmi_assert(flow->scb_num_pending == 0);
683 /* Reset congestion window if all packets acknowledged */
684 flow->credits = flow->cwin = proto->flow_credits;
685 flow->ack_interval = max((flow->credits >> 2) - 1, 1);
686 flow->flags &= ~IPS_FLOW_FLAG_CONGESTED;
687 goto ret;
688 }
689 }
690
691 psmi_assert(!STAILQ_EMPTY(unackedq)); /* sanity for above loop */
692
693 if (flow->fn.protocol.nak_post_process)
694 flow->fn.protocol.nak_post_process(flow, p_hdr);
695
696 /* Always cancel ACK timer as we are going to restart the flow */
697 psmi_timer_cancel(proto->timerq, &flow->timer_ack);
698
699 /* What's now pending is all that was unacked */
700 SLIST_FIRST(scb_pend) = STAILQ_FIRST(unackedq);
701 flow->scb_num_pending = flow->scb_num_unacked;
702
703 /* If NAK with congestion bit set - delay re-transmitting and THEN adjust
704 * CCA rate.
705 */
706 if_pf (rcv_ev->is_congested & IPS_RECV_EVENT_BECN) {
707 uint64_t offset;
708
709 /* Clear congestion event and mark flow as congested */
710 rcv_ev->is_congested &= ~IPS_RECV_EVENT_BECN;
711 flow->flags |= IPS_FLOW_FLAG_CONGESTED;
712
713 /* For congested flow use slow start i.e. reduce congestion window.
714 * For TIDFLOW we cannot reduce congestion window as peer expects
715 * header packets at regular intervals (protoexp->hdr_pkt_interval).
716 */
717 if (flow->protocol != PSM_PROTOCOL_TIDFLOW)
718 flow->credits = flow->cwin = 1;
719 else
720 flow->credits = flow->cwin;
721
722 flow->ack_interval = max((flow->credits >> 2) - 1, 1);
723
724 /* During congestion cancel send timer and delay retransmission by
725 * random interval
726 */
727 psmi_timer_cancel(proto->timerq, &flow->timer_send);
728 if (SLIST_FIRST(scb_pend)->ack_timeout != TIMEOUT_INFINITE)
729 offset = (SLIST_FIRST(scb_pend)->ack_timeout >> 1);
730 else
731 offset = 0;
732 psmi_timer_request(proto->timerq, &flow->timer_send,
733 (get_cycles() +
734 (uint64_t)(offset * (rand()/RAND_MAX + 1.0))));
735 }
736 else {
737 /* Reclaim all credits upto congestion window only */
738 flow->credits = flow->cwin;
739 flow->ack_interval = max((flow->credits >> 2) - 1, 1);
740
741 /* Flush pending scb's */
742 flow->fn.xfer.flush(flow, &num_resent);
743 ipsaddr->stats.send_rexmit += num_resent;
744 }
745
746 ret:
747 ips_ptladdr_unlock(ipsaddr);
748 return;
749 }
750
751 static void
752 _process_err_chk(struct ips_recvhdrq *recvq, ips_epaddr_t *ipsaddr,
753 struct ips_message_header *p_hdr)
754 {
755 psmi_seqnum_t seq_num;
756 int16_t seq_off;
757 ptl_epaddr_flow_t flowid = ips_proto_flowid(p_hdr);
758 struct ips_flow *flow = &ipsaddr->flows[flowid];
759
760 INC_TIME_SPEND(TIME_SPEND_USER4);
761
762 ipsaddr->stats.err_chk_recv++;
763
764 seq_num.val = __be32_to_cpu(p_hdr->bth[2]);
765 seq_off = (int16_t)(ipsaddr->flows[flowid].recv_seq_num.pkt - seq_num.pkt);
766
767 if_pf (seq_off <= 0) {
768 _IPATH_VDBG("naking for seq=%d, off=%d on flowid %d\n",
769 seq_num.pkt, seq_off, flowid);
770
771 if (seq_off < -flow->ack_interval)
772 flow->flags |= IPS_FLOW_FLAG_GEN_BECN;
773
774 ips_proto_send_nak(recvq, flow);
775 flow->flags |= IPS_FLOW_FLAG_NAK_SEND;
776 }
777 else {
778 ips_proto_send_ctrl_message(flow, OPCODE_ACK,
779 &ipsaddr->ctrl_msg_queued, NULL);
780 }
781 }
782
783 static void
784 _process_err_chk_gen(ips_epaddr_t *ipsaddr, struct ips_message_header *p_hdr)
785 {
786 struct ips_protoexp *protoexp = ipsaddr->proto->protoexp;
787 struct ips_tid_recv_desc *tidrecvc;
788 psmi_seqnum_t err_seqnum;
789 ptl_arg_t desc_id = p_hdr->data[0];
790 ptl_arg_t send_desc_id = p_hdr->data[1];
791 ptl_arg_t desc_tidrecvc;
792 ptl_arg_t args[3] = {};
793 int16_t seq_off;
794 uint8_t ack_type;
795
796 INC_TIME_SPEND(TIME_SPEND_USER4);
797
798 ipsaddr->stats.err_chk_recv++;
799
800 /* Get the flowgenseq for err chk gen */
801 err_seqnum.val = __be32_to_cpu(p_hdr->bth[2]);
802
803 ips_ptladdr_lock(ipsaddr);
804
805 /* Get receive descriptor */
806 tidrecvc = (struct ips_tid_recv_desc *)
807 psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool,
808 desc_id._desc_idx);
809
810 if (tidrecvc == NULL) {
811 _IPATH_DBG("ERR_CHK_GEN: invalid rendezvous handle\n");
812 ips_ptladdr_unlock(ipsaddr);
813 return;
814 }
815 psmi_mpool_get_obj_index_gen_count(tidrecvc,
816 &desc_tidrecvc._desc_idx,
817 &desc_tidrecvc._desc_genc);
818
819 if (desc_id.u64 != desc_tidrecvc.u64) {
820 /* Receive descriptor mismatch in time and space.
821 * Stale err chk gen, drop packet
822 */
823 _IPATH_DBG("ERR_CHK_GEN: rendezvous handle generation mismatch. Pkt: 0x%08x, Current: 0x%08x\n", desc_id._desc_genc, desc_tidrecvc._desc_genc);
824 ips_ptladdr_unlock(ipsaddr);
825 return;
826 }
827
828 psmi_assert(tidrecvc->tidflow_idx == err_seqnum.flow);
829
830 /* Note: Do not read the tidflow table to determine the sequence to restart
831 * from. Always respond with the last known "good" packet that we received
832 * which is updated in protoexp_data().
833 */
834
835 /* Either lost packets or lost ack */
836 seq_off = (int16_t) (tidrecvc->tidflow_genseq.seq - err_seqnum.seq);
837
838 if (seq_off <= 0) {
839 ack_type = OPCODE_NAK;
840
841 if (err_seqnum.gen == tidrecvc->tidflow_active_gen) {
842 /* Swap generations */
843 psm_error_t err;
844
845 /* Allocate new generation for the flow. */
846 err = ips_protoexp_flow_newgen(tidrecvc);
847 if (err != PSM_OK)
848 return; /* Out of generation. Drop packet and we will recover later */
849 }
850 }
851 else {
852 ack_type = OPCODE_ACK;
853
854 if (err_seqnum.gen != tidrecvc->tidflow_genseq.gen)
855 ack_type = OPCODE_NAK; /* NAK without allocating a new generation */
856 }
857
858 args[0] = send_desc_id;
859 args[1] = tidrecvc->tid_list.tsess_descid;
860 args[2].u16w0 = err_seqnum.gen; /* If NAK, generation number */
861
862 ips_ptladdr_unlock(ipsaddr);
863
864 /* May want to generate a BECN if a lot of swapped generations */
865 if_pf ((tidrecvc->tidflow_nswap_gen > 4) &&
866 (protoexp->proto->flags & IPS_PROTO_FLAG_CCA)) {
867 _IPATH_CCADBG("ERR_CHK_GEN: Generating BECN. Number of swapped generations: %d.\n", tidrecvc->tidflow_nswap_gen);
868 /* Mark flow to generate BECN in control packet */
869 tidrecvc->ipsaddr->tidgr_flow.flags |= IPS_FLOW_FLAG_GEN_BECN;
870
871 /* Update stats for congestion encountered */
872 ipsaddr->stats.congestion_pkts++;
873 }
874
875 ips_proto_send_ctrl_message(&tidrecvc->ipsaddr->tidgr_flow,
876 ack_type, &tidrecvc->ctrl_msg_queued, args);
877
878 /* Update stats for expected window */
879 tidrecvc->stats.nErrChkReceived++;
880 if (ack_type == OPCODE_NAK)
881 tidrecvc->stats.nReXmit++; /* Update stats for retransmit (Sent a NAK) */
882 }
883
884 static void
885 parse_ip_or_lid(char *buf, size_t len, uint32_t ip, psm_epid_t epid)
886 {
887 char ipbuf[INET_ADDRSTRLEN], *p;
888 in_addr_t in_loop = inet_addr("127.0.0.1");
889 in_addr_t in_any = inet_addr("0.0.0.0");
890
891 p = (char *) inet_ntop(AF_INET, (const void *) &ip, ipbuf, sizeof ipbuf);
892 if (ip != in_loop && ip != in_any && p)
893 snprintf(buf, len-1, "IP %s", p);
894 else
895 snprintf(buf, len-1, "LID 0x%x", (int) psm_epid_nid(epid));
896 buf[len-1] = '\0';
897 }
898
899 #define IPS_MAX_BOGUS_ERR_CHK_BAD 15
900
901 static void
902 _process_err_chk_bad(ips_epaddr_t *ipsaddr, struct ips_message_header *p_hdr)
903 {
904 uint32_t ipv4_addr = p_hdr->data[0].u32w0;
905 uint32_t pid = __be32_to_cpu(p_hdr->data[0].u32w1);
906 union psmi_envvar_val env_stray;
907 char buf[32];
908 psm_epid_t epid = ipsaddr->epaddr->epid;
909
910 parse_ip_or_lid(buf, sizeof buf, ipv4_addr, epid);
911
912 /* First make sure that we actually do have a connection to this lid+context,
913 * if not, we just ignore the err_chk_bad message since it might be some
914 * oddly timed packet */
915 if (!ips_proto_isconnected(ipsaddr)) {
916 int lid = (int) psm_epid_nid(epid);
917 int context = (int) psm_epid_context(epid);
918 if (++ipsaddr->proto->num_bogus_warnings <= IPS_MAX_BOGUS_ERR_CHK_BAD)
919 psmi_syslog(ipsaddr->proto->ep, 1, LOG_INFO,
920 "PSM pid %d on host %s complains that I am a stray process but "
921 "I'm not even connected to LID %d context %d (ignoring %s\n",
922 pid, buf, lid, context,
923 ipsaddr->proto->num_bogus_warnings == IPS_MAX_BOGUS_ERR_CHK_BAD ?
924 "all future stray warning checks from unknown endpoints)." :
925 ").");
926 return;
927 }
928
929 /* At this point the bad error check is a real one, from a host we thought
930 * we were connected to. We only go through this path once. If
931 * PSM_STRAY_ENABLED=0, we'll print this warning once, if it's 1 we'll die.
932 */
933 if (ipsaddr->proto->done_once++)
934 return;
935
936 psmi_getenv("PSM_STRAY_ENABLED", "Enable stray process detection",
937 PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_YESNO,
938 (union psmi_envvar_val) 1, /* yes by default */
939 &env_stray);
940
941 if (env_stray.e_uint)
942 psmi_handle_error(PSMI_EP_NORETURN, PSM_EPID_NETWORK_ERROR, "PSM pid "
943 "%d on host %s has detected that I am a stray process, exiting.",
944 pid, buf);
945 else
946 psmi_syslog(ipsaddr->proto->ep, 1, LOG_INFO, "PSM pid "
947 "%d on host %s has detected that I am a stray process, "
948 "PSM_STRAY_ENABLED is off and future messages are ignored.");
949 return;
950 }
951
952 static void ips_bad_opcode(uint8_t op_code, struct ips_message_header *proto)
953 {
954 _IPATH_DBG("Discarding message with bad opcode 0x%x\n",
955 op_code);
956
957 if(infinipath_debug&__IPATH_DBG) {
958 ips_proto_show_header(proto, "received bad opcode");
959 ips_proto_dump_frame(proto, sizeof(struct ips_message_header),
960 "Opcode error protocol header dump");
961 }
962 }
963
964 static void
965 _process_unknown_opcode(struct ips_proto *proto,
966 struct ips_message_header *protocol_header)
967 {
968 proto->stats.unknown_packets++;
969
970 switch (protocol_header->sub_opcode) {
971 /* A bunch of pre-PSM packets that we don't handle any more */
972 case OPCODE_SEQ_DATA:
973 case OPCODE_SEQ_CTRL:
974 case OPCODE_STARTUP:
975 case OPCODE_STARTUP_EXT:
976 case OPCODE_STARTUP_ACK:
977 case OPCODE_STARTUP_ACK_EXT:
978 case OPCODE_STARTUP_NAK:
979 case OPCODE_STARTUP_NAK_EXT:
980 case OPCODE_CLOSE:
981 case OPCODE_ABORT:
982 case OPCODE_CLOSE_ACK:
983 break;
984 default:
985 ips_bad_opcode(protocol_header->sub_opcode, protocol_header);
986 break;
987 }
988 }
989
990 PSMI_NEVER_INLINE(
991 int
992 _process_connect(const struct ips_recvhdrq_event *rcv_ev))
993 {
994 const uint16_t lmc_mask = ~((1 << rcv_ev->proto->epinfo.ep_lmc) - 1);
995
996 return ips_proto_process_connect(rcv_ev->proto,
997 ips_epid_from_phdr(lmc_mask, rcv_ev->p_hdr),
998 rcv_ev->p_hdr->sub_opcode,
999 rcv_ev->p_hdr,
1000 ips_recvhdrq_event_payload(rcv_ev),
1001 ips_recvhdrq_event_paylen(rcv_ev));
1002 }
1003
1004 // Return 1 if packet is ok.
1005 // Return 0 if packet should be skipped
1006 int
1007 ips_proto_process_unknown(const struct ips_recvhdrq_event *rcv_ev)
1008 {
1009 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
1010 uint8_t ptype = rcv_ev->ptype;
1011 const uint64_t current_count = get_cycles();
1012 struct ips_stray_epid *sepid;
1013 struct ips_proto *proto = rcv_ev->proto;
1014 psm_ep_t ep_err;
1015 psm_epid_t epid;
1016 char *pkt_type;
1017 int opcode = (int) p_hdr->sub_opcode;
1018 double t_elapsed;
1019 ptl_epaddr_flow_t flowid = IPS_FLOWID_GET_INDEX(p_hdr->flowid);
1020 const uint16_t lmc_mask = ~((1 << rcv_ev->proto->epinfo.ep_lmc) - 1);
1021
1022 /*
1023 * If the protocol is disabled or not yet enabled, no processing happens
1024 * We set it t_init to 0 when disabling the protocol
1025 */
1026 if (proto->t_init == 0)
1027 return IPS_RECVHDRQ_CONTINUE;
1028
1029 /*
1030 * If lid is 0, something bad happened in queue processing
1031 */
1032 epid = ips_epid_from_phdr(lmc_mask, p_hdr);
1033 if (psm_epid_nid(epid) == 0ULL) {
1034 proto->stats.lid_zero_errs++;
1035 _IPATH_DBG("Skipping stray packet processing with LID=0\n");
1036 return IPS_RECVHDRQ_CONTINUE;
1037 }
1038
1039 /* Connect messages don't have to be from a known epaddr */
1040 switch (opcode) {
1041 case OPCODE_CONNECT_REQUEST:
1042 case OPCODE_CONNECT_REPLY:
1043 case OPCODE_DISCONNECT_REQUEST:
1044 case OPCODE_DISCONNECT_REPLY:
1045 _process_connect(rcv_ev);
1046 return IPS_RECVHDRQ_CONTINUE;
1047 case OPCODE_ERR_CHK_BAD: /* ignore, old opcode */
1048 return IPS_RECVHDRQ_CONTINUE;
1049 default:
1050 break;
1051 }
1052
1053 /* Packet from "unknown" peer. Log the packet and payload if at appropriate
1054 * verbose level.
1055 */
1056 {
1057 char *payload = ips_recvhdrq_event_payload(rcv_ev);
1058 uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev) +
1059 ((__be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 20) & 3);
1060
1061 ips_proto_dump_err_stats(proto);
1062
1063 if(infinipath_debug & __IPATH_PKTDBG) {
1064 ips_proto_dump_frame(rcv_ev->p_hdr, IPATH_MESSAGE_HDR_SIZE, "header");
1065 if (paylen)
1066 ips_proto_dump_frame(payload, paylen, "data");
1067 }
1068 }
1069
1070 /* Other messages are definitely crosstalk. */
1071 /* out-of-context expected messages are always fatal */
1072 if (ptype == RCVHQ_RCV_TYPE_EXPECTED) {
1073 ep_err = PSMI_EP_NORETURN;
1074 pkt_type = "expected";
1075 }
1076 else if (ptype == RCVHQ_RCV_TYPE_EAGER) {
1077 ep_err = PSMI_EP_LOGEVENT;
1078 pkt_type = "eager";
1079 }
1080 else {
1081 ep_err = PSMI_EP_NORETURN;
1082 pkt_type = "unknown";
1083 }
1084
1085 proto->stats.stray_packets++;
1086
1087 /* If we have debug mode, print the complete packet every time */
1088 if (infinipath_debug & __IPATH_PKTDBG)
1089 ips_proto_show_header(p_hdr, "invalid commidx");
1090 t_elapsed = (double)
1091 cycles_to_nanosecs(get_cycles()-proto->t_init) / 1.0e9;
1092
1093 sepid = (struct ips_stray_epid *)
1094 psmi_epid_lookup(PSMI_EP_CROSSTALK, epid);
1095 if (sepid == NULL) { /* Never seen crosstalk from this node, log it */
1096 sepid = (struct ips_stray_epid *)
1097 psmi_calloc(proto->ep, UNDEFINED, 1, sizeof(struct ips_stray_epid));
1098 if (sepid == NULL) return 0; /* skip packet if no memory */
1099 psmi_epid_add(PSMI_EP_CROSSTALK, epid, (void *) sepid);
1100 sepid->epid = epid;
1101 if (proto->stray_warn_interval)
1102 sepid->t_first = sepid->t_warn_next = current_count;
1103 }
1104 sepid->num_messages++;
1105 sepid->t_last = current_count;
1106
1107 /* If we're not going to warn the user and if this not to be a fatal
1108 * packet, just skip it */
1109 if (sepid->t_warn_next > current_count && ep_err != PSMI_EP_NORETURN)
1110 return 0;
1111
1112 sepid->t_warn_next = current_count + proto->stray_warn_interval;
1113
1114 if (p_hdr->sub_opcode == OPCODE_ERR_CHK) {
1115 /* With the new err_check, we can print out extra information */
1116 char ipbuf[INET_ADDRSTRLEN], *ip = NULL;
1117 sepid->ipv4_addr = p_hdr->data[0].u32w0;
1118 sepid->pid = __be32_to_cpu(p_hdr->data[0].u32w1);
1119 ip = (char *) inet_ntop(AF_INET, &sepid->ipv4_addr, ipbuf, sizeof ipbuf);
1120
1121 /* If the IP and PID make sense, go ahead and print useful info and
1122 * even reply with ERR_CHK_BAD. If not, fall through and print the
1123 * generic bad error message
1124 */
1125 if (ip != NULL && sepid->pid) {
1126 /* Make up a fake ipsaddr and reply */
1127 ips_epaddr_t ipsaddr_f;
1128 psm_error_t err;
1129
1130 /* debugging sanity, and catch bugs */
1131 memset(&ipsaddr_f, 0, sizeof(ips_epaddr_t));
1132 ipsaddr_f.epr.epr_context = IPS_HEADER_SRCCONTEXT_GET(p_hdr);
1133 ipsaddr_f.epr.epr_subcontext = p_hdr->dst_subcontext;
1134 ipsaddr_f.epr.epr_pkt_context =
1135 ipsaddr_f.epr.epr_context & 0xf;
1136
1137 /* Get path record for peer */
1138 err = proto->ibta.get_path_rec(proto,
1139 proto->epinfo.ep_base_lid,
1140 p_hdr->lrh[3], /* SLID */
1141 PSMI_HCA_TYPE_QLE73XX,
1142 3000, &ipsaddr_f);
1143 if (err != PSM_OK)
1144 goto fail;
1145
1146 ipsaddr_f.epr.epr_qp = __be32_to_cpu(p_hdr->bth[1]);
1147 ipsaddr_f.epr.epr_qp &= 0xffffff; /* QP is 24 bits */
1148 ipsaddr_f.ptl = (ptl_t *) -1;
1149 ipsaddr_f.proto = proto;
1150 /* Pretend the ctrlmsg is already queued, so it doesn't get queued
1151 * in this fake (stack-allocated) ptladdr */
1152 ipsaddr_f.ctrl_msg_queued = ~0;
1153 flowid = EP_FLOW_GO_BACK_N_PIO;
1154 ips_flow_init(&ipsaddr_f.flows[flowid], NULL,
1155 &ipsaddr_f, PSM_TRANSFER_PIO,
1156 PSM_PROTOCOL_GO_BACK_N, IPS_PATH_LOW_PRIORITY, flowid);
1157
1158 if (!ips_proto_send_ctrl_message(&ipsaddr_f.flows[flowid],
1159 OPCODE_ERR_CHK_BAD,
1160 &ipsaddr_f.ctrl_msg_queued,NULL)){
1161 sepid->err_check_bad_sent++;
1162 _IPATH_VDBG("did reply with ERR_CHK_BAD\n");
1163 }
1164 else
1165 _IPATH_VDBG("did *NOT* reply with ERR_CHK_BAD\n");
1166
1167 fail:
1168 psmi_handle_error(ep_err, PSM_EPID_NETWORK_ERROR,
1169 "Received %d out-of-context %s message(s) from stray process "
1170 "PID=%d running on host %s (LID 0x%x, ptype=0x%x, subop=0x%x, "
1171 "elapsed=%.3fs) %s",
1172 sepid->num_messages, pkt_type, sepid->pid, ip,
1173 (int) psm_epid_nid(epid), ptype, opcode, t_elapsed,
1174 (ep_err == PSMI_EP_NORETURN) ? "Aborting." : "");
1175 return 0;
1176 }
1177 }
1178
1179 /* At this point we either have a OPCODE_ERR_CHECK where we couldn't
1180 * extract a valid ip and pid OR some other opcode */
1181 psmi_handle_error(ep_err, PSM_EPID_NETWORK_ERROR,
1182 "Received out-of-context %s message(s) from a stray process "
1183 "running on LID 0x%x ptype=0x%x subop=0x%x elapsed=%.3fs",
1184 pkt_type, (int) psm_epid_nid(epid), ptype, opcode, t_elapsed);
1185
1186 return 0; /* Always skip this packet unless the above call was a noreturn
1187 * call */
1188 }
1189
1190 /* get the error string as a number and a string */
1191 static void rhf_errnum_string(char *msg, size_t msglen, long err)
1192 {
1193 int len;
1194 char *errmsg;
1195
1196 len = snprintf(msg, msglen, "RHFerror %lx: ", err);
1197 if(len > 0 && len < msglen) {
1198 errmsg = msg + len;
1199 msglen -= len;
1200 }
1201 else
1202 errmsg = msg;
1203 *errmsg = 0;
1204 ips_proto_get_rhf_errstring(err, errmsg, msglen);
1205 }
1206
1207 /*
1208 * Error handling
1209 */
1210 int __recvpath
1211 ips_proto_process_packet_error(struct ips_recvhdrq_event *rcv_ev)
1212 {
1213 struct ips_proto *proto = rcv_ev->proto;
1214 int pkt_verbose_err = infinipath_debug & __IPATH_PKTDBG;
1215 int tiderr = rcv_ev->error_flags & INFINIPATH_RHF_H_TIDERR;
1216 int tf_seqerr = rcv_ev->error_flags & INFINIPATH_RHF_H_TFSEQERR;
1217 int tf_generr = rcv_ev->error_flags & INFINIPATH_RHF_H_TFGENERR;
1218 int data_err = rcv_ev->error_flags &
1219 (INFINIPATH_RHF_H_ICRCERR | INFINIPATH_RHF_H_VCRCERR |
1220 INFINIPATH_RHF_H_PARITYERR | INFINIPATH_RHF_H_LENERR |
1221 INFINIPATH_RHF_H_MTUERR | INFINIPATH_RHF_H_IHDRERR |
1222 INFINIPATH_RHF_H_IBERR);
1223 char pktmsg[128];
1224
1225 *pktmsg = 0;
1226 /*
1227 * Tid errors on eager pkts mean we get a headerq overflow, perfectly
1228 * safe. Tid errors on expected or other packets means trouble.
1229 */
1230 if (tiderr && rcv_ev->ptype == RCVHQ_RCV_TYPE_EAGER) {
1231 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
1232
1233
1234 /* Payload dropped - Determine flow for this header and see if
1235 * we need to generate a NAK.
1236 *
1237 * ALL PACKET DROPS IN THIS CATEGORY CAN BE FLAGGED AS DROPPED DUE TO
1238 * CONGESTION AS THE EAGER BUFFER IS FULL.
1239 *
1240 * Possible eager packet type:
1241 *
1242 * Ctrl Message - ignore
1243 * MQ message - Can get flow and see if we need to NAK.
1244 * AM message - Can get flow and see if we need to NAK.
1245 */
1246
1247 proto->stats.hdr_overflow++;
1248 if (data_err)
1249 return 0;
1250
1251 switch(p_hdr->sub_opcode) {
1252 case OPCODE_SEQ_MQ_HDR:
1253 case OPCODE_SEQ_MQ_CTRL:
1254 case OPCODE_AM_REQUEST:
1255 case OPCODE_AM_REQUEST_NOREPLY:
1256 case OPCODE_AM_REPLY:
1257 {
1258 ptl_epaddr_flow_t flowid = ips_proto_flowid(p_hdr);
1259 struct ips_epstate_entry *epstaddr;
1260 struct ips_flow *flow;
1261 psmi_seqnum_t sequence_num;
1262 int16_t diff;
1263
1264 /* Obtain ipsaddr for packet */
1265 epstaddr = ips_epstate_lookup(rcv_ev->recvq->epstate,
1266 rcv_ev->p_hdr->commidx +
1267 INFINIPATH_KPF_RESERVED_BITS(p_hdr->iph.pkt_flags));
1268 if_pf (epstaddr == NULL || epstaddr->epid != rcv_ev->epid)
1269 return 0; /* Unknown packet - drop */
1270
1271 rcv_ev->ipsaddr = epstaddr->ipsaddr;
1272 flow = &rcv_ev->ipsaddr->flows[flowid];
1273 sequence_num.val = __be32_to_cpu(p_hdr->bth[2]);
1274 diff = (int16_t) (sequence_num.pkt - flow->recv_seq_num.pkt);
1275
1276 if (diff >= 0 && !(flow->flags & IPS_FLOW_FLAG_NAK_SEND)) {
1277 /* Mark flow as congested and attempt to generate NAK */
1278 flow->flags |= IPS_FLOW_FLAG_GEN_BECN;
1279 rcv_ev->ipsaddr->stats.congestion_pkts++;
1280 flow->last_seq_num = sequence_num;
1281
1282 flow->flags |= IPS_FLOW_FLAG_NAK_SEND;
1283 flow->cca_ooo_pkts = 0;
1284 ips_proto_send_nak((struct ips_recvhdrq *) rcv_ev->recvq, flow);
1285 }
1286
1287 /* Safe to process ACKs from header */
1288 ips_proto_process_ack(rcv_ev);
1289 }
1290 break;
1291 default:
1292 break;
1293 }
1294 }
1295 else if (tiderr) /* tid error, but not on an eager pkt */
1296 {
1297 psm_ep_t ep_err = PSMI_EP_LOGEVENT;
1298 int many_tids = 0, many_epids = 0;
1299 uint32_t context_tid_off =
1300 __le32_to_cpu(rcv_ev->p_hdr->iph.ver_context_tid_offset);
1301 uint16_t tid, offset;
1302 uint64_t t_now = get_cycles();
1303
1304 proto->tiderr_cnt++;
1305
1306 /* Whether and how we will be logging this event */
1307 if (proto->tiderr_max > 0 && proto->tiderr_cnt >= proto->tiderr_max)
1308 ep_err = PSMI_EP_NORETURN;
1309 else if (proto->tiderr_warn_interval != UINT64_MAX &&
1310 proto->tiderr_tnext <= t_now)
1311 proto->tiderr_tnext = get_cycles() + proto->tiderr_warn_interval;
1312 else
1313 ep_err = NULL;
1314
1315 if (ep_err != NULL) {
1316 if (proto->tiderr_context_tid_off != context_tid_off) { /* many tids */
1317 if (proto->tiderr_context_tid_off != 0)
1318 many_tids = 1;
1319 proto->tiderr_context_tid_off = context_tid_off;
1320 }
1321
1322 if (proto->tiderr_epid != rcv_ev->epid) { /* many epids */
1323 if (proto->tiderr_epid != 0)
1324 many_epids = 1;
1325 proto->tiderr_epid = rcv_ev->epid;
1326 }
1327
1328 rhf_errnum_string(pktmsg, sizeof(pktmsg), rcv_ev->error_flags);
1329
1330 tid = (context_tid_off >> INFINIPATH_I_TID_SHIFT) &
1331 INFINIPATH_I_TID_MASK;
1332 offset = (context_tid_off>>INFINIPATH_I_OFFSET_SHIFT) &
1333 INFINIPATH_I_OFFSET_MASK;
1334
1335 psmi_handle_error(ep_err, PSM_EP_DEVICE_FAILURE,
1336 "%s with tid=%d,offset=%d,count=%d "
1337 "from %s%s %s %s",
1338 many_tids ? "Multiple TID Errors" : "TID Error",
1339 tid, offset, proto->tiderr_cnt,
1340 psmi_epaddr_get_name(rcv_ev->epid),
1341 many_epids ? " (and other hosts)" : "",
1342 pktmsg, ep_err == PSMI_EP_NORETURN ?
1343 "(Terminating...)" : "");
1344 }
1345
1346 if (proto->protoexp && rcv_ev->ptype == RCVHQ_RCV_TYPE_EXPECTED)
1347 ips_protoexp_handle_tiderr(rcv_ev);
1348 }
1349 else if (tf_generr)
1350 ips_protoexp_handle_tf_generr(rcv_ev);
1351 else if (tf_seqerr)
1352 ips_protoexp_handle_tf_seqerr(rcv_ev);
1353 else if (data_err) {
1354 uint8_t op_code = __be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 24 & 0xFF;
1355
1356 if (!pkt_verbose_err) {
1357 rhf_errnum_string(pktmsg, sizeof(pktmsg), rcv_ev->error_flags);
1358 _IPATH_DBG("Error %s pkt type opcode 0x%x at hd=0x%x %s\n",
1359 (rcv_ev->ptype == RCVHQ_RCV_TYPE_EAGER) ? "eager" :
1360 (rcv_ev->ptype == RCVHQ_RCV_TYPE_EXPECTED) ? "expected" :
1361 (rcv_ev->ptype == RCVHQ_RCV_TYPE_NON_KD) ? "non-kd" :
1362 "<error>",
1363 op_code, rcv_ev->recvq->state->hdrq_head, pktmsg);
1364 }
1365
1366 if (proto->protoexp && rcv_ev->ptype == RCVHQ_RCV_TYPE_EXPECTED)
1367 ips_protoexp_handle_data_err(rcv_ev);
1368 }
1369 else { /* not a tid or data error -- some other error */
1370 uint8_t op_code = __be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 24 & 0xFF;
1371
1372 if (!pkt_verbose_err)
1373 rhf_errnum_string(pktmsg, sizeof(pktmsg), rcv_ev->error_flags);
1374
1375 /* else RHFerr decode printed below */
1376 _IPATH_DBG("Error pkt type 0x%x opcode 0x%x at hd=0x%x %s\n",
1377 rcv_ev->ptype, op_code, rcv_ev->recvq->state->hdrq_head, pktmsg);
1378 }
1379 if (pkt_verbose_err) {
1380 if(!*pktmsg)
1381 rhf_errnum_string(pktmsg, sizeof(pktmsg), rcv_ev->error_flags);
1382 ips_proto_show_header(rcv_ev->p_hdr, pktmsg);
1383 }
1384
1385 return 0;
1386 }
1387
1388 /*
1389 * Only valid packets make it to this point.
1390 */
1391 int __recvpath
1392 ips_proto_process_packet_inner(struct ips_recvhdrq_event *rcv_ev)
1393 {
1394 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
1395 ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr;
1396 int ret = IPS_RECVHDRQ_CONTINUE;
1397
1398 /* NOTE: Fault injection will currently not work with hardware suppression
1399 * on QLE73XX. See TODO below for reason why as we currently do not update
1400 * the hardware tidflow table if FI is dropping the packet.
1401 *
1402 * TODO: We need to look into the packet before dropping it and
1403 * if it's an expected packet AND we have hardware suppression then we
1404 * need to update the hardware tidflow table and the associated tidrecvc
1405 * state to fake having received a packet uptil some point in the window
1406 * defined by the loss rate. This way the subsequent err chk will be NAKd
1407 * and we can resync the flow with the sender.
1408 *
1409 * Note: For real errors the hardware generates seq/gen errors which are
1410 * handled appropriately by the protocol.
1411 */
1412
1413 if_pf (PSMI_FAULTINJ_ENABLED()) {
1414 PSMI_FAULTINJ_STATIC_DECL(fi_recv, "recvlost", 1, IPS_FAULTINJ_RECVLOST);
1415 if (psmi_faultinj_is_fault(fi_recv))
1416 return ret;
1417 }
1418
1419 switch (rcv_ev->ptype) {
1420 case RCVHQ_RCV_TYPE_EAGER:
1421 #if 0
1422 _IPATH_VDBG("got packet from %d with opcode=%x, seqno=%d\n",
1423 p_hdr->commidx,
1424 p_hdr->sub_opcode,
1425 __be32_to_cpu(p_hdr->bth[2]));
1426 #endif
1427
1428 switch ( p_hdr->sub_opcode ) {
1429 case OPCODE_SEQ_MQ_HDR:
1430 case OPCODE_SEQ_MQ_CTRL:
1431 ret = _process_mq(rcv_ev);
1432 break;
1433
1434 case OPCODE_ACK:
1435 ips_proto_process_ack(rcv_ev);
1436 break;
1437
1438 case OPCODE_NAK:
1439 _process_nak(rcv_ev);
1440 break;
1441
1442 case OPCODE_AM_REQUEST:
1443 case OPCODE_AM_REQUEST_NOREPLY:
1444 case OPCODE_AM_REPLY:
1445 ret = ips_proto_am(rcv_ev);
1446 break;
1447 case OPCODE_FLOW_CCA_BECN:
1448 {
1449 struct ips_proto *proto = ipsaddr->proto;
1450 struct ips_flow *flow = NULL;
1451 psm_protocol_type_t protocol;
1452 ptl_epaddr_flow_t flowid;
1453
1454 protocol = IPS_FLOWID_GET_PROTO(p_hdr->flowid);
1455 flowid = IPS_FLOWID_GET_INDEX(p_hdr->flowid);
1456 psmi_assert_always(protocol == PSM_PROTOCOL_GO_BACK_N);
1457 flow = &ipsaddr->flows[flowid];
1458
1459 if ((flow->path->epr_ccti +
1460 proto->cace[flow->path->epr_sl].ccti_increase) <=
1461 proto->ccti_limit) {
1462 ips_cca_adjust_rate(flow->path,
1463 proto->cace[flow->path->epr_sl].ccti_increase);
1464 /* Clear congestion event */
1465 rcv_ev->is_congested &= ~IPS_RECV_EVENT_BECN;
1466 }
1467 }
1468 break;
1469
1470 case OPCODE_ERR_CHK:
1471 case OPCODE_ERR_CHK_OLD:
1472 _process_err_chk((struct ips_recvhdrq *) rcv_ev->recvq,
1473 ipsaddr, p_hdr);
1474 /* Ignore FECN bit since this is the control path */
1475 rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN;
1476 break;
1477
1478 case OPCODE_ERR_CHK_GEN:
1479 _process_err_chk_gen(ipsaddr, p_hdr);
1480 /* Ignore FECN bit since this is the control path */
1481 rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN;
1482 break;
1483
1484 case OPCODE_ERR_CHK_PLS: /* skip for now */
1485 break;
1486
1487 case OPCODE_ERR_CHK_BAD:
1488 _process_err_chk_bad(ipsaddr, p_hdr);
1489 break;
1490
1491 case OPCODE_TIDS_GRANT:
1492 ips_protoexp_tid_grant(rcv_ev);
1493 /* Ignore FECN bit since this is the control path */
1494 rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN;
1495 break;
1496
1497 case OPCODE_TIDS_GRANT_ACK:
1498 ips_protoexp_tid_grant_ack(rcv_ev);
1499 /* Ignore FECN bit since this is the control path */
1500 rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN;
1501 break;
1502
1503 case OPCODE_TIDS_RELEASE:
1504 ret = ips_protoexp_tid_release(rcv_ev);
1505 /* Ignore FECN bit since this is the control path */
1506 rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN;
1507 break;
1508
1509 case OPCODE_TIDS_RELEASE_CONFIRM:
1510 ips_protoexp_tid_release_ack(rcv_ev);
1511 /* Ignore FECN bit since this is the control path */
1512 rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN;
1513 break;
1514
1515 case OPCODE_SEQ_MQ_EXPTID:
1516 ips_protoexp_data(rcv_ev);
1517 break;
1518
1519 case OPCODE_SEQ_MQ_EXPTID_UNALIGNED:
1520 ips_protoexp_recv_unaligned_data(rcv_ev);
1521 break;
1522
1523 case OPCODE_CONNECT_REQUEST:
1524 case OPCODE_CONNECT_REPLY:
1525 case OPCODE_DISCONNECT_REQUEST:
1526 case OPCODE_DISCONNECT_REPLY:
1527 _process_connect(rcv_ev);
1528 break;
1529
1530 default: /* skip unsupported opcodes */
1531 _process_unknown_opcode(rcv_ev->proto, p_hdr);
1532 break;
1533 } /* switch (op_code) */
1534 break;
1535
1536 case RCVHQ_RCV_TYPE_EXPECTED:
1537 ips_protoexp_data(rcv_ev);
1538 break;
1539
1540 default: /* unknown frame type */
1541 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
1542 "Unknown frame type %x", rcv_ev->ptype);
1543 break;
1544 } /* switch (ptype) */
1545
1546 return ret;
1547 }