Debian Codegraph @

ptl_ips/ips_recvhdrq.c

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "ips_proto.h"
36 #include "ips_proto_internal.h"
37 #include "ips_recvhdrq.h"
38
39 /*
40 * TUNABLES TUNABLES TUNABLES
41 */
42
43 /*
44 * Receive Queue progress optimizations
45 *
46 * The recvhdrq_progress function supports 2 chip features, so can be written
47 * to support 4 possible combinations in chip features (although only 3/4 are
48 * currently implemented in our chips).
49 *
50 * We can either support recvhdrq_progress by implementing the function in 4
51 * ways and calling it through a function pointer
52 * (IPS_RCVHDRQ_THRU_FUNCTION_POINTER=1) or having one implementation that
53 * covers all possible combinations (IPS_RCVHDRQ_THRU_FUNCTION_POINTER=0).
54 */
55 #define IPS_RCVHDRQ_THRU_FUNCTION_POINTER 1
56
57 #if IPS_RCVHDRQ_THRU_FUNCTION_POINTER
58 static psm_error_t ips_recvhdrq_progress_none(struct ips_recvhdrq *recvq);
59 static psm_error_t ips_recvhdrq_progress_nortail(struct ips_recvhdrq *recvq);
60 #endif
61
62
63 psm_error_t
64 ips_recvhdrq_init(const psmi_context_t *context,
65 const struct ips_epstate *epstate,
66 const struct ips_proto *proto,
67 const struct ips_recvq_params *hdrq_params,
68 const struct ips_recvq_params *egrq_params,
69 const struct ips_recvhdrq_callbacks *callbacks,
70 uint32_t runtime_flags,
71 uint32_t subcontext,
72 struct ips_recvhdrq *recvq,
73 struct ips_recvhdrq_state *recvq_state)
74 {
75 const struct ipath_base_info *base_info = &context->base_info;
76 psm_error_t err = PSM_OK;
77
78 memset(recvq, 0, sizeof(*recvq));
79 recvq->proto = (struct ips_proto *) proto;
80 recvq->state = recvq_state;
81 recvq->context = context;
82 recvq->subcontext = subcontext;
83 /* This runtime flags may be different from the context's runtime flags since
84 * a receive queue may be initialised to represent a "software" receive
85 * queue (shared contexts) or a hardware receive queue */
86 recvq->runtime_flags = runtime_flags;
87 recvq->hdrq = *hdrq_params; /* deep copy */
88 pthread_spin_init(&recvq->hdrq_lock, PTHREAD_PROCESS_SHARED);
89 recvq->hdrq_rhf_off = base_info->spi_rhf_offset;
90
91 if (recvq->runtime_flags & IPATH_RUNTIME_NODMA_RTAIL) {
92 recvq->hdrq_rhf_notail = 1;
93 recvq->state->hdrq_rhf_seq = 1;
94 }
95 else {
96 recvq->hdrq_rhf_notail = 0;
97 recvq->state->hdrq_rhf_seq = 0; /* _seq is ignored */
98 }
99 recvq->hdrq_elemlast = ((recvq->hdrq.elemcnt - 1) * recvq->hdrq.elemsz);
100
101 recvq->egrq = *egrq_params; /* deep copy */
102 recvq->egrq_buftable =
103 ips_recvq_egrbuf_table_alloc(context->ep, recvq->egrq.base_addr,
104 base_info->spi_rcv_egrchunksize,
105 recvq->egrq.elemcnt, recvq->egrq.elemsz);
106 if (recvq->egrq_buftable == NULL) {
107 err = psmi_handle_error(proto->ep, PSM_NO_MEMORY,
108 "Couldn't allocate memory for eager buffer index table");
109 goto fail;
110 }
111
112 recvq->epstate = epstate;
113
114 /* NOTE: We should document PSM_RCVHDRCOPY is not available with QIB? */
115
116 #if IPS_RCVHDRQ_THRU_FUNCTION_POINTER
117 /* Only either have NODMA RTAIL (for QLE73XX/QLE72XX) or just the vanilla
118 version for QLE71XX where RTAIL is DMA'd */
119 if (recvq->runtime_flags & IPATH_RUNTIME_NODMA_RTAIL)
120 recvq->progress_fn = ips_recvhdrq_progress_nortail;
121 else
122 recvq->progress_fn = ips_recvhdrq_progress_none;
123 #endif
124
125 recvq->recvq_callbacks = *callbacks; /* deep copy */
126 SLIST_INIT(&recvq->pending_acks);
127
128 recvq->state->hdrq_head = 0;
129 recvq->state->rcv_egr_index_head = NO_EAGER_UPDATE;
130 recvq->state->num_hdrq_done = 0;
131 recvq->state->hdr_countdown = 0;
132
133 {
134 union psmi_envvar_val env_hdr_update;
135 psmi_getenv("PSM_HEAD_UPDATE",
136 "header queue update interval (0 to update after all entries are processed). Default is 16",
137 PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT_FLAGS,
138 (union psmi_envvar_val) 16, &env_hdr_update);
139
140 /* Cap max header update interval to size of header/eager queue */
141 recvq->state->head_update_interval =
142 min(env_hdr_update.e_uint,
143 min(recvq->hdrq.elemcnt-1, recvq->egrq.elemcnt-1));
144 }
145
146 fail:
147 return err;
148 }
149
150 psm_error_t
151 ips_recvhdrq_fini(struct ips_recvhdrq *recvq)
152 {
153 ips_recvq_egrbuf_table_free(recvq->egrq_buftable);
154 return PSM_OK;
155 }
156
157 // flush the eager buffers, by setting the eager index head to eager index tail
158 // if eager buffer queue is full.
159 //
160 // Called when we had eager buffer overflows (ERR_TID/INFINIPATH_RHF_H_TIDERR
161 // was set in RHF errors), and no good eager packets were received, so
162 // that eager head wasn't advanced.
163 //
164
165 #if 0
166 static void ips_flush_egrq_if_required(struct ips_recvhdrq *recvq)
167 {
168 const uint32_t tail = ips_recvq_tail_get(&recvq->egrq);
169 const uint32_t head = ips_recvq_head_get(&recvq->egrq);
170 uint32_t egr_cnt = recvq->egrq.elemcnt;
171
172 if ((head % egr_cnt) == ((tail+1)%egr_cnt)) {
173 _IPATH_DBG("eager array full after overflow, flushing "
174 "(head %llx, tail %llx)\n",
175 (long long)head, (long long)tail);
176 recvq->proto->stats.egr_overflow++;
177 }
178 return;
179 }
180 #endif
181
182 /*
183 * Helpers for ips_recvhdrq_progress.
184 */
185
186 static __inline__ int
187 _get_proto_subcontext(const struct ips_message_header *p_hdr)
188 {
189 return p_hdr->dst_subcontext;
190 }
191
192 /* ipath_opcode is not the ips-level opcode. */
193 static __inline__ uint8_t
194 _get_proto_ipath_opcode(const struct ips_message_header *p_hdr)
195 {
196 return __be32_to_cpu(p_hdr->bth[0]) >> BTH_OPCODE_SHIFT & 0xFF;
197 }
198
199 /* Detrmine if FECN bit is set IBTA 1.2.1 CCA Annex A*/
200 static __inline__ uint8_t
201 _is_cca_fecn_set(const struct ips_message_header *p_hdr)
202 {
203 return (__be32_to_cpu(p_hdr->bth[1]) >> BTH_FECN_SHIFT);
204 }
205
206 /* Detrmine if BECN bit is set IBTA 1.2.1 CCA Annex A*/
207 static __inline__ uint8_t
208 _is_cca_becn_set(const struct ips_message_header *p_hdr)
209 {
210 return (__be32_to_cpu(p_hdr->bth[1]) >> BTH_BECN_SHIFT) & 0x1;
211 }
212
213 static __inline__ struct ips_message_header *
214 _get_proto_hdr_from_rhf(const uint32_t *rcv_hdr, const __le32 *rhf)
215 {
216 return (struct ips_message_header *) (rcv_hdr + ipath_hdrget_offset(rhf));
217 }
218
219 static __inline__ struct ips_message_header *
220 _get_proto_hdr(const uint32_t *rcv_hdr)
221 {
222 return (struct ips_message_header *) &rcv_hdr[2];
223 }
224
225 static __inline__ uint32_t
226 _get_rhf_seq(struct ips_recvhdrq *recvq, const __u32 *rcv_hdr)
227 {
228 return ipath_hdrget_seq((const __le32 *) rcv_hdr + recvq->hdrq_rhf_off);
229 }
230
231 static __inline__ uint32_t
232 _get_rhf_len_in_bytes(struct ips_recvhdrq *recvq, const __u32 *rcv_hdr)
233 {
234 return ipath_hdrget_length_in_bytes((const __le32*) rcv_hdr + recvq->hdrq_rhf_off);
235 }
236
237 static __inline__ void
238 _dump_invalid_pkt(struct ips_recvhdrq_event *rcv_ev)
239 {
240 char *payload = ips_recvhdrq_event_payload(rcv_ev);
241 uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev) +
242 ((__be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 20) & 3);
243
244 if(infinipath_debug & __IPATH_PKTDBG) {
245 ips_proto_dump_frame(rcv_ev->p_hdr, IPATH_MESSAGE_HDR_SIZE, "header");
246 if (paylen)
247 ips_proto_dump_frame(payload, paylen, "data");
248 }
249
250 }
251
252 static __inline__ void
253 _update_error_stats(struct ips_proto *proto, uint32_t err)
254 {
255
256 if (err & INFINIPATH_RHF_H_ICRCERR)
257 proto->error_stats.num_icrc_err++;
258 if (err & INFINIPATH_RHF_H_VCRCERR)
259 proto->error_stats.num_vcrc_err++;
260 if (err & INFINIPATH_RHF_H_PARITYERR)
261 proto->error_stats.num_ecc_err++;
262 if (err & INFINIPATH_RHF_H_LENERR)
263 proto->error_stats.num_len_err++;
264 if (err & INFINIPATH_RHF_H_MTUERR)
265 proto->error_stats.num_mtu_err++;
266 if (err & INFINIPATH_RHF_H_IHDRERR)
267 proto->error_stats.num_khdr_err++;
268 if (err & INFINIPATH_RHF_H_TIDERR)
269 proto->error_stats.num_tid_err++;
270 if (err & INFINIPATH_RHF_H_MKERR)
271 proto->error_stats.num_mk_err++;
272 if (err & INFINIPATH_RHF_H_IBERR)
273 proto->error_stats.num_ib_err++;
274 }
275
276 static int
277 _check_headers(struct ips_recvhdrq_event *rcv_ev)
278 {
279 struct ips_recvhdrq *recvq = (struct ips_recvhdrq*) rcv_ev->recvq;
280 struct ips_proto *proto = rcv_ev->proto;
281 uint32_t *lrh = (uint32_t*) rcv_ev->p_hdr;
282 const uint32_t *rcv_hdr = rcv_ev->rcv_hdr;
283 uint32_t dest_context;
284 const uint16_t pkt_dlid = __be16_to_cpu(rcv_ev->p_hdr->lrh[1]);
285 const uint16_t base_dlid = __be16_to_cpu(recvq->proto->epinfo.ep_base_lid);
286
287 /* Check that the receive header queue entry has a sane sequence number */
288 if (_get_rhf_seq(recvq, rcv_hdr) > LAST_RHF_SEQNO) {
289 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
290 "ErrPkt: Invalid header queue entry! RHF Sequence in Hdrq Seq: %d, Recvq State Seq: %d. LRH[0]: 0x%08x, LRH[1] (PktCount): 0x%08x\n", _get_rhf_seq(recvq, rcv_hdr), recvq->state->hdrq_rhf_seq, lrh[0], lrh[1]);
291 return -1;
292 }
293
294 /* Verify that the packet was destined for our context */
295 dest_context = ips_proto_dest_context_from_header(proto, rcv_ev->p_hdr);
296 if_pf (dest_context != recvq->proto->epinfo.ep_context) {
297
298 struct ips_recvhdrq_state *state = recvq->state;
299
300 /* Packet not targetted at us. Drop packet and continue */
301 ips_proto_dump_err_stats(proto);
302 _dump_invalid_pkt(rcv_ev);
303
304 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
305 "ErrPkt: Received packet for context %d on context %d. Receive Header Queue offset: 0x%x. Exiting.\n", dest_context, recvq->proto->epinfo.ep_context, state->hdrq_head);
306
307 return -1;
308 }
309
310
311 if_pf (rcv_ev->error_flags ||
312 (_get_proto_ipath_opcode(rcv_ev->p_hdr) != IPATH_OPCODE_USER1)) {
313
314 return 0; /* Error flags are special case. Let main receive loop handle
315 * packet processing after we account for it.
316 */
317 }
318
319 /* Verify that rhf packet length matches the length in LRH */
320 if_pf (_get_rhf_len_in_bytes(recvq, rcv_hdr) !=
321 (__be16_to_cpu(rcv_ev->p_hdr->lrh[2]) << 2)) {
322 _IPATH_EPDBG("ErrPkt: RHF Packet Len (0x%x) does not match LRH (0x%x).\n", _get_rhf_len_in_bytes(recvq, rcv_hdr) >> 2, __be16_to_cpu(rcv_ev->p_hdr->lrh[2]));
323
324 ips_proto_dump_err_stats(proto);
325 _dump_invalid_pkt(rcv_ev);
326 return -1;
327 }
328
329 /* Verify that the DLID matches our local LID. */
330 if_pf (!((base_dlid <= pkt_dlid) &&
331 (pkt_dlid <= (base_dlid + (1 << recvq->proto->epinfo.ep_lmc))))) {
332 _IPATH_EPDBG("ErrPkt: DLID in LRH (0x%04x) does not match local LID (0x%04x) Skipping packet!\n", rcv_ev->p_hdr->lrh[1], recvq->proto->epinfo.ep_base_lid);
333 ips_proto_dump_err_stats(proto);
334 _dump_invalid_pkt(rcv_ev);
335 return -1;
336 }
337
338 return 0;
339 }
340
341 static __inline__
342 int
343 do_pkt_cksum(struct ips_recvhdrq_event *rcv_ev)
344 {
345 char *payload = ips_recvhdrq_event_payload(rcv_ev);
346 uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev) +
347 ((__be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 20) & 3);
348 uint32_t *ckptr;
349 uint32_t recv_cksum, cksum, dest_subcontext;
350
351 /* With checksum every packet has a payload */
352 psmi_assert_always(payload);
353
354 ckptr = (uint32_t*) (payload + paylen);
355 recv_cksum = ckptr[0];
356
357 /* Calculate checksum hdr + payload (includes any padding words) */
358 cksum = 0xffffffff;
359 cksum = ips_crc_calculate(IPATH_MESSAGE_HDR_SIZE,
360 (uint8_t*) rcv_ev->p_hdr,cksum);
361 if (paylen)
362 cksum = ips_crc_calculate(paylen, (uint8_t*) payload, cksum);
363
364 if ((cksum != recv_cksum) || (ckptr[0] != ckptr[1])) {
365 struct ips_epstate_entry *epstaddr;
366 uint32_t lcontext;
367 uint32_t hd, tl;
368
369 epstaddr =
370 ips_epstate_lookup(rcv_ev->recvq->epstate, rcv_ev->p_hdr->commidx +
371 INFINIPATH_KPF_RESERVED_BITS(rcv_ev->p_hdr->iph.pkt_flags));
372 epstaddr = (epstaddr && epstaddr->ipsaddr) ? epstaddr : NULL;
373
374 lcontext =
375 epstaddr ? epstaddr->ipsaddr->proto->epinfo.ep_context : -1;
376
377 hd = rcv_ev->recvq->context->ctrl->__ipath_rcvhdrhead[0];
378 tl = rcv_ev->recvq->context->ctrl->__ipath_rcvhdrhead[-2];
379
380 dest_subcontext = _get_proto_subcontext(rcv_ev->p_hdr);
381
382 _IPATH_ERROR("ErrPkt: SharedContext: %s. Local Context: %i, Checksum mismatch from LID %d! Received Checksum: 0x%08x, Expected: 0x%08x & 0x%08x. Opcode: 0x%08x, Error Flag: 0x%08x. hdrq hd 0x%x tl 0x%x rhf 0x%x,%x, rhfseq 0x%x\n", (dest_subcontext != rcv_ev->recvq->subcontext) ? "Yes" : "No", lcontext, epstaddr ? __be16_to_cpu(epstaddr->ipsaddr->epr.epr_base_lid) : -1, cksum, ckptr[0], ckptr[1], rcv_ev->p_hdr->sub_opcode, rcv_ev->error_flags,hd, tl, rcv_ev->rhf[0], rcv_ev->rhf[1], _get_rhf_seq((struct ips_recvhdrq *) rcv_ev->recvq, rcv_ev->rcv_hdr));
383
384 /* Dump packet */
385 _dump_invalid_pkt(rcv_ev);
386 return 0; /* Packet checksum error */
387 }
388
389 return 1;
390 }
391
392 PSMI_ALWAYS_INLINE(
393 void
394 process_pending_acks(struct ips_recvhdrq *recvq))
395 {
396 /* If any pending acks, dispatch them now */
397 while (!SLIST_EMPTY(&recvq->pending_acks)) {
398 struct ips_flow *flow = SLIST_FIRST(&recvq->pending_acks);
399
400 SLIST_REMOVE_HEAD(&recvq->pending_acks, next);
401 SLIST_NEXT(flow, next) = NULL;
402
403 if (flow->flags & IPS_FLOW_FLAG_PENDING_ACK) {
404 psmi_assert_always((flow->flags & IPS_FLOW_FLAG_PENDING_NAK) == 0);
405
406 flow->flags &= ~IPS_FLOW_FLAG_PENDING_ACK;
407 ips_proto_send_ctrl_message(flow, OPCODE_ACK,
408 &flow->ipsaddr->ctrl_msg_queued, NULL);
409 }
410 else {
411 psmi_assert_always(flow->flags & IPS_FLOW_FLAG_PENDING_NAK);
412
413 flow->flags &= ~IPS_FLOW_FLAG_PENDING_NAK;
414 ips_proto_send_ctrl_message(flow, OPCODE_NAK,
415 &flow->ipsaddr->ctrl_msg_queued, NULL);
416 }
417
418 }
419
420 }
421
422 /*
423 * Core receive progress function
424 *
425 * recvhdrq_progress is the core function that services the receive header
426 * queue and optionally, the eager queue. At the lowest level, it identifies
427 * packets marked with errors by the chip and also detects and corrects when
428 * eager overflow conditions occur. At the highest level, it queries the
429 * 'epstate' interface to classify packets from "known" and "unknown"
430 * endpoints. In order to support shared contexts, it can also handle packets
431 * destined for other contexts (or "subcontexts").
432 */
433
434 #if IPS_RCVHDRQ_THRU_FUNCTION_POINTER
435 PSMI_ALWAYS_INLINE(
436 psm_error_t
437 ips_recvhdrq_progress_inner(struct ips_recvhdrq *recvq,
438 const int has_no_rtail))
439 #else
440 psm_error_t __recvpath
441 ips_recvhdrq_progress(struct ips_recvhdrq *recvq)
442 #endif
443 {
444 struct ips_recvhdrq_state *state = recvq->state;
445 const __le32 *rhf;
446 PSMI_CACHEALIGN struct ips_recvhdrq_event rcv_ev = { .proto = recvq->proto,
447 .recvq = recvq };
448
449 uint32_t num_hdrq_done = 0;
450 const int num_hdrq_todo = recvq->hdrq.elemcnt;
451 const uint32_t hdrq_elemsz = recvq->hdrq.elemsz;
452 uint32_t dest_subcontext;
453
454 int ret = IPS_RECVHDRQ_CONTINUE;
455 int done = 0;
456 int do_hdr_update = 0;
457 const uint16_t lmc_mask = ~((1 << recvq->proto->epinfo.ep_lmc) - 1);
458
459 /* Chip features */
460 #if !IPS_RCVHDRQ_THRU_FUNCTION_POINTER
461 const int has_no_rtail = recvq->runtime_flags & IPATH_RUNTIME_NODMA_RTAIL;
462 #endif
463
464 /* Both optional_eager and no_rtail features are in the same chip rev */
465 #define has_optional_eagerbuf recvq->hdrq_rhf_off
466
467 /* Returns whether the currently set 'rcv_hdr'/head is a readable entry */
468 #define next_hdrq_is_ready() \
469 (has_no_rtail ? \
470 recvq->state->hdrq_rhf_seq == _get_rhf_seq(recvq, rcv_hdr) \
471 : state->hdrq_head != hdrq_tail)
472
473 const uint32_t hdrq_tail = has_no_rtail ? 0
474 : ips_recvq_tail_get(&recvq->hdrq);
475 const uint32_t *rcv_hdr =
476 (const uint32_t *) recvq->hdrq.base_addr + state->hdrq_head;
477 uint32_t tmp_hdrq_head;
478
479 done = !next_hdrq_is_ready();
480
481 while (!done)
482 {
483
484 rhf = (const __le32 *) rcv_hdr + recvq->hdrq_rhf_off;
485 rcv_ev.error_flags = ipath_hdrget_err_flags(rhf);
486 rcv_ev.ptype = ipath_hdrget_rcv_type(rhf);
487 rcv_ev.rhf = rhf;
488 rcv_ev.rcv_hdr= rcv_hdr;
489 rcv_ev.p_hdr = recvq->hdrq_rhf_off ? _get_proto_hdr_from_rhf(rcv_hdr, rhf)
490 : _get_proto_hdr(rcv_hdr);
491 rcv_ev.epid = ips_epid_from_phdr(lmc_mask, rcv_ev.p_hdr);
492 rcv_ev.has_cksum =
493 ((recvq->proto->flags & IPS_PROTO_FLAG_CKSUM) &&
494 (rcv_ev.ptype == RCVHQ_RCV_TYPE_EAGER) &&
495 (rcv_ev.p_hdr->mqhdr != MQ_MSG_DATA_BLK) &&
496 (rcv_ev.p_hdr->mqhdr != MQ_MSG_DATA_REQ_BLK));
497
498 if_pt (recvq->proto->flags & IPS_PROTO_FLAG_CCA) {
499 /* IBTA CCA handling:
500 * If FECN bit set handle IBTA CCA protocol. For the flow that
501 * suffered congestion we flag it to generate a control packet with
502 * the BECN bit set - This is currently an unsolicited ACK.
503 *
504 * For all MQ packets the FECN processing/BECN generation is done
505 * in the is_expected_or_nak function as each eager packet is
506 * inspected there.
507 *
508 * For TIDFLOW/Expected data transfers the FECN bit/BECN generation
509 * is done in protoexp_data. Since header suppression can result
510 * in even FECN packets being suppressed the expected protocol
511 * generated addiional BECN packets if a "large" number of generations
512 * are swapped without progress being made for receive. "Large" is
513 * set empirically to 4.
514 *
515 * FECN packets are ignored for all control messages (except ACKs
516 * and NAKs) since they indicate congestion on the control path which
517 * is not rate controlled. The CCA specification allows FECN on
518 * ACKs to be disregarded as well.
519 */
520 rcv_ev.is_congested =
521 _is_cca_fecn_set(rcv_ev.p_hdr) & IPS_RECV_EVENT_FECN;
522 rcv_ev.is_congested |=
523 (_is_cca_becn_set(rcv_ev.p_hdr) << (IPS_RECV_EVENT_BECN - 1));
524 }
525 else
526 rcv_ev.is_congested = 0;
527
528 dest_subcontext = _get_proto_subcontext(rcv_ev.p_hdr);
529
530 if_pf (_check_headers(&rcv_ev))
531 goto skip_packet;
532
533 if_pf (rcv_ev.error_flags ||
534 (_get_proto_ipath_opcode(rcv_ev.p_hdr) != IPATH_OPCODE_USER1))
535 {
536
537 _update_error_stats(recvq->proto, rcv_ev.error_flags);
538
539 if ((rcv_ev.error_flags & INFINIPATH_RHF_H_TIDERR) ||
540 (rcv_ev.error_flags & INFINIPATH_RHF_H_TFSEQERR) ||
541 (rcv_ev.error_flags & INFINIPATH_RHF_H_TFGENERR)) {
542 /* Subcontexts need to see expected tid errors */
543 if (rcv_ev.ptype == RCVHQ_RCV_TYPE_EXPECTED &&
544 dest_subcontext != recvq->subcontext)
545 goto subcontext_packet;
546
547 recvq->recvq_callbacks.callback_error(&rcv_ev);
548
549 if (rcv_ev.ptype == RCVHQ_RCV_TYPE_EAGER) {
550 /* tiderr and eager, don't consider updating egr head */
551 if (state->hdr_countdown == 0 &&
552 state->rcv_egr_index_head == NO_EAGER_UPDATE) {
553 /* eager-full is not currently under tracing. */
554 uint32_t egr_cnt = recvq->egrq.elemcnt;
555 const uint32_t etail = ips_recvq_tail_get(&recvq->egrq);
556 const uint32_t ehead = ips_recvq_head_get(&recvq->egrq);
557
558 if (ehead == ((etail+1)%egr_cnt)) {
559 /* eager is full, trace existing header entries */
560 uint32_t hdr_size = recvq->hdrq_elemlast + hdrq_elemsz;
561 const uint32_t htail = ips_recvq_tail_get(&recvq->hdrq);
562 const uint32_t hhead = state->hdrq_head;
563
564 state->hdr_countdown = (htail > hhead) ?
565 (htail - hhead) : (htail + hdr_size - hhead);
566 }
567 }
568 goto skip_packet_no_egr_update;
569 }
570 }
571 else
572 recvq->recvq_callbacks.callback_error(&rcv_ev);
573 goto skip_packet;
574 }
575
576 /* If checksum is enabled, verify that it is valid */
577 if_pf (rcv_ev.has_cksum && !do_pkt_cksum(&rcv_ev))
578 goto skip_packet;
579
580 if (dest_subcontext == recvq->subcontext) {
581 /* Classify packet from a known or unknown endpoint */
582 struct ips_epstate_entry *epstaddr;
583
584 epstaddr =
585 ips_epstate_lookup(recvq->epstate, rcv_ev.p_hdr->commidx +
586 INFINIPATH_KPF_RESERVED_BITS(rcv_ev.p_hdr->iph.pkt_flags));
587 if_pf (epstaddr == NULL || epstaddr->epid != rcv_ev.epid) {
588 rcv_ev.ipsaddr = NULL;
589 recvq->recvq_callbacks.callback_packet_unknown(&rcv_ev);
590 }
591 else {
592 rcv_ev.ipsaddr = epstaddr->ipsaddr;
593 ret = ips_proto_process_packet(&rcv_ev);
594 if (ret == IPS_RECVHDRQ_OOO) return PSM_OK_NO_PROGRESS;
595 }
596 }
597 else {
598 subcontext_packet:
599 /* If the destination is not our subcontext, process message
600 * as a subcontext message (shared contexts) */
601 rcv_ev.ipsaddr = NULL;
602
603 ret = recvq->recvq_callbacks.callback_subcontext(&rcv_ev,
604 dest_subcontext);
605 }
606
607 skip_packet:
608 /*
609 * important to update rcv_egr_index_head iff
610 * 1. Packet was of type eager
611 * 2. Packet actually consumed an eagerbuf (post QLE72XX)
612 * 3. Packet was *not* an eager header with RHF_H_TIDERR to mark
613 * an eager overflow
614 */
615 if (has_optional_eagerbuf ? ipath_hdrget_use_egr_buf(rhf)
616 : (rcv_ev.ptype == RCVHQ_RCV_TYPE_EAGER)) {
617 state->rcv_egr_index_head = ipath_hdrget_index(rhf);
618 /* a header entry is using an eager entry, stop tracing. */
619 state->hdr_countdown = 0;
620 }
621
622 skip_packet_no_egr_update:
623 /* Note that state->hdrq_head is sampled speculatively by the code
624 * in ips_ptl_shared_poll() when context sharing, so it is not safe
625 * for this shared variable to temporarily exceed the last element. */
626 tmp_hdrq_head = state->hdrq_head + hdrq_elemsz;
627 if_pt (tmp_hdrq_head <= recvq->hdrq_elemlast)
628 state->hdrq_head = tmp_hdrq_head;
629 else
630 state->hdrq_head = 0;
631
632 if_pf (has_no_rtail && ++recvq->state->hdrq_rhf_seq > LAST_RHF_SEQNO)
633 recvq->state->hdrq_rhf_seq = 1;
634
635 state->num_hdrq_done++;
636 num_hdrq_done++;
637 rcv_hdr = (const uint32_t *) recvq->hdrq.base_addr + state->hdrq_head;
638 done = (!next_hdrq_is_ready() || (ret == IPS_RECVHDRQ_BREAK) ||
639 (num_hdrq_done == num_hdrq_todo));
640
641 do_hdr_update = (state->head_update_interval ?
642 (state->num_hdrq_done == state->head_update_interval) : done);
643 if (do_hdr_update) {
644 ips_recvq_head_update(&recvq->hdrq, state->hdrq_head);
645
646 /* Lazy update of egrq */
647 if (state->rcv_egr_index_head != NO_EAGER_UPDATE) {
648 ips_recvq_head_update(&recvq->egrq, state->rcv_egr_index_head);
649 state->rcv_egr_index_head = NO_EAGER_UPDATE;
650 }
651
652 /* Process any pending acks while updated eager/headq queue */
653 process_pending_acks(recvq);
654
655 /* Reset header queue entries processed */
656 state->num_hdrq_done = 0;
657 }
658
659 if (state->hdr_countdown > 0) {
660 /* a header entry is consumed. */
661 state->hdr_countdown -= hdrq_elemsz;
662 if (state->hdr_countdown == 0) {
663 /* header entry count reaches zero. */
664 const uint32_t tail = ips_recvq_tail_get(&recvq->egrq);
665 const uint32_t head = ips_recvq_head_get(&recvq->egrq);
666 uint32_t egr_cnt = recvq->egrq.elemcnt;
667
668 /* Checks eager-full again. This is a real false-egr-full */
669 if (head == ((tail+1)%egr_cnt)) {
670 ips_recvq_head_update(&recvq->egrq, tail);
671 _IPATH_DBG("eager array full after overflow, flushing "
672 "(head %llx, tail %llx)\n",
673 (long long)head, (long long)tail);
674 recvq->proto->stats.egr_overflow++;
675 } else
676 _IPATH_ERROR("PSM BUG: EgrOverflow: eager queue is not full\n");
677 }
678 }
679 }
680 /* while (hdrq_entries_to_read) */
681
682 /* Process any pending acks before exiting */
683 process_pending_acks(recvq);
684
685 return num_hdrq_done ? PSM_OK : PSM_OK_NO_PROGRESS;
686 }
687
688 #if IPS_RCVHDRQ_THRU_FUNCTION_POINTER
689 /*
690 * QLE71XX
691 */
692 static
693 psm_error_t __recvpath
694 ips_recvhdrq_progress_none(struct ips_recvhdrq *recvq)
695 {
696 const int has_no_rtail = 0;
697 return ips_recvhdrq_progress_inner(recvq, has_no_rtail);
698 }
699
700 /*
701 * QLE72XX+
702 */
703 static
704 psm_error_t __recvpath
705 ips_recvhdrq_progress_nortail(struct ips_recvhdrq *recvq)
706 {
707 const int has_no_rtail = 1;
708 return ips_recvhdrq_progress_inner(recvq, has_no_rtail);
709 }
710
711 psm_error_t __recvpath
712 ips_recvhdrq_progress(struct ips_recvhdrq *recvq)
713 {
714 /* Call the progress function with the right chip features. */
715 return recvq->progress_fn(recvq);
716 }
717 #endif