| 1 |
/* |
| 2 |
* Copyright (c) 2013. Intel Corporation. All rights reserved. |
| 3 |
* Copyright (c) 2006-2012. QLogic Corporation. All rights reserved. |
| 4 |
* Copyright (c) 2003-2006, PathScale, Inc. All rights reserved. |
| 5 |
* |
| 6 |
* This software is available to you under a choice of one of two |
| 7 |
* licenses. You may choose to be licensed under the terms of the GNU |
| 8 |
* General Public License (GPL) Version 2, available from the file |
| 9 |
* COPYING in the main directory of this source tree, or the |
| 10 |
* OpenIB.org BSD license below: |
| 11 |
* |
| 12 |
* Redistribution and use in source and binary forms, with or |
| 13 |
* without modification, are permitted provided that the following |
| 14 |
* conditions are met: |
| 15 |
* |
| 16 |
* - Redistributions of source code must retain the above |
| 17 |
* copyright notice, this list of conditions and the following |
| 18 |
* disclaimer. |
| 19 |
* |
| 20 |
* - Redistributions in binary form must reproduce the above |
| 21 |
* copyright notice, this list of conditions and the following |
| 22 |
* disclaimer in the documentation and/or other materials |
| 23 |
* provided with the distribution. |
| 24 |
* |
| 25 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 26 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 27 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 28 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 29 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 30 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 31 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 32 |
* SOFTWARE. |
| 33 |
*/ |
| 34 |
|
| 35 |
#include "ips_proto.h" |
| 36 |
#include "ips_proto_internal.h" |
| 37 |
#include "ips_recvhdrq.h" |
| 38 |
|
| 39 |
/* |
| 40 |
* TUNABLES TUNABLES TUNABLES |
| 41 |
*/ |
| 42 |
|
| 43 |
/* |
| 44 |
* Receive Queue progress optimizations |
| 45 |
* |
| 46 |
* The recvhdrq_progress function supports 2 chip features, so can be written |
| 47 |
* to support 4 possible combinations in chip features (although only 3/4 are |
| 48 |
* currently implemented in our chips). |
| 49 |
* |
| 50 |
* We can either support recvhdrq_progress by implementing the function in 4 |
| 51 |
* ways and calling it through a function pointer |
| 52 |
* (IPS_RCVHDRQ_THRU_FUNCTION_POINTER=1) or having one implementation that |
| 53 |
* covers all possible combinations (IPS_RCVHDRQ_THRU_FUNCTION_POINTER=0). |
| 54 |
*/ |
| 55 |
#define IPS_RCVHDRQ_THRU_FUNCTION_POINTER 1 |
| 56 |
|
| 57 |
#if IPS_RCVHDRQ_THRU_FUNCTION_POINTER |
| 58 |
static psm_error_t ips_recvhdrq_progress_none(struct ips_recvhdrq *recvq); |
| 59 |
static psm_error_t ips_recvhdrq_progress_nortail(struct ips_recvhdrq *recvq); |
| 60 |
#endif |
| 61 |
|
| 62 |
|
| 63 |
psm_error_t |
| 64 |
ips_recvhdrq_init(const psmi_context_t *context, |
| 65 |
const struct ips_epstate *epstate, |
| 66 |
const struct ips_proto *proto, |
| 67 |
const struct ips_recvq_params *hdrq_params, |
| 68 |
const struct ips_recvq_params *egrq_params, |
| 69 |
const struct ips_recvhdrq_callbacks *callbacks, |
| 70 |
uint32_t runtime_flags, |
| 71 |
uint32_t subcontext, |
| 72 |
struct ips_recvhdrq *recvq, |
| 73 |
struct ips_recvhdrq_state *recvq_state) |
| 74 |
{ |
| 75 |
const struct ipath_base_info *base_info = &context->base_info; |
| 76 |
psm_error_t err = PSM_OK; |
| 77 |
|
| 78 |
memset(recvq, 0, sizeof(*recvq)); |
| 79 |
recvq->proto = (struct ips_proto *) proto; |
| 80 |
recvq->state = recvq_state; |
| 81 |
recvq->context = context; |
| 82 |
recvq->subcontext = subcontext; |
| 83 |
/* This runtime flags may be different from the context's runtime flags since |
| 84 |
* a receive queue may be initialised to represent a "software" receive |
| 85 |
* queue (shared contexts) or a hardware receive queue */ |
| 86 |
recvq->runtime_flags = runtime_flags; |
| 87 |
recvq->hdrq = *hdrq_params; /* deep copy */ |
| 88 |
pthread_spin_init(&recvq->hdrq_lock, PTHREAD_PROCESS_SHARED); |
| 89 |
recvq->hdrq_rhf_off = base_info->spi_rhf_offset; |
| 90 |
|
| 91 |
if (recvq->runtime_flags & IPATH_RUNTIME_NODMA_RTAIL) { |
| 92 |
recvq->hdrq_rhf_notail = 1; |
| 93 |
recvq->state->hdrq_rhf_seq = 1; |
| 94 |
} |
| 95 |
else { |
| 96 |
recvq->hdrq_rhf_notail = 0; |
| 97 |
recvq->state->hdrq_rhf_seq = 0; /* _seq is ignored */ |
| 98 |
} |
| 99 |
recvq->hdrq_elemlast = ((recvq->hdrq.elemcnt - 1) * recvq->hdrq.elemsz); |
| 100 |
|
| 101 |
recvq->egrq = *egrq_params; /* deep copy */ |
| 102 |
recvq->egrq_buftable = |
| 103 |
ips_recvq_egrbuf_table_alloc(context->ep, recvq->egrq.base_addr, |
| 104 |
base_info->spi_rcv_egrchunksize, |
| 105 |
recvq->egrq.elemcnt, recvq->egrq.elemsz); |
| 106 |
if (recvq->egrq_buftable == NULL) { |
| 107 |
err = psmi_handle_error(proto->ep, PSM_NO_MEMORY, |
| 108 |
"Couldn't allocate memory for eager buffer index table"); |
| 109 |
goto fail; |
| 110 |
} |
| 111 |
|
| 112 |
recvq->epstate = epstate; |
| 113 |
|
| 114 |
/* NOTE: We should document PSM_RCVHDRCOPY is not available with QIB? */ |
| 115 |
|
| 116 |
#if IPS_RCVHDRQ_THRU_FUNCTION_POINTER |
| 117 |
/* Only either have NODMA RTAIL (for QLE73XX/QLE72XX) or just the vanilla |
| 118 |
version for QLE71XX where RTAIL is DMA'd */ |
| 119 |
if (recvq->runtime_flags & IPATH_RUNTIME_NODMA_RTAIL) |
| 120 |
recvq->progress_fn = ips_recvhdrq_progress_nortail; |
| 121 |
else |
| 122 |
recvq->progress_fn = ips_recvhdrq_progress_none; |
| 123 |
#endif |
| 124 |
|
| 125 |
recvq->recvq_callbacks = *callbacks; /* deep copy */ |
| 126 |
SLIST_INIT(&recvq->pending_acks); |
| 127 |
|
| 128 |
recvq->state->hdrq_head = 0; |
| 129 |
recvq->state->rcv_egr_index_head = NO_EAGER_UPDATE; |
| 130 |
recvq->state->num_hdrq_done = 0; |
| 131 |
recvq->state->hdr_countdown = 0; |
| 132 |
|
| 133 |
{ |
| 134 |
union psmi_envvar_val env_hdr_update; |
| 135 |
psmi_getenv("PSM_HEAD_UPDATE", |
| 136 |
"header queue update interval (0 to update after all entries are processed). Default is 16", |
| 137 |
PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT_FLAGS, |
| 138 |
(union psmi_envvar_val) 16, &env_hdr_update); |
| 139 |
|
| 140 |
/* Cap max header update interval to size of header/eager queue */ |
| 141 |
recvq->state->head_update_interval = |
| 142 |
min(env_hdr_update.e_uint, |
| 143 |
min(recvq->hdrq.elemcnt-1, recvq->egrq.elemcnt-1)); |
| 144 |
} |
| 145 |
|
| 146 |
fail: |
| 147 |
return err; |
| 148 |
} |
| 149 |
|
| 150 |
psm_error_t |
| 151 |
ips_recvhdrq_fini(struct ips_recvhdrq *recvq) |
| 152 |
{ |
| 153 |
ips_recvq_egrbuf_table_free(recvq->egrq_buftable); |
| 154 |
return PSM_OK; |
| 155 |
} |
| 156 |
|
| 157 |
// flush the eager buffers, by setting the eager index head to eager index tail |
| 158 |
// if eager buffer queue is full. |
| 159 |
// |
| 160 |
// Called when we had eager buffer overflows (ERR_TID/INFINIPATH_RHF_H_TIDERR |
| 161 |
// was set in RHF errors), and no good eager packets were received, so |
| 162 |
// that eager head wasn't advanced. |
| 163 |
// |
| 164 |
|
| 165 |
#if 0 |
| 166 |
static void ips_flush_egrq_if_required(struct ips_recvhdrq *recvq) |
| 167 |
{ |
| 168 |
const uint32_t tail = ips_recvq_tail_get(&recvq->egrq); |
| 169 |
const uint32_t head = ips_recvq_head_get(&recvq->egrq); |
| 170 |
uint32_t egr_cnt = recvq->egrq.elemcnt; |
| 171 |
|
| 172 |
if ((head % egr_cnt) == ((tail+1)%egr_cnt)) { |
| 173 |
_IPATH_DBG("eager array full after overflow, flushing " |
| 174 |
"(head %llx, tail %llx)\n", |
| 175 |
(long long)head, (long long)tail); |
| 176 |
recvq->proto->stats.egr_overflow++; |
| 177 |
} |
| 178 |
return; |
| 179 |
} |
| 180 |
#endif |
| 181 |
|
| 182 |
/* |
| 183 |
* Helpers for ips_recvhdrq_progress. |
| 184 |
*/ |
| 185 |
|
| 186 |
static __inline__ int |
| 187 |
_get_proto_subcontext(const struct ips_message_header *p_hdr) |
| 188 |
{ |
| 189 |
return p_hdr->dst_subcontext; |
| 190 |
} |
| 191 |
|
| 192 |
/* ipath_opcode is not the ips-level opcode. */ |
| 193 |
static __inline__ uint8_t |
| 194 |
_get_proto_ipath_opcode(const struct ips_message_header *p_hdr) |
| 195 |
{ |
| 196 |
return __be32_to_cpu(p_hdr->bth[0]) >> BTH_OPCODE_SHIFT & 0xFF; |
| 197 |
} |
| 198 |
|
| 199 |
/* Detrmine if FECN bit is set IBTA 1.2.1 CCA Annex A*/ |
| 200 |
static __inline__ uint8_t |
| 201 |
_is_cca_fecn_set(const struct ips_message_header *p_hdr) |
| 202 |
{ |
| 203 |
return (__be32_to_cpu(p_hdr->bth[1]) >> BTH_FECN_SHIFT); |
| 204 |
} |
| 205 |
|
| 206 |
/* Detrmine if BECN bit is set IBTA 1.2.1 CCA Annex A*/ |
| 207 |
static __inline__ uint8_t |
| 208 |
_is_cca_becn_set(const struct ips_message_header *p_hdr) |
| 209 |
{ |
| 210 |
return (__be32_to_cpu(p_hdr->bth[1]) >> BTH_BECN_SHIFT) & 0x1; |
| 211 |
} |
| 212 |
|
| 213 |
static __inline__ struct ips_message_header * |
| 214 |
_get_proto_hdr_from_rhf(const uint32_t *rcv_hdr, const __le32 *rhf) |
| 215 |
{ |
| 216 |
return (struct ips_message_header *) (rcv_hdr + ipath_hdrget_offset(rhf)); |
| 217 |
} |
| 218 |
|
| 219 |
static __inline__ struct ips_message_header * |
| 220 |
_get_proto_hdr(const uint32_t *rcv_hdr) |
| 221 |
{ |
| 222 |
return (struct ips_message_header *) &rcv_hdr[2]; |
| 223 |
} |
| 224 |
|
| 225 |
static __inline__ uint32_t |
| 226 |
_get_rhf_seq(struct ips_recvhdrq *recvq, const __u32 *rcv_hdr) |
| 227 |
{ |
| 228 |
return ipath_hdrget_seq((const __le32 *) rcv_hdr + recvq->hdrq_rhf_off); |
| 229 |
} |
| 230 |
|
| 231 |
static __inline__ uint32_t |
| 232 |
_get_rhf_len_in_bytes(struct ips_recvhdrq *recvq, const __u32 *rcv_hdr) |
| 233 |
{ |
| 234 |
return ipath_hdrget_length_in_bytes((const __le32*) rcv_hdr + recvq->hdrq_rhf_off); |
| 235 |
} |
| 236 |
|
| 237 |
static __inline__ void |
| 238 |
_dump_invalid_pkt(struct ips_recvhdrq_event *rcv_ev) |
| 239 |
{ |
| 240 |
char *payload = ips_recvhdrq_event_payload(rcv_ev); |
| 241 |
uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev) + |
| 242 |
((__be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 20) & 3); |
| 243 |
|
| 244 |
if(infinipath_debug & __IPATH_PKTDBG) { |
| 245 |
ips_proto_dump_frame(rcv_ev->p_hdr, IPATH_MESSAGE_HDR_SIZE, "header"); |
| 246 |
if (paylen) |
| 247 |
ips_proto_dump_frame(payload, paylen, "data"); |
| 248 |
} |
| 249 |
|
| 250 |
} |
| 251 |
|
| 252 |
static __inline__ void |
| 253 |
_update_error_stats(struct ips_proto *proto, uint32_t err) |
| 254 |
{ |
| 255 |
|
| 256 |
if (err & INFINIPATH_RHF_H_ICRCERR) |
| 257 |
proto->error_stats.num_icrc_err++; |
| 258 |
if (err & INFINIPATH_RHF_H_VCRCERR) |
| 259 |
proto->error_stats.num_vcrc_err++; |
| 260 |
if (err & INFINIPATH_RHF_H_PARITYERR) |
| 261 |
proto->error_stats.num_ecc_err++; |
| 262 |
if (err & INFINIPATH_RHF_H_LENERR) |
| 263 |
proto->error_stats.num_len_err++; |
| 264 |
if (err & INFINIPATH_RHF_H_MTUERR) |
| 265 |
proto->error_stats.num_mtu_err++; |
| 266 |
if (err & INFINIPATH_RHF_H_IHDRERR) |
| 267 |
proto->error_stats.num_khdr_err++; |
| 268 |
if (err & INFINIPATH_RHF_H_TIDERR) |
| 269 |
proto->error_stats.num_tid_err++; |
| 270 |
if (err & INFINIPATH_RHF_H_MKERR) |
| 271 |
proto->error_stats.num_mk_err++; |
| 272 |
if (err & INFINIPATH_RHF_H_IBERR) |
| 273 |
proto->error_stats.num_ib_err++; |
| 274 |
} |
| 275 |
|
| 276 |
static int |
| 277 |
_check_headers(struct ips_recvhdrq_event *rcv_ev) |
| 278 |
{ |
| 279 |
struct ips_recvhdrq *recvq = (struct ips_recvhdrq*) rcv_ev->recvq; |
| 280 |
struct ips_proto *proto = rcv_ev->proto; |
| 281 |
uint32_t *lrh = (uint32_t*) rcv_ev->p_hdr; |
| 282 |
const uint32_t *rcv_hdr = rcv_ev->rcv_hdr; |
| 283 |
uint32_t dest_context; |
| 284 |
const uint16_t pkt_dlid = __be16_to_cpu(rcv_ev->p_hdr->lrh[1]); |
| 285 |
const uint16_t base_dlid = __be16_to_cpu(recvq->proto->epinfo.ep_base_lid); |
| 286 |
|
| 287 |
/* Check that the receive header queue entry has a sane sequence number */ |
| 288 |
if (_get_rhf_seq(recvq, rcv_hdr) > LAST_RHF_SEQNO) { |
| 289 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 290 |
"ErrPkt: Invalid header queue entry! RHF Sequence in Hdrq Seq: %d, Recvq State Seq: %d. LRH[0]: 0x%08x, LRH[1] (PktCount): 0x%08x\n", _get_rhf_seq(recvq, rcv_hdr), recvq->state->hdrq_rhf_seq, lrh[0], lrh[1]); |
| 291 |
return -1; |
| 292 |
} |
| 293 |
|
| 294 |
/* Verify that the packet was destined for our context */ |
| 295 |
dest_context = ips_proto_dest_context_from_header(proto, rcv_ev->p_hdr); |
| 296 |
if_pf (dest_context != recvq->proto->epinfo.ep_context) { |
| 297 |
|
| 298 |
struct ips_recvhdrq_state *state = recvq->state; |
| 299 |
|
| 300 |
/* Packet not targetted at us. Drop packet and continue */ |
| 301 |
ips_proto_dump_err_stats(proto); |
| 302 |
_dump_invalid_pkt(rcv_ev); |
| 303 |
|
| 304 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 305 |
"ErrPkt: Received packet for context %d on context %d. Receive Header Queue offset: 0x%x. Exiting.\n", dest_context, recvq->proto->epinfo.ep_context, state->hdrq_head); |
| 306 |
|
| 307 |
return -1; |
| 308 |
} |
| 309 |
|
| 310 |
|
| 311 |
if_pf (rcv_ev->error_flags || |
| 312 |
(_get_proto_ipath_opcode(rcv_ev->p_hdr) != IPATH_OPCODE_USER1)) { |
| 313 |
|
| 314 |
return 0; /* Error flags are special case. Let main receive loop handle |
| 315 |
* packet processing after we account for it. |
| 316 |
*/ |
| 317 |
} |
| 318 |
|
| 319 |
/* Verify that rhf packet length matches the length in LRH */ |
| 320 |
if_pf (_get_rhf_len_in_bytes(recvq, rcv_hdr) != |
| 321 |
(__be16_to_cpu(rcv_ev->p_hdr->lrh[2]) << 2)) { |
| 322 |
_IPATH_EPDBG("ErrPkt: RHF Packet Len (0x%x) does not match LRH (0x%x).\n", _get_rhf_len_in_bytes(recvq, rcv_hdr) >> 2, __be16_to_cpu(rcv_ev->p_hdr->lrh[2])); |
| 323 |
|
| 324 |
ips_proto_dump_err_stats(proto); |
| 325 |
_dump_invalid_pkt(rcv_ev); |
| 326 |
return -1; |
| 327 |
} |
| 328 |
|
| 329 |
/* Verify that the DLID matches our local LID. */ |
| 330 |
if_pf (!((base_dlid <= pkt_dlid) && |
| 331 |
(pkt_dlid <= (base_dlid + (1 << recvq->proto->epinfo.ep_lmc))))) { |
| 332 |
_IPATH_EPDBG("ErrPkt: DLID in LRH (0x%04x) does not match local LID (0x%04x) Skipping packet!\n", rcv_ev->p_hdr->lrh[1], recvq->proto->epinfo.ep_base_lid); |
| 333 |
ips_proto_dump_err_stats(proto); |
| 334 |
_dump_invalid_pkt(rcv_ev); |
| 335 |
return -1; |
| 336 |
} |
| 337 |
|
| 338 |
return 0; |
| 339 |
} |
| 340 |
|
| 341 |
static __inline__ |
| 342 |
int |
| 343 |
do_pkt_cksum(struct ips_recvhdrq_event *rcv_ev) |
| 344 |
{ |
| 345 |
char *payload = ips_recvhdrq_event_payload(rcv_ev); |
| 346 |
uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev) + |
| 347 |
((__be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 20) & 3); |
| 348 |
uint32_t *ckptr; |
| 349 |
uint32_t recv_cksum, cksum, dest_subcontext; |
| 350 |
|
| 351 |
/* With checksum every packet has a payload */ |
| 352 |
psmi_assert_always(payload); |
| 353 |
|
| 354 |
ckptr = (uint32_t*) (payload + paylen); |
| 355 |
recv_cksum = ckptr[0]; |
| 356 |
|
| 357 |
/* Calculate checksum hdr + payload (includes any padding words) */ |
| 358 |
cksum = 0xffffffff; |
| 359 |
cksum = ips_crc_calculate(IPATH_MESSAGE_HDR_SIZE, |
| 360 |
(uint8_t*) rcv_ev->p_hdr,cksum); |
| 361 |
if (paylen) |
| 362 |
cksum = ips_crc_calculate(paylen, (uint8_t*) payload, cksum); |
| 363 |
|
| 364 |
if ((cksum != recv_cksum) || (ckptr[0] != ckptr[1])) { |
| 365 |
struct ips_epstate_entry *epstaddr; |
| 366 |
uint32_t lcontext; |
| 367 |
uint32_t hd, tl; |
| 368 |
|
| 369 |
epstaddr = |
| 370 |
ips_epstate_lookup(rcv_ev->recvq->epstate, rcv_ev->p_hdr->commidx + |
| 371 |
INFINIPATH_KPF_RESERVED_BITS(rcv_ev->p_hdr->iph.pkt_flags)); |
| 372 |
epstaddr = (epstaddr && epstaddr->ipsaddr) ? epstaddr : NULL; |
| 373 |
|
| 374 |
lcontext = |
| 375 |
epstaddr ? epstaddr->ipsaddr->proto->epinfo.ep_context : -1; |
| 376 |
|
| 377 |
hd = rcv_ev->recvq->context->ctrl->__ipath_rcvhdrhead[0]; |
| 378 |
tl = rcv_ev->recvq->context->ctrl->__ipath_rcvhdrhead[-2]; |
| 379 |
|
| 380 |
dest_subcontext = _get_proto_subcontext(rcv_ev->p_hdr); |
| 381 |
|
| 382 |
_IPATH_ERROR("ErrPkt: SharedContext: %s. Local Context: %i, Checksum mismatch from LID %d! Received Checksum: 0x%08x, Expected: 0x%08x & 0x%08x. Opcode: 0x%08x, Error Flag: 0x%08x. hdrq hd 0x%x tl 0x%x rhf 0x%x,%x, rhfseq 0x%x\n", (dest_subcontext != rcv_ev->recvq->subcontext) ? "Yes" : "No", lcontext, epstaddr ? __be16_to_cpu(epstaddr->ipsaddr->epr.epr_base_lid) : -1, cksum, ckptr[0], ckptr[1], rcv_ev->p_hdr->sub_opcode, rcv_ev->error_flags,hd, tl, rcv_ev->rhf[0], rcv_ev->rhf[1], _get_rhf_seq((struct ips_recvhdrq *) rcv_ev->recvq, rcv_ev->rcv_hdr)); |
| 383 |
|
| 384 |
/* Dump packet */ |
| 385 |
_dump_invalid_pkt(rcv_ev); |
| 386 |
return 0; /* Packet checksum error */ |
| 387 |
} |
| 388 |
|
| 389 |
return 1; |
| 390 |
} |
| 391 |
|
| 392 |
PSMI_ALWAYS_INLINE( |
| 393 |
void |
| 394 |
process_pending_acks(struct ips_recvhdrq *recvq)) |
| 395 |
{ |
| 396 |
/* If any pending acks, dispatch them now */ |
| 397 |
while (!SLIST_EMPTY(&recvq->pending_acks)) { |
| 398 |
struct ips_flow *flow = SLIST_FIRST(&recvq->pending_acks); |
| 399 |
|
| 400 |
SLIST_REMOVE_HEAD(&recvq->pending_acks, next); |
| 401 |
SLIST_NEXT(flow, next) = NULL; |
| 402 |
|
| 403 |
if (flow->flags & IPS_FLOW_FLAG_PENDING_ACK) { |
| 404 |
psmi_assert_always((flow->flags & IPS_FLOW_FLAG_PENDING_NAK) == 0); |
| 405 |
|
| 406 |
flow->flags &= ~IPS_FLOW_FLAG_PENDING_ACK; |
| 407 |
ips_proto_send_ctrl_message(flow, OPCODE_ACK, |
| 408 |
&flow->ipsaddr->ctrl_msg_queued, NULL); |
| 409 |
} |
| 410 |
else { |
| 411 |
psmi_assert_always(flow->flags & IPS_FLOW_FLAG_PENDING_NAK); |
| 412 |
|
| 413 |
flow->flags &= ~IPS_FLOW_FLAG_PENDING_NAK; |
| 414 |
ips_proto_send_ctrl_message(flow, OPCODE_NAK, |
| 415 |
&flow->ipsaddr->ctrl_msg_queued, NULL); |
| 416 |
} |
| 417 |
|
| 418 |
} |
| 419 |
|
| 420 |
} |
| 421 |
|
| 422 |
/* |
| 423 |
* Core receive progress function |
| 424 |
* |
| 425 |
* recvhdrq_progress is the core function that services the receive header |
| 426 |
* queue and optionally, the eager queue. At the lowest level, it identifies |
| 427 |
* packets marked with errors by the chip and also detects and corrects when |
| 428 |
* eager overflow conditions occur. At the highest level, it queries the |
| 429 |
* 'epstate' interface to classify packets from "known" and "unknown" |
| 430 |
* endpoints. In order to support shared contexts, it can also handle packets |
| 431 |
* destined for other contexts (or "subcontexts"). |
| 432 |
*/ |
| 433 |
|
| 434 |
#if IPS_RCVHDRQ_THRU_FUNCTION_POINTER |
| 435 |
PSMI_ALWAYS_INLINE( |
| 436 |
psm_error_t |
| 437 |
ips_recvhdrq_progress_inner(struct ips_recvhdrq *recvq, |
| 438 |
const int has_no_rtail)) |
| 439 |
#else |
| 440 |
psm_error_t __recvpath |
| 441 |
ips_recvhdrq_progress(struct ips_recvhdrq *recvq) |
| 442 |
#endif |
| 443 |
{ |
| 444 |
struct ips_recvhdrq_state *state = recvq->state; |
| 445 |
const __le32 *rhf; |
| 446 |
PSMI_CACHEALIGN struct ips_recvhdrq_event rcv_ev = { .proto = recvq->proto, |
| 447 |
.recvq = recvq }; |
| 448 |
|
| 449 |
uint32_t num_hdrq_done = 0; |
| 450 |
const int num_hdrq_todo = recvq->hdrq.elemcnt; |
| 451 |
const uint32_t hdrq_elemsz = recvq->hdrq.elemsz; |
| 452 |
uint32_t dest_subcontext; |
| 453 |
|
| 454 |
int ret = IPS_RECVHDRQ_CONTINUE; |
| 455 |
int done = 0; |
| 456 |
int do_hdr_update = 0; |
| 457 |
const uint16_t lmc_mask = ~((1 << recvq->proto->epinfo.ep_lmc) - 1); |
| 458 |
|
| 459 |
/* Chip features */ |
| 460 |
#if !IPS_RCVHDRQ_THRU_FUNCTION_POINTER |
| 461 |
const int has_no_rtail = recvq->runtime_flags & IPATH_RUNTIME_NODMA_RTAIL; |
| 462 |
#endif |
| 463 |
|
| 464 |
/* Both optional_eager and no_rtail features are in the same chip rev */ |
| 465 |
#define has_optional_eagerbuf recvq->hdrq_rhf_off |
| 466 |
|
| 467 |
/* Returns whether the currently set 'rcv_hdr'/head is a readable entry */ |
| 468 |
#define next_hdrq_is_ready() \ |
| 469 |
(has_no_rtail ? \ |
| 470 |
recvq->state->hdrq_rhf_seq == _get_rhf_seq(recvq, rcv_hdr) \ |
| 471 |
: state->hdrq_head != hdrq_tail) |
| 472 |
|
| 473 |
const uint32_t hdrq_tail = has_no_rtail ? 0 |
| 474 |
: ips_recvq_tail_get(&recvq->hdrq); |
| 475 |
const uint32_t *rcv_hdr = |
| 476 |
(const uint32_t *) recvq->hdrq.base_addr + state->hdrq_head; |
| 477 |
uint32_t tmp_hdrq_head; |
| 478 |
|
| 479 |
done = !next_hdrq_is_ready(); |
| 480 |
|
| 481 |
while (!done) |
| 482 |
{ |
| 483 |
|
| 484 |
rhf = (const __le32 *) rcv_hdr + recvq->hdrq_rhf_off; |
| 485 |
rcv_ev.error_flags = ipath_hdrget_err_flags(rhf); |
| 486 |
rcv_ev.ptype = ipath_hdrget_rcv_type(rhf); |
| 487 |
rcv_ev.rhf = rhf; |
| 488 |
rcv_ev.rcv_hdr= rcv_hdr; |
| 489 |
rcv_ev.p_hdr = recvq->hdrq_rhf_off ? _get_proto_hdr_from_rhf(rcv_hdr, rhf) |
| 490 |
: _get_proto_hdr(rcv_hdr); |
| 491 |
rcv_ev.epid = ips_epid_from_phdr(lmc_mask, rcv_ev.p_hdr); |
| 492 |
rcv_ev.has_cksum = |
| 493 |
((recvq->proto->flags & IPS_PROTO_FLAG_CKSUM) && |
| 494 |
(rcv_ev.ptype == RCVHQ_RCV_TYPE_EAGER) && |
| 495 |
(rcv_ev.p_hdr->mqhdr != MQ_MSG_DATA_BLK) && |
| 496 |
(rcv_ev.p_hdr->mqhdr != MQ_MSG_DATA_REQ_BLK)); |
| 497 |
|
| 498 |
if_pt (recvq->proto->flags & IPS_PROTO_FLAG_CCA) { |
| 499 |
/* IBTA CCA handling: |
| 500 |
* If FECN bit set handle IBTA CCA protocol. For the flow that |
| 501 |
* suffered congestion we flag it to generate a control packet with |
| 502 |
* the BECN bit set - This is currently an unsolicited ACK. |
| 503 |
* |
| 504 |
* For all MQ packets the FECN processing/BECN generation is done |
| 505 |
* in the is_expected_or_nak function as each eager packet is |
| 506 |
* inspected there. |
| 507 |
* |
| 508 |
* For TIDFLOW/Expected data transfers the FECN bit/BECN generation |
| 509 |
* is done in protoexp_data. Since header suppression can result |
| 510 |
* in even FECN packets being suppressed the expected protocol |
| 511 |
* generated addiional BECN packets if a "large" number of generations |
| 512 |
* are swapped without progress being made for receive. "Large" is |
| 513 |
* set empirically to 4. |
| 514 |
* |
| 515 |
* FECN packets are ignored for all control messages (except ACKs |
| 516 |
* and NAKs) since they indicate congestion on the control path which |
| 517 |
* is not rate controlled. The CCA specification allows FECN on |
| 518 |
* ACKs to be disregarded as well. |
| 519 |
*/ |
| 520 |
rcv_ev.is_congested = |
| 521 |
_is_cca_fecn_set(rcv_ev.p_hdr) & IPS_RECV_EVENT_FECN; |
| 522 |
rcv_ev.is_congested |= |
| 523 |
(_is_cca_becn_set(rcv_ev.p_hdr) << (IPS_RECV_EVENT_BECN - 1)); |
| 524 |
} |
| 525 |
else |
| 526 |
rcv_ev.is_congested = 0; |
| 527 |
|
| 528 |
dest_subcontext = _get_proto_subcontext(rcv_ev.p_hdr); |
| 529 |
|
| 530 |
if_pf (_check_headers(&rcv_ev)) |
| 531 |
goto skip_packet; |
| 532 |
|
| 533 |
if_pf (rcv_ev.error_flags || |
| 534 |
(_get_proto_ipath_opcode(rcv_ev.p_hdr) != IPATH_OPCODE_USER1)) |
| 535 |
{ |
| 536 |
|
| 537 |
_update_error_stats(recvq->proto, rcv_ev.error_flags); |
| 538 |
|
| 539 |
if ((rcv_ev.error_flags & INFINIPATH_RHF_H_TIDERR) || |
| 540 |
(rcv_ev.error_flags & INFINIPATH_RHF_H_TFSEQERR) || |
| 541 |
(rcv_ev.error_flags & INFINIPATH_RHF_H_TFGENERR)) { |
| 542 |
/* Subcontexts need to see expected tid errors */ |
| 543 |
if (rcv_ev.ptype == RCVHQ_RCV_TYPE_EXPECTED && |
| 544 |
dest_subcontext != recvq->subcontext) |
| 545 |
goto subcontext_packet; |
| 546 |
|
| 547 |
recvq->recvq_callbacks.callback_error(&rcv_ev); |
| 548 |
|
| 549 |
if (rcv_ev.ptype == RCVHQ_RCV_TYPE_EAGER) { |
| 550 |
/* tiderr and eager, don't consider updating egr head */ |
| 551 |
if (state->hdr_countdown == 0 && |
| 552 |
state->rcv_egr_index_head == NO_EAGER_UPDATE) { |
| 553 |
/* eager-full is not currently under tracing. */ |
| 554 |
uint32_t egr_cnt = recvq->egrq.elemcnt; |
| 555 |
const uint32_t etail = ips_recvq_tail_get(&recvq->egrq); |
| 556 |
const uint32_t ehead = ips_recvq_head_get(&recvq->egrq); |
| 557 |
|
| 558 |
if (ehead == ((etail+1)%egr_cnt)) { |
| 559 |
/* eager is full, trace existing header entries */ |
| 560 |
uint32_t hdr_size = recvq->hdrq_elemlast + hdrq_elemsz; |
| 561 |
const uint32_t htail = ips_recvq_tail_get(&recvq->hdrq); |
| 562 |
const uint32_t hhead = state->hdrq_head; |
| 563 |
|
| 564 |
state->hdr_countdown = (htail > hhead) ? |
| 565 |
(htail - hhead) : (htail + hdr_size - hhead); |
| 566 |
} |
| 567 |
} |
| 568 |
goto skip_packet_no_egr_update; |
| 569 |
} |
| 570 |
} |
| 571 |
else |
| 572 |
recvq->recvq_callbacks.callback_error(&rcv_ev); |
| 573 |
goto skip_packet; |
| 574 |
} |
| 575 |
|
| 576 |
/* If checksum is enabled, verify that it is valid */ |
| 577 |
if_pf (rcv_ev.has_cksum && !do_pkt_cksum(&rcv_ev)) |
| 578 |
goto skip_packet; |
| 579 |
|
| 580 |
if (dest_subcontext == recvq->subcontext) { |
| 581 |
/* Classify packet from a known or unknown endpoint */ |
| 582 |
struct ips_epstate_entry *epstaddr; |
| 583 |
|
| 584 |
epstaddr = |
| 585 |
ips_epstate_lookup(recvq->epstate, rcv_ev.p_hdr->commidx + |
| 586 |
INFINIPATH_KPF_RESERVED_BITS(rcv_ev.p_hdr->iph.pkt_flags)); |
| 587 |
if_pf (epstaddr == NULL || epstaddr->epid != rcv_ev.epid) { |
| 588 |
rcv_ev.ipsaddr = NULL; |
| 589 |
recvq->recvq_callbacks.callback_packet_unknown(&rcv_ev); |
| 590 |
} |
| 591 |
else { |
| 592 |
rcv_ev.ipsaddr = epstaddr->ipsaddr; |
| 593 |
ret = ips_proto_process_packet(&rcv_ev); |
| 594 |
if (ret == IPS_RECVHDRQ_OOO) return PSM_OK_NO_PROGRESS; |
| 595 |
} |
| 596 |
} |
| 597 |
else { |
| 598 |
subcontext_packet: |
| 599 |
/* If the destination is not our subcontext, process message |
| 600 |
* as a subcontext message (shared contexts) */ |
| 601 |
rcv_ev.ipsaddr = NULL; |
| 602 |
|
| 603 |
ret = recvq->recvq_callbacks.callback_subcontext(&rcv_ev, |
| 604 |
dest_subcontext); |
| 605 |
} |
| 606 |
|
| 607 |
skip_packet: |
| 608 |
/* |
| 609 |
* important to update rcv_egr_index_head iff |
| 610 |
* 1. Packet was of type eager |
| 611 |
* 2. Packet actually consumed an eagerbuf (post QLE72XX) |
| 612 |
* 3. Packet was *not* an eager header with RHF_H_TIDERR to mark |
| 613 |
* an eager overflow |
| 614 |
*/ |
| 615 |
if (has_optional_eagerbuf ? ipath_hdrget_use_egr_buf(rhf) |
| 616 |
: (rcv_ev.ptype == RCVHQ_RCV_TYPE_EAGER)) { |
| 617 |
state->rcv_egr_index_head = ipath_hdrget_index(rhf); |
| 618 |
/* a header entry is using an eager entry, stop tracing. */ |
| 619 |
state->hdr_countdown = 0; |
| 620 |
} |
| 621 |
|
| 622 |
skip_packet_no_egr_update: |
| 623 |
/* Note that state->hdrq_head is sampled speculatively by the code |
| 624 |
* in ips_ptl_shared_poll() when context sharing, so it is not safe |
| 625 |
* for this shared variable to temporarily exceed the last element. */ |
| 626 |
tmp_hdrq_head = state->hdrq_head + hdrq_elemsz; |
| 627 |
if_pt (tmp_hdrq_head <= recvq->hdrq_elemlast) |
| 628 |
state->hdrq_head = tmp_hdrq_head; |
| 629 |
else |
| 630 |
state->hdrq_head = 0; |
| 631 |
|
| 632 |
if_pf (has_no_rtail && ++recvq->state->hdrq_rhf_seq > LAST_RHF_SEQNO) |
| 633 |
recvq->state->hdrq_rhf_seq = 1; |
| 634 |
|
| 635 |
state->num_hdrq_done++; |
| 636 |
num_hdrq_done++; |
| 637 |
rcv_hdr = (const uint32_t *) recvq->hdrq.base_addr + state->hdrq_head; |
| 638 |
done = (!next_hdrq_is_ready() || (ret == IPS_RECVHDRQ_BREAK) || |
| 639 |
(num_hdrq_done == num_hdrq_todo)); |
| 640 |
|
| 641 |
do_hdr_update = (state->head_update_interval ? |
| 642 |
(state->num_hdrq_done == state->head_update_interval) : done); |
| 643 |
if (do_hdr_update) { |
| 644 |
ips_recvq_head_update(&recvq->hdrq, state->hdrq_head); |
| 645 |
|
| 646 |
/* Lazy update of egrq */ |
| 647 |
if (state->rcv_egr_index_head != NO_EAGER_UPDATE) { |
| 648 |
ips_recvq_head_update(&recvq->egrq, state->rcv_egr_index_head); |
| 649 |
state->rcv_egr_index_head = NO_EAGER_UPDATE; |
| 650 |
} |
| 651 |
|
| 652 |
/* Process any pending acks while updated eager/headq queue */ |
| 653 |
process_pending_acks(recvq); |
| 654 |
|
| 655 |
/* Reset header queue entries processed */ |
| 656 |
state->num_hdrq_done = 0; |
| 657 |
} |
| 658 |
|
| 659 |
if (state->hdr_countdown > 0) { |
| 660 |
/* a header entry is consumed. */ |
| 661 |
state->hdr_countdown -= hdrq_elemsz; |
| 662 |
if (state->hdr_countdown == 0) { |
| 663 |
/* header entry count reaches zero. */ |
| 664 |
const uint32_t tail = ips_recvq_tail_get(&recvq->egrq); |
| 665 |
const uint32_t head = ips_recvq_head_get(&recvq->egrq); |
| 666 |
uint32_t egr_cnt = recvq->egrq.elemcnt; |
| 667 |
|
| 668 |
/* Checks eager-full again. This is a real false-egr-full */ |
| 669 |
if (head == ((tail+1)%egr_cnt)) { |
| 670 |
ips_recvq_head_update(&recvq->egrq, tail); |
| 671 |
_IPATH_DBG("eager array full after overflow, flushing " |
| 672 |
"(head %llx, tail %llx)\n", |
| 673 |
(long long)head, (long long)tail); |
| 674 |
recvq->proto->stats.egr_overflow++; |
| 675 |
} else |
| 676 |
_IPATH_ERROR("PSM BUG: EgrOverflow: eager queue is not full\n"); |
| 677 |
} |
| 678 |
} |
| 679 |
} |
| 680 |
/* while (hdrq_entries_to_read) */ |
| 681 |
|
| 682 |
/* Process any pending acks before exiting */ |
| 683 |
process_pending_acks(recvq); |
| 684 |
|
| 685 |
return num_hdrq_done ? PSM_OK : PSM_OK_NO_PROGRESS; |
| 686 |
} |
| 687 |
|
| 688 |
#if IPS_RCVHDRQ_THRU_FUNCTION_POINTER |
| 689 |
/* |
| 690 |
* QLE71XX |
| 691 |
*/ |
| 692 |
static |
| 693 |
psm_error_t __recvpath |
| 694 |
ips_recvhdrq_progress_none(struct ips_recvhdrq *recvq) |
| 695 |
{ |
| 696 |
const int has_no_rtail = 0; |
| 697 |
return ips_recvhdrq_progress_inner(recvq, has_no_rtail); |
| 698 |
} |
| 699 |
|
| 700 |
/* |
| 701 |
* QLE72XX+ |
| 702 |
*/ |
| 703 |
static |
| 704 |
psm_error_t __recvpath |
| 705 |
ips_recvhdrq_progress_nortail(struct ips_recvhdrq *recvq) |
| 706 |
{ |
| 707 |
const int has_no_rtail = 1; |
| 708 |
return ips_recvhdrq_progress_inner(recvq, has_no_rtail); |
| 709 |
} |
| 710 |
|
| 711 |
psm_error_t __recvpath |
| 712 |
ips_recvhdrq_progress(struct ips_recvhdrq *recvq) |
| 713 |
{ |
| 714 |
/* Call the progress function with the right chip features. */ |
| 715 |
return recvq->progress_fn(recvq); |
| 716 |
} |
| 717 |
#endif |