| 1 |
/* |
| 2 |
* Copyright (c) 2013. Intel Corporation. All rights reserved. |
| 3 |
* Copyright (c) 2006-2012. QLogic Corporation. All rights reserved. |
| 4 |
* Copyright (c) 2003-2006, PathScale, Inc. All rights reserved. |
| 5 |
* |
| 6 |
* This software is available to you under a choice of one of two |
| 7 |
* licenses. You may choose to be licensed under the terms of the GNU |
| 8 |
* General Public License (GPL) Version 2, available from the file |
| 9 |
* COPYING in the main directory of this source tree, or the |
| 10 |
* OpenIB.org BSD license below: |
| 11 |
* |
| 12 |
* Redistribution and use in source and binary forms, with or |
| 13 |
* without modification, are permitted provided that the following |
| 14 |
* conditions are met: |
| 15 |
* |
| 16 |
* - Redistributions of source code must retain the above |
| 17 |
* copyright notice, this list of conditions and the following |
| 18 |
* disclaimer. |
| 19 |
* |
| 20 |
* - Redistributions in binary form must reproduce the above |
| 21 |
* copyright notice, this list of conditions and the following |
| 22 |
* disclaimer in the documentation and/or other materials |
| 23 |
* provided with the distribution. |
| 24 |
* |
| 25 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 26 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 27 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 28 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 29 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 30 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 31 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 32 |
* SOFTWARE. |
| 33 |
*/ |
| 34 |
|
| 35 |
#include "psm_user.h" |
| 36 |
#include "ipserror.h" |
| 37 |
#include "ips_proto.h" |
| 38 |
#include "ips_proto_internal.h" |
| 39 |
|
| 40 |
#define PSM_STRAY_WARN_INTERVAL_DEFAULT_SECS 30 |
| 41 |
static void ips_report_strays(struct ips_proto *proto); |
| 42 |
|
| 43 |
#define INC_TIME_SPEND(timer) |
| 44 |
|
| 45 |
#define _desc_idx u32w0 |
| 46 |
#define _desc_genc u32w1 |
| 47 |
|
| 48 |
psm_error_t |
| 49 |
ips_proto_recv_init(struct ips_proto *proto) |
| 50 |
{ |
| 51 |
uint32_t interval_secs; |
| 52 |
union psmi_envvar_val env_stray; |
| 53 |
|
| 54 |
psmi_getenv("PSM_STRAY_WARNINTERVAL", |
| 55 |
"min secs between stray process warnings", |
| 56 |
PSMI_ENVVAR_LEVEL_HIDDEN, |
| 57 |
PSMI_ENVVAR_TYPE_UINT, |
| 58 |
(union psmi_envvar_val) PSM_STRAY_WARN_INTERVAL_DEFAULT_SECS, |
| 59 |
&env_stray); |
| 60 |
interval_secs = env_stray.e_uint; |
| 61 |
if (interval_secs > 0) |
| 62 |
proto->stray_warn_interval = sec_2_cycles(interval_secs); |
| 63 |
else |
| 64 |
proto->stray_warn_interval = 0; |
| 65 |
|
| 66 |
return PSM_OK; |
| 67 |
} |
| 68 |
|
| 69 |
psm_error_t |
| 70 |
ips_proto_recv_fini(struct ips_proto *proto) |
| 71 |
{ |
| 72 |
ips_report_strays(proto); |
| 73 |
return PSM_OK; |
| 74 |
} |
| 75 |
|
| 76 |
#define cycles_to_sec_f(cycles) \ |
| 77 |
(((double)cycles_to_nanosecs(cycles)) / 1000000000.0) |
| 78 |
|
| 79 |
struct ips_stray_epid { |
| 80 |
psm_epid_t epid; |
| 81 |
uint32_t err_check_bad_sent; |
| 82 |
uint32_t ipv4_addr; |
| 83 |
uint32_t pid; |
| 84 |
uint32_t num_messages; |
| 85 |
uint64_t t_warn_next; |
| 86 |
uint64_t t_first; |
| 87 |
uint64_t t_last; |
| 88 |
}; |
| 89 |
|
| 90 |
static |
| 91 |
void |
| 92 |
ips_report_strays(struct ips_proto *proto) |
| 93 |
{ |
| 94 |
struct ips_stray_epid *sepid; |
| 95 |
struct psmi_eptab_iterator itor; |
| 96 |
psmi_epid_itor_init(&itor, PSMI_EP_CROSSTALK); |
| 97 |
double t_runtime = cycles_to_sec_f(proto->t_fini - proto->t_init); |
| 98 |
|
| 99 |
while ((sepid = psmi_epid_itor_next(&itor))) { |
| 100 |
char ipbuf[INET_ADDRSTRLEN], *ip = NULL; |
| 101 |
char bufpid[32]; |
| 102 |
uint32_t lid = psm_epid_nid(sepid->epid); |
| 103 |
double t_first = cycles_to_sec_f(sepid->t_first - proto->t_init); |
| 104 |
double t_last = cycles_to_sec_f(sepid->t_last - proto->t_init); |
| 105 |
if (sepid->ipv4_addr) |
| 106 |
ip = (char *) |
| 107 |
inet_ntop(AF_INET, &sepid->ipv4_addr, ipbuf, sizeof ipbuf); |
| 108 |
if (!ip) |
| 109 |
snprintf(ipbuf, sizeof ipbuf, "%d (%x)", lid, lid); |
| 110 |
|
| 111 |
if (sepid->pid) |
| 112 |
snprintf(bufpid, sizeof bufpid, "PID=%d", sepid->pid); |
| 113 |
else |
| 114 |
snprintf(bufpid, sizeof bufpid, "PID unknown"); |
| 115 |
|
| 116 |
_IPATH_INFO("Process %s on host %s=%s sent %d stray message(s) and " |
| 117 |
"was told so %d time(s) (first stray message at %.1fs " |
| 118 |
"(%d%%), last at %.1fs (%d%%) into application run)\n", |
| 119 |
bufpid, ip ? "IP" : "LID", ipbuf, sepid->num_messages, |
| 120 |
sepid->err_check_bad_sent, t_first, |
| 121 |
(int) (t_first * 100.0 / t_runtime), t_last, |
| 122 |
(int) (t_last * 100.0 / t_runtime)); |
| 123 |
|
| 124 |
psmi_epid_remove(PSMI_EP_CROSSTALK, sepid->epid); |
| 125 |
psmi_free(sepid); |
| 126 |
} |
| 127 |
psmi_epid_itor_fini(&itor); |
| 128 |
return; |
| 129 |
} |
| 130 |
|
| 131 |
/* New scbs now available. If we have pending sends because we were out of |
| 132 |
* scbs, put the pendq on the timerq so it can be processed. */ |
| 133 |
void |
| 134 |
ips_proto_rv_scbavail_callback(struct ips_scbctrl *scbc, void *context) |
| 135 |
{ |
| 136 |
struct ips_proto *proto = (struct ips_proto *) context; |
| 137 |
struct ips_pend_sreq *sreq = STAILQ_FIRST(&proto->pend_sends.pendq); |
| 138 |
if (sreq != NULL) |
| 139 |
psmi_timer_request(proto->timerq, |
| 140 |
&proto->pend_sends.timer, PSMI_TIMER_PRIO_1); |
| 141 |
return; |
| 142 |
} |
| 143 |
|
| 144 |
psm_error_t __recvpath |
| 145 |
ips_proto_timer_pendq_callback(struct psmi_timer *timer, uint64_t current) |
| 146 |
{ |
| 147 |
psm_error_t err = PSM_OK; |
| 148 |
struct ips_pend_sends *pend_sends = |
| 149 |
(struct ips_pend_sends *) timer->context; |
| 150 |
struct ips_pendsendq *phead = &pend_sends->pendq; |
| 151 |
struct ips_proto *proto = (struct ips_proto *) pend_sends->proto; |
| 152 |
struct ips_pend_sreq *sreq; |
| 153 |
|
| 154 |
while (!STAILQ_EMPTY(phead)) { |
| 155 |
sreq = STAILQ_FIRST(phead); |
| 156 |
switch (sreq->type) { |
| 157 |
case IPS_PENDSEND_EAGER_REQ: |
| 158 |
err = ips_proto_mq_push_eager_req(proto, sreq->req); |
| 159 |
break; |
| 160 |
case IPS_PENDSEND_EAGER_DATA: |
| 161 |
err = ips_proto_mq_push_eager_data(proto, sreq->req); |
| 162 |
break; |
| 163 |
|
| 164 |
default: |
| 165 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 166 |
"Unknown pendq state %d\n", sreq->type); |
| 167 |
} |
| 168 |
|
| 169 |
if (err == PSM_OK) { |
| 170 |
STAILQ_REMOVE_HEAD(phead, next); |
| 171 |
psmi_mpool_put(sreq); |
| 172 |
} |
| 173 |
else { /* out of scbs. wait for the next scb_avail callback */ |
| 174 |
//printf("!!!!! breaking out of pendq progress\n"); |
| 175 |
break; |
| 176 |
} |
| 177 |
} |
| 178 |
|
| 179 |
return err; |
| 180 |
} |
| 181 |
|
| 182 |
static |
| 183 |
int __recvpath |
| 184 |
_process_mq(struct ips_recvhdrq_event *rcv_ev) |
| 185 |
{ |
| 186 |
struct ips_message_header *p_hdr = rcv_ev->p_hdr; |
| 187 |
char *payload = ips_recvhdrq_event_payload(rcv_ev); |
| 188 |
uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev); |
| 189 |
uint32_t msglen = paylen; |
| 190 |
uint16_t mode = p_hdr->mqhdr; |
| 191 |
psm_mq_req_t req; |
| 192 |
psmi_egrid_t egrid; |
| 193 |
ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr; |
| 194 |
psm_epaddr_t epaddr = ipsaddr->epaddr; |
| 195 |
psm_mq_t mq = rcv_ev->proto->mq; |
| 196 |
ptl_arg_t *args; |
| 197 |
ptl_epaddr_flow_t flowid = ips_proto_flowid(p_hdr); |
| 198 |
struct ips_flow *flow = &ipsaddr->flows[flowid]; |
| 199 |
int ret = IPS_RECVHDRQ_CONTINUE; |
| 200 |
|
| 201 |
if (!ips_proto_is_expected_or_nak((struct ips_recvhdrq_event*) rcv_ev)) |
| 202 |
goto skip_ack_req; |
| 203 |
|
| 204 |
_IPATH_VDBG("Rcvd ctrl packet %s length = %i, mode=%d, arg0=%llx arg1=%llx\n", |
| 205 |
psmi_epaddr_get_name(epaddr->epid), |
| 206 |
paylen, p_hdr->mqhdr, |
| 207 |
(long long) p_hdr->data[0].u64, (long long) p_hdr->data[1].u64); |
| 208 |
|
| 209 |
if (mode <= MQ_MSG_RTS_WAIT) { |
| 210 |
ret = ips_proto_check_msg_order(epaddr, flow, p_hdr); |
| 211 |
if (ret == 0) return IPS_RECVHDRQ_OOO; |
| 212 |
|
| 213 |
if (mode <= MQ_MSG_LONG) { |
| 214 |
egrid.egr_data = 0; |
| 215 |
if (mode == MQ_MSG_SHORT) { |
| 216 |
/* May have padded writes, account for it */ |
| 217 |
paylen -= p_hdr->hdr_dlen; |
| 218 |
msglen = paylen; |
| 219 |
} |
| 220 |
else if (mode == MQ_MSG_TINY) { |
| 221 |
payload = (void *) &p_hdr->data[1]; |
| 222 |
msglen = paylen = p_hdr->hdr_dlen; |
| 223 |
} |
| 224 |
else if (mode == MQ_MSG_LONG) { |
| 225 |
msglen = p_hdr->data[1].u32w1; |
| 226 |
if (ipsaddr->flags & SESS_FLAG_HAS_FLOWID) { |
| 227 |
egrid.egr_data = p_hdr->data[1].u32w0; |
| 228 |
_IPATH_VDBG("egrid-msglong is 0x%x\n", egrid.egr_data); |
| 229 |
} |
| 230 |
} |
| 231 |
|
| 232 |
if (ret == 1) |
| 233 |
psmi_mq_handle_envelope( |
| 234 |
mq, mode, epaddr, p_hdr->data[0].u64, /* tag */ |
| 235 |
egrid, msglen, (void *) payload, paylen); |
| 236 |
else |
| 237 |
psmi_mq_handle_envelope_outoforder( |
| 238 |
mq, mode, epaddr, flow->msg_ooo_seqnum, |
| 239 |
p_hdr->data[0].u64, /* tag */ |
| 240 |
egrid, msglen, (void *) payload, paylen); |
| 241 |
} else { |
| 242 |
args = (ptl_arg_t *) p_hdr->data; |
| 243 |
if (ret == 1) |
| 244 |
ips_proto_mq_handle_rts_envelope(mq, mode, epaddr, |
| 245 |
args[0].u64, args[1].u32w0, args[1].u32w1); |
| 246 |
else |
| 247 |
ips_proto_mq_handle_rts_envelope_outoforder(mq, mode, |
| 248 |
epaddr, flow->msg_ooo_seqnum, |
| 249 |
args[0].u64, args[1].u32w0, args[1].u32w1); |
| 250 |
} |
| 251 |
|
| 252 |
if (ret == 1) { |
| 253 |
if (epaddr->mctxt_master->outoforder_c) { |
| 254 |
psmi_mq_handle_outoforder_queue(epaddr->mctxt_master); |
| 255 |
} |
| 256 |
ret = IPS_RECVHDRQ_CONTINUE; |
| 257 |
} else { |
| 258 |
ret = IPS_RECVHDRQ_BREAK; |
| 259 |
} |
| 260 |
} else if (mode == MQ_MSG_DATA || mode == MQ_MSG_DATA_BLK) { |
| 261 |
psm_mq_req_t req; |
| 262 |
|
| 263 |
req = STAILQ_FIRST(&epaddr->mctxt_master->egrlong); |
| 264 |
while (req) { |
| 265 |
if (req->egrid.egr_data == p_hdr->data[1].u32w0) break; |
| 266 |
req = STAILQ_NEXT(req, nextq); |
| 267 |
} |
| 268 |
|
| 269 |
/* |
| 270 |
* Even with single context, since the header is sent via pio-flow, |
| 271 |
* and data is sent via sdma-flow, data could be received first, |
| 272 |
* thus causes req=NULL. |
| 273 |
*/ |
| 274 |
if (req == NULL) { |
| 275 |
flow->msg_ooo_toggle = !flow->msg_ooo_toggle; |
| 276 |
if (flow->msg_ooo_toggle) { |
| 277 |
flow->recv_seq_num.pkt -= 1; |
| 278 |
return IPS_RECVHDRQ_OOO; |
| 279 |
} |
| 280 |
} else { |
| 281 |
flow->msg_ooo_toggle = 0; |
| 282 |
} |
| 283 |
|
| 284 |
psmi_mq_handle_data(req, epaddr, p_hdr->data[1].u32w0, |
| 285 |
p_hdr->data[1].u32w1, payload, paylen); |
| 286 |
|
| 287 |
/* If checksum is enabled, this matches what is done for tid-sdma */ |
| 288 |
/* if OOO and req is NULL, header is not received and we ignore chksum */ |
| 289 |
if (rcv_ev->proto->flags & IPS_PROTO_FLAG_CKSUM && |
| 290 |
mode == MQ_MSG_DATA_BLK && |
| 291 |
req && req->state == MQ_STATE_COMPLETE) { |
| 292 |
uint32_t cksum = ips_crc_calculate( |
| 293 |
req->recv_msglen - p_hdr->data[0].u32w1, |
| 294 |
(uint8_t *)req->buf + p_hdr->data[0].u32w1, |
| 295 |
0xffffffff); |
| 296 |
if (p_hdr->data[0].u32w0 != cksum) { |
| 297 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 298 |
"ErrPkt: Checksum mismatch. Expected: 0x%08x, Received: 0x%08x Source LID: %i. Aborting! \n", p_hdr->data[0].u32w0, cksum, __be16_to_cpu(flow->path->epr_dlid)); |
| 299 |
ips_proto_dump_data(req->buf, req->recv_msglen); |
| 300 |
} |
| 301 |
} |
| 302 |
|
| 303 |
} else if (mode == MQ_MSG_DATA_REQ || mode == MQ_MSG_DATA_REQ_BLK) { |
| 304 |
req = psmi_mpool_find_obj_by_index(mq->rreq_pool, |
| 305 |
p_hdr->data[1].u32w0); |
| 306 |
if (!req) goto skip_ack_req; |
| 307 |
psmi_mq_handle_data(req, epaddr, p_hdr->data[1].u32w0, |
| 308 |
p_hdr->data[1].u32w1, (void *) payload, paylen); |
| 309 |
|
| 310 |
/* If checksum is enabled, this matches what is done for tid-sdma */ |
| 311 |
if (rcv_ev->proto->flags & IPS_PROTO_FLAG_CKSUM && |
| 312 |
mode == MQ_MSG_DATA_REQ_BLK && |
| 313 |
req->state == MQ_STATE_COMPLETE) { |
| 314 |
uint32_t cksum = ips_crc_calculate( |
| 315 |
req->recv_msglen, req->buf, 0xffffffff); |
| 316 |
if (p_hdr->data[0].u32w0 != cksum) { |
| 317 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 318 |
"ErrPkt: Checksum mismatch. Expected: 0x%08x, Received: 0x%08x Source LID: %i. Aborting! \n", p_hdr->data[0].u32w0, cksum, __be16_to_cpu(flow->path->epr_dlid)); |
| 319 |
ips_proto_dump_data(req->buf, req->recv_msglen); |
| 320 |
} |
| 321 |
} |
| 322 |
|
| 323 |
} else if (mode == MQ_MSG_CTS_EGR) { |
| 324 |
args = p_hdr->data; |
| 325 |
ips_proto_mq_handle_cts(rcv_ev->proto, args); |
| 326 |
} else { |
| 327 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 328 |
"Unknown frame mode %x", mode); |
| 329 |
} |
| 330 |
|
| 331 |
if ((p_hdr->flags & IPS_SEND_FLAG_ACK_REQ) || |
| 332 |
(flow->flags & IPS_FLOW_FLAG_GEN_BECN)) |
| 333 |
ips_proto_send_ack((struct ips_recvhdrq *) rcv_ev->recvq, flow); |
| 334 |
|
| 335 |
skip_ack_req: |
| 336 |
ips_proto_process_ack(rcv_ev); |
| 337 |
|
| 338 |
return ret; // skip |
| 339 |
} |
| 340 |
|
| 341 |
PSMI_INLINE( |
| 342 |
int between(int first_seq, int last_seq, int seq)) |
| 343 |
{ |
| 344 |
if (last_seq >= first_seq) { |
| 345 |
if (seq < first_seq || seq > last_seq) { |
| 346 |
return 0; |
| 347 |
} |
| 348 |
} else { |
| 349 |
if (seq > last_seq && seq < first_seq) { |
| 350 |
return 0; |
| 351 |
} |
| 352 |
} |
| 353 |
return 1; |
| 354 |
} |
| 355 |
|
| 356 |
PSMI_INLINE( |
| 357 |
int pio_dma_ack_valid(struct ips_flow *flow, psmi_seqnum_t ack_seq_num, |
| 358 |
uint32_t ack_window)) |
| 359 |
{ |
| 360 |
uint32_t first_pkt, last_pkt; |
| 361 |
struct ips_scb_unackedq *unackedq = &flow->scb_unacked; |
| 362 |
|
| 363 |
if (STAILQ_EMPTY(unackedq)) |
| 364 |
return 0; |
| 365 |
|
| 366 |
first_pkt = flow->xmit_ack_num.pkt + 1; |
| 367 |
last_pkt = STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num.pkt; |
| 368 |
return between(first_pkt, last_pkt, ack_seq_num.pkt); |
| 369 |
} |
| 370 |
|
| 371 |
PSMI_INLINE( |
| 372 |
struct ips_flow* get_tidflow(ips_epaddr_t *ipsaddr, |
| 373 |
struct ips_message_header *p_hdr, |
| 374 |
psmi_seqnum_t ack_seq_num, |
| 375 |
uint32_t ack_window)) |
| 376 |
{ |
| 377 |
struct ips_flow *flow; |
| 378 |
struct ips_protoexp *protoexp = ipsaddr->proto->protoexp; |
| 379 |
struct ips_tid_send_desc *tidsendc; |
| 380 |
ptl_arg_t desc_id = p_hdr->data[0]; |
| 381 |
ptl_arg_t desc_tidsendc; |
| 382 |
uint32_t first_seq, last_seq; |
| 383 |
struct ips_scb_unackedq *unackedq; |
| 384 |
|
| 385 |
tidsendc = (struct ips_tid_send_desc*) |
| 386 |
psmi_mpool_find_obj_by_index(protoexp->tid_desc_send_pool, |
| 387 |
desc_id._desc_idx); |
| 388 |
if (tidsendc == NULL) { |
| 389 |
_IPATH_ERROR("OPCODE_ACK: Index %d is out of range in tidflow ack\n", desc_id._desc_idx); |
| 390 |
return NULL; |
| 391 |
} |
| 392 |
|
| 393 |
/* Ensure generation matches */ |
| 394 |
psmi_mpool_get_obj_index_gen_count(tidsendc, |
| 395 |
&desc_tidsendc._desc_idx, |
| 396 |
&desc_tidsendc._desc_genc); |
| 397 |
if (desc_tidsendc.u64 != desc_id.u64) |
| 398 |
return NULL; |
| 399 |
|
| 400 |
/* Ensure ack is within window */ |
| 401 |
flow = &tidsendc->tidflow; |
| 402 |
unackedq = &flow->scb_unacked; |
| 403 |
|
| 404 |
/* No unacked scbs */ |
| 405 |
if (STAILQ_EMPTY(unackedq)) |
| 406 |
return NULL; |
| 407 |
|
| 408 |
first_seq = flow->xmit_ack_num.seq + 1; |
| 409 |
last_seq = STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num.seq; |
| 410 |
if (between(first_seq, last_seq, ack_seq_num.seq) == 0) { |
| 411 |
return NULL; |
| 412 |
} |
| 413 |
|
| 414 |
/* Generation for ack should match */ |
| 415 |
if (STAILQ_FIRST(unackedq)->seq_num.gen != ack_seq_num.gen) |
| 416 |
return NULL; |
| 417 |
|
| 418 |
return flow; |
| 419 |
} |
| 420 |
|
| 421 |
/* NAK post process for tid flow */ |
| 422 |
void ips_tidflow_nak_post_process(struct ips_flow *flow, |
| 423 |
struct ips_message_header *p_hdr) |
| 424 |
{ |
| 425 |
|
| 426 |
ips_scb_t *scb; |
| 427 |
struct ips_scb_unackedq *unackedq = &flow->scb_unacked; |
| 428 |
#ifdef PSM_DEBUG |
| 429 |
psmi_seqnum_t new_flowgenseq; |
| 430 |
|
| 431 |
new_flowgenseq.val = p_hdr->data[1].u32w0; |
| 432 |
/* Update any pending scb's to the new generation count. |
| 433 |
* Note: flow->xmit_seq_num was updated to the new generation when the |
| 434 |
* NAK was received. |
| 435 |
*/ |
| 436 |
psmi_assert(STAILQ_FIRST(unackedq)->seq_num.flow==new_flowgenseq.flow); |
| 437 |
psmi_assert(STAILQ_FIRST(unackedq)->seq_num.gen != new_flowgenseq.gen); |
| 438 |
psmi_assert(STAILQ_FIRST(unackedq)-> |
| 439 |
seq_num.seq-STAILQ_FIRST(unackedq)->nfrag+1 == new_flowgenseq.seq); |
| 440 |
#endif |
| 441 |
|
| 442 |
/* Update unacked scb's to use the new flowgenseq */ |
| 443 |
scb = STAILQ_FIRST(unackedq); |
| 444 |
while (scb) { |
| 445 |
scb->ips_lrh.bth[2] = __cpu_to_be32(flow->xmit_seq_num.psn); |
| 446 |
flow->xmit_seq_num.seq += scb->nfrag; |
| 447 |
scb->seq_num = flow->xmit_seq_num; |
| 448 |
scb->seq_num.seq--; |
| 449 |
scb = SLIST_NEXT(scb, next); |
| 450 |
} |
| 451 |
|
| 452 |
} |
| 453 |
|
| 454 |
// process an incoming ack message. Separate function to allow |
| 455 |
// for better optimization by compiler |
| 456 |
void __recvpath |
| 457 |
ips_proto_process_ack(struct ips_recvhdrq_event *rcv_ev) |
| 458 |
{ |
| 459 |
ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr; |
| 460 |
struct ips_message_header *p_hdr = rcv_ev->p_hdr; |
| 461 |
psmi_seqnum_t ack_seq_num, last_seq_num; |
| 462 |
ips_scb_t *scb; |
| 463 |
struct ips_proto *proto = ipsaddr->proto; |
| 464 |
struct ips_flow *flow = NULL; |
| 465 |
struct ips_scb_unackedq *unackedq; |
| 466 |
struct ips_scb_pendlist *scb_pend; |
| 467 |
psm_protocol_type_t protocol; |
| 468 |
ptl_epaddr_flow_t flowid; |
| 469 |
|
| 470 |
ips_ptladdr_lock(ipsaddr); |
| 471 |
|
| 472 |
protocol = IPS_FLOWID_GET_PROTO(p_hdr->flowid); |
| 473 |
flowid = IPS_FLOWID_GET_INDEX(p_hdr->flowid); |
| 474 |
ack_seq_num.psn = p_hdr->ack_seq_num; |
| 475 |
|
| 476 |
switch(protocol){ |
| 477 |
case PSM_PROTOCOL_GO_BACK_N: |
| 478 |
flow = &ipsaddr->flows[flowid]; |
| 479 |
ack_seq_num.pkt -= 1; |
| 480 |
if (!pio_dma_ack_valid(flow, ack_seq_num, proto->scb_max_inflight)) |
| 481 |
goto ret; |
| 482 |
flow->xmit_ack_num = ack_seq_num; |
| 483 |
break; |
| 484 |
case PSM_PROTOCOL_TIDFLOW: |
| 485 |
ack_seq_num.seq -= 1; |
| 486 |
flow = get_tidflow(ipsaddr, p_hdr, ack_seq_num, proto->scb_max_inflight); |
| 487 |
if (!flow) /* Invalid ack for flow */ |
| 488 |
goto ret; |
| 489 |
flow->xmit_ack_num = ack_seq_num; |
| 490 |
break; |
| 491 |
default: |
| 492 |
_IPATH_ERROR("OPCODE_ACK: Unknown flow type %d in ACK\n", flowid); |
| 493 |
goto ret; |
| 494 |
} |
| 495 |
|
| 496 |
unackedq = &flow->scb_unacked; |
| 497 |
scb_pend = &flow->scb_pend; |
| 498 |
if (STAILQ_EMPTY(unackedq)) goto ret; // only for Klockwork scan. |
| 499 |
last_seq_num = STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num; |
| 500 |
|
| 501 |
INC_TIME_SPEND(TIME_SPEND_USER2); |
| 502 |
|
| 503 |
/* For tidflow, we want to match all flow/gen/seq, |
| 504 |
for gobackn, we only match pkt#, msg# is not known. |
| 505 |
msg# is the message envelope number in the stream, |
| 506 |
you don't know if the next packet has the old msg# |
| 507 |
or starts a new msg#. |
| 508 |
*/ |
| 509 |
/* first release all xmit buffer that has been receveid */ |
| 510 |
while ((protocol==PSM_PROTOCOL_GO_BACK_N) ? |
| 511 |
between(STAILQ_FIRST(unackedq)->seq_num.pkt, |
| 512 |
last_seq_num.pkt, ack_seq_num.pkt) : |
| 513 |
between(STAILQ_FIRST(unackedq)->seq_num.psn, |
| 514 |
last_seq_num.psn, ack_seq_num.psn) |
| 515 |
) { |
| 516 |
|
| 517 |
/* take it out of the xmit queue and .. */ |
| 518 |
scb = STAILQ_FIRST(unackedq); |
| 519 |
STAILQ_REMOVE_HEAD(unackedq, nextq); |
| 520 |
flow->scb_num_unacked--; |
| 521 |
flow->credits++; |
| 522 |
|
| 523 |
if (scb == SLIST_FIRST(scb_pend)) { |
| 524 |
flow->scb_num_pending--; |
| 525 |
SLIST_REMOVE_HEAD(scb_pend, next); |
| 526 |
} |
| 527 |
|
| 528 |
if (scb->flags & IPS_SEND_FLAG_WAIT_SDMA) |
| 529 |
ips_proto_dma_wait_until(proto, scb->dma_ctr); |
| 530 |
|
| 531 |
if (scb->callback) |
| 532 |
(*scb->callback) (scb->cb_param, scb->payload_size-scb->extra_bytes); |
| 533 |
|
| 534 |
if (!(scb->flags & IPS_SEND_FLAG_PERSISTENT)) |
| 535 |
ips_scbctrl_free(scb); |
| 536 |
|
| 537 |
/* set all index pointer to NULL if all frames have been |
| 538 |
* acked */ |
| 539 |
if (STAILQ_EMPTY(unackedq)) { |
| 540 |
psmi_timer_cancel(proto->timerq, &flow->timer_ack); |
| 541 |
psmi_timer_cancel(proto->timerq, &flow->timer_send); |
| 542 |
SLIST_FIRST(scb_pend) = NULL; |
| 543 |
psmi_assert(flow->scb_num_pending == 0); |
| 544 |
/* Reset congestion window - all packets ACK'd */ |
| 545 |
flow->credits = flow->cwin = proto->flow_credits; |
| 546 |
flow->ack_interval = max((flow->credits >> 2) - 1, 1); |
| 547 |
flow->flags &= ~IPS_FLOW_FLAG_CONGESTED; |
| 548 |
goto ret; |
| 549 |
} |
| 550 |
} |
| 551 |
|
| 552 |
/* CCA: If flow is congested adjust rate */ |
| 553 |
if_pf (rcv_ev->is_congested & IPS_RECV_EVENT_BECN) { |
| 554 |
if ((flow->path->epr_ccti + |
| 555 |
proto->cace[flow->path->epr_sl].ccti_increase) <= |
| 556 |
proto->ccti_limit) { |
| 557 |
ips_cca_adjust_rate(flow->path, |
| 558 |
proto->cace[flow->path->epr_sl].ccti_increase); |
| 559 |
/* Clear congestion event */ |
| 560 |
rcv_ev->is_congested &= ~IPS_RECV_EVENT_BECN; |
| 561 |
} |
| 562 |
} |
| 563 |
else { |
| 564 |
/* Increase congestion window if flow is not congested */ |
| 565 |
if_pf (flow->cwin < proto->flow_credits) { |
| 566 |
flow->credits += |
| 567 |
min(flow->cwin << 1, proto->flow_credits) - flow->cwin; |
| 568 |
flow->cwin = min(flow->cwin << 1, proto->flow_credits); |
| 569 |
flow->ack_interval = max((flow->credits >> 2) - 1, 1); |
| 570 |
} |
| 571 |
} |
| 572 |
|
| 573 |
/* Reclaimed some credits - attempt to flush flow */ |
| 574 |
flow->fn.xfer.flush(flow, NULL); |
| 575 |
|
| 576 |
/* |
| 577 |
* If the next packet has not even been put on the wire, cancel the |
| 578 |
* retransmission timer since we're still presumably waiting on free |
| 579 |
* pio bufs |
| 580 |
*/ |
| 581 |
if (STAILQ_FIRST(unackedq)->abs_timeout == TIMEOUT_INFINITE) |
| 582 |
psmi_timer_cancel(proto->timerq, &flow->timer_ack); |
| 583 |
|
| 584 |
ret: |
| 585 |
ips_ptladdr_unlock(ipsaddr); |
| 586 |
return; |
| 587 |
} |
| 588 |
|
| 589 |
// process an incoming nack message. Separate function to allow |
| 590 |
// for better optimization by compiler |
| 591 |
static void |
| 592 |
_process_nak(struct ips_recvhdrq_event *rcv_ev) |
| 593 |
{ |
| 594 |
ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr; |
| 595 |
struct ips_message_header *p_hdr = rcv_ev->p_hdr; |
| 596 |
psmi_seqnum_t ack_seq_num, last_seq_num; |
| 597 |
ips_scb_t *scb; |
| 598 |
struct ips_proto *proto = ipsaddr->proto; |
| 599 |
struct ips_flow *flow = NULL; |
| 600 |
struct ips_scb_unackedq *unackedq; |
| 601 |
struct ips_scb_pendlist *scb_pend; |
| 602 |
psm_protocol_type_t protocol; |
| 603 |
ptl_epaddr_flow_t flowid; |
| 604 |
int num_resent = 0; |
| 605 |
|
| 606 |
ips_ptladdr_lock(ipsaddr); |
| 607 |
|
| 608 |
protocol = IPS_FLOWID_GET_PROTO(p_hdr->flowid); |
| 609 |
flowid = IPS_FLOWID_GET_INDEX(p_hdr->flowid); |
| 610 |
|
| 611 |
INC_TIME_SPEND(TIME_SPEND_USER3); |
| 612 |
|
| 613 |
ack_seq_num.psn = p_hdr->ack_seq_num; |
| 614 |
|
| 615 |
switch(protocol){ |
| 616 |
case PSM_PROTOCOL_GO_BACK_N: |
| 617 |
flow = &ipsaddr->flows[flowid]; |
| 618 |
if (!pio_dma_ack_valid(flow, ack_seq_num, proto->scb_max_inflight)) |
| 619 |
goto ret; |
| 620 |
ack_seq_num.pkt--; |
| 621 |
flow->xmit_ack_num = ack_seq_num; |
| 622 |
break; |
| 623 |
case PSM_PROTOCOL_TIDFLOW: |
| 624 |
flow = get_tidflow(ipsaddr, p_hdr, ack_seq_num, proto->scb_max_inflight); |
| 625 |
if (!flow) |
| 626 |
goto ret; /* Invalid ack for flow */ |
| 627 |
ack_seq_num.seq--; |
| 628 |
/* Update xmit seq num to the new flowgenseq */ |
| 629 |
flow->xmit_seq_num = (psmi_seqnum_t)p_hdr->data[1].u32w0; |
| 630 |
flow->xmit_ack_num = flow->xmit_seq_num; |
| 631 |
flow->xmit_ack_num.seq--; |
| 632 |
break; |
| 633 |
default: |
| 634 |
_IPATH_ERROR("OPCODE_NAK: Unknown flow type %d in ACK\n", flowid); |
| 635 |
goto ret; |
| 636 |
} |
| 637 |
|
| 638 |
unackedq = &flow->scb_unacked; |
| 639 |
scb_pend = &flow->scb_pend; |
| 640 |
if (STAILQ_EMPTY(unackedq)) goto ret; // only for Klockwork scan. |
| 641 |
last_seq_num = STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num; |
| 642 |
|
| 643 |
ipsaddr->stats.nak_recv++; |
| 644 |
|
| 645 |
_IPATH_VDBG("got a nack %d on flow %d, " |
| 646 |
"first is %d, last is %d\n", ack_seq_num.psn, |
| 647 |
flowid, STAILQ_EMPTY(unackedq)?-1:STAILQ_FIRST(unackedq)->seq_num.psn, |
| 648 |
STAILQ_EMPTY(unackedq)?-1:STAILQ_LAST(unackedq, ips_scb, nextq)->seq_num.psn); |
| 649 |
|
| 650 |
/* For tidflow, we want to match all flow/gen/seq, |
| 651 |
for gobackn, we only match pkt#, msg# is not known. |
| 652 |
msg# is the message envelope number in the stream, |
| 653 |
you don't know if the next packet has the old msg# |
| 654 |
or starts a new msg#. |
| 655 |
*/ |
| 656 |
/* first release all xmit buffer that has been receveid */ |
| 657 |
while ((protocol==PSM_PROTOCOL_GO_BACK_N) ? |
| 658 |
between(STAILQ_FIRST(unackedq)->seq_num.pkt, |
| 659 |
last_seq_num.pkt, ack_seq_num.pkt) : |
| 660 |
between(STAILQ_FIRST(unackedq)->seq_num.psn, |
| 661 |
last_seq_num.psn, ack_seq_num.psn) |
| 662 |
) { |
| 663 |
/* take it out of the xmit queue and .. */ |
| 664 |
scb = STAILQ_FIRST(unackedq); |
| 665 |
STAILQ_REMOVE_HEAD(unackedq, nextq); |
| 666 |
flow->scb_num_unacked--; |
| 667 |
|
| 668 |
if (scb->flags & IPS_SEND_FLAG_WAIT_SDMA) |
| 669 |
ips_proto_dma_wait_until(proto, scb->dma_ctr); |
| 670 |
|
| 671 |
if (scb->callback) |
| 672 |
(*scb->callback) (scb->cb_param, scb->payload_size-scb->extra_bytes); |
| 673 |
|
| 674 |
if (!(scb->flags & IPS_SEND_FLAG_PERSISTENT)) |
| 675 |
ips_scbctrl_free(scb); |
| 676 |
|
| 677 |
/* set all index pointer to NULL if all frames has been acked */ |
| 678 |
if (STAILQ_EMPTY(unackedq)) { |
| 679 |
psmi_timer_cancel(proto->timerq, &flow->timer_ack); |
| 680 |
psmi_timer_cancel(proto->timerq, &flow->timer_send); |
| 681 |
SLIST_FIRST(scb_pend) = NULL; |
| 682 |
psmi_assert(flow->scb_num_pending == 0); |
| 683 |
/* Reset congestion window if all packets acknowledged */ |
| 684 |
flow->credits = flow->cwin = proto->flow_credits; |
| 685 |
flow->ack_interval = max((flow->credits >> 2) - 1, 1); |
| 686 |
flow->flags &= ~IPS_FLOW_FLAG_CONGESTED; |
| 687 |
goto ret; |
| 688 |
} |
| 689 |
} |
| 690 |
|
| 691 |
psmi_assert(!STAILQ_EMPTY(unackedq)); /* sanity for above loop */ |
| 692 |
|
| 693 |
if (flow->fn.protocol.nak_post_process) |
| 694 |
flow->fn.protocol.nak_post_process(flow, p_hdr); |
| 695 |
|
| 696 |
/* Always cancel ACK timer as we are going to restart the flow */ |
| 697 |
psmi_timer_cancel(proto->timerq, &flow->timer_ack); |
| 698 |
|
| 699 |
/* What's now pending is all that was unacked */ |
| 700 |
SLIST_FIRST(scb_pend) = STAILQ_FIRST(unackedq); |
| 701 |
flow->scb_num_pending = flow->scb_num_unacked; |
| 702 |
|
| 703 |
/* If NAK with congestion bit set - delay re-transmitting and THEN adjust |
| 704 |
* CCA rate. |
| 705 |
*/ |
| 706 |
if_pf (rcv_ev->is_congested & IPS_RECV_EVENT_BECN) { |
| 707 |
uint64_t offset; |
| 708 |
|
| 709 |
/* Clear congestion event and mark flow as congested */ |
| 710 |
rcv_ev->is_congested &= ~IPS_RECV_EVENT_BECN; |
| 711 |
flow->flags |= IPS_FLOW_FLAG_CONGESTED; |
| 712 |
|
| 713 |
/* For congested flow use slow start i.e. reduce congestion window. |
| 714 |
* For TIDFLOW we cannot reduce congestion window as peer expects |
| 715 |
* header packets at regular intervals (protoexp->hdr_pkt_interval). |
| 716 |
*/ |
| 717 |
if (flow->protocol != PSM_PROTOCOL_TIDFLOW) |
| 718 |
flow->credits = flow->cwin = 1; |
| 719 |
else |
| 720 |
flow->credits = flow->cwin; |
| 721 |
|
| 722 |
flow->ack_interval = max((flow->credits >> 2) - 1, 1); |
| 723 |
|
| 724 |
/* During congestion cancel send timer and delay retransmission by |
| 725 |
* random interval |
| 726 |
*/ |
| 727 |
psmi_timer_cancel(proto->timerq, &flow->timer_send); |
| 728 |
if (SLIST_FIRST(scb_pend)->ack_timeout != TIMEOUT_INFINITE) |
| 729 |
offset = (SLIST_FIRST(scb_pend)->ack_timeout >> 1); |
| 730 |
else |
| 731 |
offset = 0; |
| 732 |
psmi_timer_request(proto->timerq, &flow->timer_send, |
| 733 |
(get_cycles() + |
| 734 |
(uint64_t)(offset * (rand()/RAND_MAX + 1.0)))); |
| 735 |
} |
| 736 |
else { |
| 737 |
/* Reclaim all credits upto congestion window only */ |
| 738 |
flow->credits = flow->cwin; |
| 739 |
flow->ack_interval = max((flow->credits >> 2) - 1, 1); |
| 740 |
|
| 741 |
/* Flush pending scb's */ |
| 742 |
flow->fn.xfer.flush(flow, &num_resent); |
| 743 |
ipsaddr->stats.send_rexmit += num_resent; |
| 744 |
} |
| 745 |
|
| 746 |
ret: |
| 747 |
ips_ptladdr_unlock(ipsaddr); |
| 748 |
return; |
| 749 |
} |
| 750 |
|
| 751 |
static void |
| 752 |
_process_err_chk(struct ips_recvhdrq *recvq, ips_epaddr_t *ipsaddr, |
| 753 |
struct ips_message_header *p_hdr) |
| 754 |
{ |
| 755 |
psmi_seqnum_t seq_num; |
| 756 |
int16_t seq_off; |
| 757 |
ptl_epaddr_flow_t flowid = ips_proto_flowid(p_hdr); |
| 758 |
struct ips_flow *flow = &ipsaddr->flows[flowid]; |
| 759 |
|
| 760 |
INC_TIME_SPEND(TIME_SPEND_USER4); |
| 761 |
|
| 762 |
ipsaddr->stats.err_chk_recv++; |
| 763 |
|
| 764 |
seq_num.val = __be32_to_cpu(p_hdr->bth[2]); |
| 765 |
seq_off = (int16_t)(ipsaddr->flows[flowid].recv_seq_num.pkt - seq_num.pkt); |
| 766 |
|
| 767 |
if_pf (seq_off <= 0) { |
| 768 |
_IPATH_VDBG("naking for seq=%d, off=%d on flowid %d\n", |
| 769 |
seq_num.pkt, seq_off, flowid); |
| 770 |
|
| 771 |
if (seq_off < -flow->ack_interval) |
| 772 |
flow->flags |= IPS_FLOW_FLAG_GEN_BECN; |
| 773 |
|
| 774 |
ips_proto_send_nak(recvq, flow); |
| 775 |
flow->flags |= IPS_FLOW_FLAG_NAK_SEND; |
| 776 |
} |
| 777 |
else { |
| 778 |
ips_proto_send_ctrl_message(flow, OPCODE_ACK, |
| 779 |
&ipsaddr->ctrl_msg_queued, NULL); |
| 780 |
} |
| 781 |
} |
| 782 |
|
| 783 |
static void |
| 784 |
_process_err_chk_gen(ips_epaddr_t *ipsaddr, struct ips_message_header *p_hdr) |
| 785 |
{ |
| 786 |
struct ips_protoexp *protoexp = ipsaddr->proto->protoexp; |
| 787 |
struct ips_tid_recv_desc *tidrecvc; |
| 788 |
psmi_seqnum_t err_seqnum; |
| 789 |
ptl_arg_t desc_id = p_hdr->data[0]; |
| 790 |
ptl_arg_t send_desc_id = p_hdr->data[1]; |
| 791 |
ptl_arg_t desc_tidrecvc; |
| 792 |
ptl_arg_t args[3] = {}; |
| 793 |
int16_t seq_off; |
| 794 |
uint8_t ack_type; |
| 795 |
|
| 796 |
INC_TIME_SPEND(TIME_SPEND_USER4); |
| 797 |
|
| 798 |
ipsaddr->stats.err_chk_recv++; |
| 799 |
|
| 800 |
/* Get the flowgenseq for err chk gen */ |
| 801 |
err_seqnum.val = __be32_to_cpu(p_hdr->bth[2]); |
| 802 |
|
| 803 |
ips_ptladdr_lock(ipsaddr); |
| 804 |
|
| 805 |
/* Get receive descriptor */ |
| 806 |
tidrecvc = (struct ips_tid_recv_desc *) |
| 807 |
psmi_mpool_find_obj_by_index(protoexp->tid_desc_recv_pool, |
| 808 |
desc_id._desc_idx); |
| 809 |
|
| 810 |
if (tidrecvc == NULL) { |
| 811 |
_IPATH_DBG("ERR_CHK_GEN: invalid rendezvous handle\n"); |
| 812 |
ips_ptladdr_unlock(ipsaddr); |
| 813 |
return; |
| 814 |
} |
| 815 |
psmi_mpool_get_obj_index_gen_count(tidrecvc, |
| 816 |
&desc_tidrecvc._desc_idx, |
| 817 |
&desc_tidrecvc._desc_genc); |
| 818 |
|
| 819 |
if (desc_id.u64 != desc_tidrecvc.u64) { |
| 820 |
/* Receive descriptor mismatch in time and space. |
| 821 |
* Stale err chk gen, drop packet |
| 822 |
*/ |
| 823 |
_IPATH_DBG("ERR_CHK_GEN: rendezvous handle generation mismatch. Pkt: 0x%08x, Current: 0x%08x\n", desc_id._desc_genc, desc_tidrecvc._desc_genc); |
| 824 |
ips_ptladdr_unlock(ipsaddr); |
| 825 |
return; |
| 826 |
} |
| 827 |
|
| 828 |
psmi_assert(tidrecvc->tidflow_idx == err_seqnum.flow); |
| 829 |
|
| 830 |
/* Note: Do not read the tidflow table to determine the sequence to restart |
| 831 |
* from. Always respond with the last known "good" packet that we received |
| 832 |
* which is updated in protoexp_data(). |
| 833 |
*/ |
| 834 |
|
| 835 |
/* Either lost packets or lost ack */ |
| 836 |
seq_off = (int16_t) (tidrecvc->tidflow_genseq.seq - err_seqnum.seq); |
| 837 |
|
| 838 |
if (seq_off <= 0) { |
| 839 |
ack_type = OPCODE_NAK; |
| 840 |
|
| 841 |
if (err_seqnum.gen == tidrecvc->tidflow_active_gen) { |
| 842 |
/* Swap generations */ |
| 843 |
psm_error_t err; |
| 844 |
|
| 845 |
/* Allocate new generation for the flow. */ |
| 846 |
err = ips_protoexp_flow_newgen(tidrecvc); |
| 847 |
if (err != PSM_OK) |
| 848 |
return; /* Out of generation. Drop packet and we will recover later */ |
| 849 |
} |
| 850 |
} |
| 851 |
else { |
| 852 |
ack_type = OPCODE_ACK; |
| 853 |
|
| 854 |
if (err_seqnum.gen != tidrecvc->tidflow_genseq.gen) |
| 855 |
ack_type = OPCODE_NAK; /* NAK without allocating a new generation */ |
| 856 |
} |
| 857 |
|
| 858 |
args[0] = send_desc_id; |
| 859 |
args[1] = tidrecvc->tid_list.tsess_descid; |
| 860 |
args[2].u16w0 = err_seqnum.gen; /* If NAK, generation number */ |
| 861 |
|
| 862 |
ips_ptladdr_unlock(ipsaddr); |
| 863 |
|
| 864 |
/* May want to generate a BECN if a lot of swapped generations */ |
| 865 |
if_pf ((tidrecvc->tidflow_nswap_gen > 4) && |
| 866 |
(protoexp->proto->flags & IPS_PROTO_FLAG_CCA)) { |
| 867 |
_IPATH_CCADBG("ERR_CHK_GEN: Generating BECN. Number of swapped generations: %d.\n", tidrecvc->tidflow_nswap_gen); |
| 868 |
/* Mark flow to generate BECN in control packet */ |
| 869 |
tidrecvc->ipsaddr->tidgr_flow.flags |= IPS_FLOW_FLAG_GEN_BECN; |
| 870 |
|
| 871 |
/* Update stats for congestion encountered */ |
| 872 |
ipsaddr->stats.congestion_pkts++; |
| 873 |
} |
| 874 |
|
| 875 |
ips_proto_send_ctrl_message(&tidrecvc->ipsaddr->tidgr_flow, |
| 876 |
ack_type, &tidrecvc->ctrl_msg_queued, args); |
| 877 |
|
| 878 |
/* Update stats for expected window */ |
| 879 |
tidrecvc->stats.nErrChkReceived++; |
| 880 |
if (ack_type == OPCODE_NAK) |
| 881 |
tidrecvc->stats.nReXmit++; /* Update stats for retransmit (Sent a NAK) */ |
| 882 |
} |
| 883 |
|
| 884 |
static void |
| 885 |
parse_ip_or_lid(char *buf, size_t len, uint32_t ip, psm_epid_t epid) |
| 886 |
{ |
| 887 |
char ipbuf[INET_ADDRSTRLEN], *p; |
| 888 |
in_addr_t in_loop = inet_addr("127.0.0.1"); |
| 889 |
in_addr_t in_any = inet_addr("0.0.0.0"); |
| 890 |
|
| 891 |
p = (char *) inet_ntop(AF_INET, (const void *) &ip, ipbuf, sizeof ipbuf); |
| 892 |
if (ip != in_loop && ip != in_any && p) |
| 893 |
snprintf(buf, len-1, "IP %s", p); |
| 894 |
else |
| 895 |
snprintf(buf, len-1, "LID 0x%x", (int) psm_epid_nid(epid)); |
| 896 |
buf[len-1] = '\0'; |
| 897 |
} |
| 898 |
|
| 899 |
#define IPS_MAX_BOGUS_ERR_CHK_BAD 15 |
| 900 |
|
| 901 |
static void |
| 902 |
_process_err_chk_bad(ips_epaddr_t *ipsaddr, struct ips_message_header *p_hdr) |
| 903 |
{ |
| 904 |
uint32_t ipv4_addr = p_hdr->data[0].u32w0; |
| 905 |
uint32_t pid = __be32_to_cpu(p_hdr->data[0].u32w1); |
| 906 |
union psmi_envvar_val env_stray; |
| 907 |
char buf[32]; |
| 908 |
psm_epid_t epid = ipsaddr->epaddr->epid; |
| 909 |
|
| 910 |
parse_ip_or_lid(buf, sizeof buf, ipv4_addr, epid); |
| 911 |
|
| 912 |
/* First make sure that we actually do have a connection to this lid+context, |
| 913 |
* if not, we just ignore the err_chk_bad message since it might be some |
| 914 |
* oddly timed packet */ |
| 915 |
if (!ips_proto_isconnected(ipsaddr)) { |
| 916 |
int lid = (int) psm_epid_nid(epid); |
| 917 |
int context = (int) psm_epid_context(epid); |
| 918 |
if (++ipsaddr->proto->num_bogus_warnings <= IPS_MAX_BOGUS_ERR_CHK_BAD) |
| 919 |
psmi_syslog(ipsaddr->proto->ep, 1, LOG_INFO, |
| 920 |
"PSM pid %d on host %s complains that I am a stray process but " |
| 921 |
"I'm not even connected to LID %d context %d (ignoring %s\n", |
| 922 |
pid, buf, lid, context, |
| 923 |
ipsaddr->proto->num_bogus_warnings == IPS_MAX_BOGUS_ERR_CHK_BAD ? |
| 924 |
"all future stray warning checks from unknown endpoints)." : |
| 925 |
")."); |
| 926 |
return; |
| 927 |
} |
| 928 |
|
| 929 |
/* At this point the bad error check is a real one, from a host we thought |
| 930 |
* we were connected to. We only go through this path once. If |
| 931 |
* PSM_STRAY_ENABLED=0, we'll print this warning once, if it's 1 we'll die. |
| 932 |
*/ |
| 933 |
if (ipsaddr->proto->done_once++) |
| 934 |
return; |
| 935 |
|
| 936 |
psmi_getenv("PSM_STRAY_ENABLED", "Enable stray process detection", |
| 937 |
PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_YESNO, |
| 938 |
(union psmi_envvar_val) 1, /* yes by default */ |
| 939 |
&env_stray); |
| 940 |
|
| 941 |
if (env_stray.e_uint) |
| 942 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_EPID_NETWORK_ERROR, "PSM pid " |
| 943 |
"%d on host %s has detected that I am a stray process, exiting.", |
| 944 |
pid, buf); |
| 945 |
else |
| 946 |
psmi_syslog(ipsaddr->proto->ep, 1, LOG_INFO, "PSM pid " |
| 947 |
"%d on host %s has detected that I am a stray process, " |
| 948 |
"PSM_STRAY_ENABLED is off and future messages are ignored."); |
| 949 |
return; |
| 950 |
} |
| 951 |
|
| 952 |
static void ips_bad_opcode(uint8_t op_code, struct ips_message_header *proto) |
| 953 |
{ |
| 954 |
_IPATH_DBG("Discarding message with bad opcode 0x%x\n", |
| 955 |
op_code); |
| 956 |
|
| 957 |
if(infinipath_debug&__IPATH_DBG) { |
| 958 |
ips_proto_show_header(proto, "received bad opcode"); |
| 959 |
ips_proto_dump_frame(proto, sizeof(struct ips_message_header), |
| 960 |
"Opcode error protocol header dump"); |
| 961 |
} |
| 962 |
} |
| 963 |
|
| 964 |
static void |
| 965 |
_process_unknown_opcode(struct ips_proto *proto, |
| 966 |
struct ips_message_header *protocol_header) |
| 967 |
{ |
| 968 |
proto->stats.unknown_packets++; |
| 969 |
|
| 970 |
switch (protocol_header->sub_opcode) { |
| 971 |
/* A bunch of pre-PSM packets that we don't handle any more */ |
| 972 |
case OPCODE_SEQ_DATA: |
| 973 |
case OPCODE_SEQ_CTRL: |
| 974 |
case OPCODE_STARTUP: |
| 975 |
case OPCODE_STARTUP_EXT: |
| 976 |
case OPCODE_STARTUP_ACK: |
| 977 |
case OPCODE_STARTUP_ACK_EXT: |
| 978 |
case OPCODE_STARTUP_NAK: |
| 979 |
case OPCODE_STARTUP_NAK_EXT: |
| 980 |
case OPCODE_CLOSE: |
| 981 |
case OPCODE_ABORT: |
| 982 |
case OPCODE_CLOSE_ACK: |
| 983 |
break; |
| 984 |
default: |
| 985 |
ips_bad_opcode(protocol_header->sub_opcode, protocol_header); |
| 986 |
break; |
| 987 |
} |
| 988 |
} |
| 989 |
|
| 990 |
PSMI_NEVER_INLINE( |
| 991 |
int |
| 992 |
_process_connect(const struct ips_recvhdrq_event *rcv_ev)) |
| 993 |
{ |
| 994 |
const uint16_t lmc_mask = ~((1 << rcv_ev->proto->epinfo.ep_lmc) - 1); |
| 995 |
|
| 996 |
return ips_proto_process_connect(rcv_ev->proto, |
| 997 |
ips_epid_from_phdr(lmc_mask, rcv_ev->p_hdr), |
| 998 |
rcv_ev->p_hdr->sub_opcode, |
| 999 |
rcv_ev->p_hdr, |
| 1000 |
ips_recvhdrq_event_payload(rcv_ev), |
| 1001 |
ips_recvhdrq_event_paylen(rcv_ev)); |
| 1002 |
} |
| 1003 |
|
| 1004 |
// Return 1 if packet is ok. |
| 1005 |
// Return 0 if packet should be skipped |
| 1006 |
int |
| 1007 |
ips_proto_process_unknown(const struct ips_recvhdrq_event *rcv_ev) |
| 1008 |
{ |
| 1009 |
struct ips_message_header *p_hdr = rcv_ev->p_hdr; |
| 1010 |
uint8_t ptype = rcv_ev->ptype; |
| 1011 |
const uint64_t current_count = get_cycles(); |
| 1012 |
struct ips_stray_epid *sepid; |
| 1013 |
struct ips_proto *proto = rcv_ev->proto; |
| 1014 |
psm_ep_t ep_err; |
| 1015 |
psm_epid_t epid; |
| 1016 |
char *pkt_type; |
| 1017 |
int opcode = (int) p_hdr->sub_opcode; |
| 1018 |
double t_elapsed; |
| 1019 |
ptl_epaddr_flow_t flowid = IPS_FLOWID_GET_INDEX(p_hdr->flowid); |
| 1020 |
const uint16_t lmc_mask = ~((1 << rcv_ev->proto->epinfo.ep_lmc) - 1); |
| 1021 |
|
| 1022 |
/* |
| 1023 |
* If the protocol is disabled or not yet enabled, no processing happens |
| 1024 |
* We set it t_init to 0 when disabling the protocol |
| 1025 |
*/ |
| 1026 |
if (proto->t_init == 0) |
| 1027 |
return IPS_RECVHDRQ_CONTINUE; |
| 1028 |
|
| 1029 |
/* |
| 1030 |
* If lid is 0, something bad happened in queue processing |
| 1031 |
*/ |
| 1032 |
epid = ips_epid_from_phdr(lmc_mask, p_hdr); |
| 1033 |
if (psm_epid_nid(epid) == 0ULL) { |
| 1034 |
proto->stats.lid_zero_errs++; |
| 1035 |
_IPATH_DBG("Skipping stray packet processing with LID=0\n"); |
| 1036 |
return IPS_RECVHDRQ_CONTINUE; |
| 1037 |
} |
| 1038 |
|
| 1039 |
/* Connect messages don't have to be from a known epaddr */ |
| 1040 |
switch (opcode) { |
| 1041 |
case OPCODE_CONNECT_REQUEST: |
| 1042 |
case OPCODE_CONNECT_REPLY: |
| 1043 |
case OPCODE_DISCONNECT_REQUEST: |
| 1044 |
case OPCODE_DISCONNECT_REPLY: |
| 1045 |
_process_connect(rcv_ev); |
| 1046 |
return IPS_RECVHDRQ_CONTINUE; |
| 1047 |
case OPCODE_ERR_CHK_BAD: /* ignore, old opcode */ |
| 1048 |
return IPS_RECVHDRQ_CONTINUE; |
| 1049 |
default: |
| 1050 |
break; |
| 1051 |
} |
| 1052 |
|
| 1053 |
/* Packet from "unknown" peer. Log the packet and payload if at appropriate |
| 1054 |
* verbose level. |
| 1055 |
*/ |
| 1056 |
{ |
| 1057 |
char *payload = ips_recvhdrq_event_payload(rcv_ev); |
| 1058 |
uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev) + |
| 1059 |
((__be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 20) & 3); |
| 1060 |
|
| 1061 |
ips_proto_dump_err_stats(proto); |
| 1062 |
|
| 1063 |
if(infinipath_debug & __IPATH_PKTDBG) { |
| 1064 |
ips_proto_dump_frame(rcv_ev->p_hdr, IPATH_MESSAGE_HDR_SIZE, "header"); |
| 1065 |
if (paylen) |
| 1066 |
ips_proto_dump_frame(payload, paylen, "data"); |
| 1067 |
} |
| 1068 |
} |
| 1069 |
|
| 1070 |
/* Other messages are definitely crosstalk. */ |
| 1071 |
/* out-of-context expected messages are always fatal */ |
| 1072 |
if (ptype == RCVHQ_RCV_TYPE_EXPECTED) { |
| 1073 |
ep_err = PSMI_EP_NORETURN; |
| 1074 |
pkt_type = "expected"; |
| 1075 |
} |
| 1076 |
else if (ptype == RCVHQ_RCV_TYPE_EAGER) { |
| 1077 |
ep_err = PSMI_EP_LOGEVENT; |
| 1078 |
pkt_type = "eager"; |
| 1079 |
} |
| 1080 |
else { |
| 1081 |
ep_err = PSMI_EP_NORETURN; |
| 1082 |
pkt_type = "unknown"; |
| 1083 |
} |
| 1084 |
|
| 1085 |
proto->stats.stray_packets++; |
| 1086 |
|
| 1087 |
/* If we have debug mode, print the complete packet every time */ |
| 1088 |
if (infinipath_debug & __IPATH_PKTDBG) |
| 1089 |
ips_proto_show_header(p_hdr, "invalid commidx"); |
| 1090 |
t_elapsed = (double) |
| 1091 |
cycles_to_nanosecs(get_cycles()-proto->t_init) / 1.0e9; |
| 1092 |
|
| 1093 |
sepid = (struct ips_stray_epid *) |
| 1094 |
psmi_epid_lookup(PSMI_EP_CROSSTALK, epid); |
| 1095 |
if (sepid == NULL) { /* Never seen crosstalk from this node, log it */ |
| 1096 |
sepid = (struct ips_stray_epid *) |
| 1097 |
psmi_calloc(proto->ep, UNDEFINED, 1, sizeof(struct ips_stray_epid)); |
| 1098 |
if (sepid == NULL) return 0; /* skip packet if no memory */ |
| 1099 |
psmi_epid_add(PSMI_EP_CROSSTALK, epid, (void *) sepid); |
| 1100 |
sepid->epid = epid; |
| 1101 |
if (proto->stray_warn_interval) |
| 1102 |
sepid->t_first = sepid->t_warn_next = current_count; |
| 1103 |
} |
| 1104 |
sepid->num_messages++; |
| 1105 |
sepid->t_last = current_count; |
| 1106 |
|
| 1107 |
/* If we're not going to warn the user and if this not to be a fatal |
| 1108 |
* packet, just skip it */ |
| 1109 |
if (sepid->t_warn_next > current_count && ep_err != PSMI_EP_NORETURN) |
| 1110 |
return 0; |
| 1111 |
|
| 1112 |
sepid->t_warn_next = current_count + proto->stray_warn_interval; |
| 1113 |
|
| 1114 |
if (p_hdr->sub_opcode == OPCODE_ERR_CHK) { |
| 1115 |
/* With the new err_check, we can print out extra information */ |
| 1116 |
char ipbuf[INET_ADDRSTRLEN], *ip = NULL; |
| 1117 |
sepid->ipv4_addr = p_hdr->data[0].u32w0; |
| 1118 |
sepid->pid = __be32_to_cpu(p_hdr->data[0].u32w1); |
| 1119 |
ip = (char *) inet_ntop(AF_INET, &sepid->ipv4_addr, ipbuf, sizeof ipbuf); |
| 1120 |
|
| 1121 |
/* If the IP and PID make sense, go ahead and print useful info and |
| 1122 |
* even reply with ERR_CHK_BAD. If not, fall through and print the |
| 1123 |
* generic bad error message |
| 1124 |
*/ |
| 1125 |
if (ip != NULL && sepid->pid) { |
| 1126 |
/* Make up a fake ipsaddr and reply */ |
| 1127 |
ips_epaddr_t ipsaddr_f; |
| 1128 |
psm_error_t err; |
| 1129 |
|
| 1130 |
/* debugging sanity, and catch bugs */ |
| 1131 |
memset(&ipsaddr_f, 0, sizeof(ips_epaddr_t)); |
| 1132 |
ipsaddr_f.epr.epr_context = IPS_HEADER_SRCCONTEXT_GET(p_hdr); |
| 1133 |
ipsaddr_f.epr.epr_subcontext = p_hdr->dst_subcontext; |
| 1134 |
ipsaddr_f.epr.epr_pkt_context = |
| 1135 |
ipsaddr_f.epr.epr_context & 0xf; |
| 1136 |
|
| 1137 |
/* Get path record for peer */ |
| 1138 |
err = proto->ibta.get_path_rec(proto, |
| 1139 |
proto->epinfo.ep_base_lid, |
| 1140 |
p_hdr->lrh[3], /* SLID */ |
| 1141 |
PSMI_HCA_TYPE_QLE73XX, |
| 1142 |
3000, &ipsaddr_f); |
| 1143 |
if (err != PSM_OK) |
| 1144 |
goto fail; |
| 1145 |
|
| 1146 |
ipsaddr_f.epr.epr_qp = __be32_to_cpu(p_hdr->bth[1]); |
| 1147 |
ipsaddr_f.epr.epr_qp &= 0xffffff; /* QP is 24 bits */ |
| 1148 |
ipsaddr_f.ptl = (ptl_t *) -1; |
| 1149 |
ipsaddr_f.proto = proto; |
| 1150 |
/* Pretend the ctrlmsg is already queued, so it doesn't get queued |
| 1151 |
* in this fake (stack-allocated) ptladdr */ |
| 1152 |
ipsaddr_f.ctrl_msg_queued = ~0; |
| 1153 |
flowid = EP_FLOW_GO_BACK_N_PIO; |
| 1154 |
ips_flow_init(&ipsaddr_f.flows[flowid], NULL, |
| 1155 |
&ipsaddr_f, PSM_TRANSFER_PIO, |
| 1156 |
PSM_PROTOCOL_GO_BACK_N, IPS_PATH_LOW_PRIORITY, flowid); |
| 1157 |
|
| 1158 |
if (!ips_proto_send_ctrl_message(&ipsaddr_f.flows[flowid], |
| 1159 |
OPCODE_ERR_CHK_BAD, |
| 1160 |
&ipsaddr_f.ctrl_msg_queued,NULL)){ |
| 1161 |
sepid->err_check_bad_sent++; |
| 1162 |
_IPATH_VDBG("did reply with ERR_CHK_BAD\n"); |
| 1163 |
} |
| 1164 |
else |
| 1165 |
_IPATH_VDBG("did *NOT* reply with ERR_CHK_BAD\n"); |
| 1166 |
|
| 1167 |
fail: |
| 1168 |
psmi_handle_error(ep_err, PSM_EPID_NETWORK_ERROR, |
| 1169 |
"Received %d out-of-context %s message(s) from stray process " |
| 1170 |
"PID=%d running on host %s (LID 0x%x, ptype=0x%x, subop=0x%x, " |
| 1171 |
"elapsed=%.3fs) %s", |
| 1172 |
sepid->num_messages, pkt_type, sepid->pid, ip, |
| 1173 |
(int) psm_epid_nid(epid), ptype, opcode, t_elapsed, |
| 1174 |
(ep_err == PSMI_EP_NORETURN) ? "Aborting." : ""); |
| 1175 |
return 0; |
| 1176 |
} |
| 1177 |
} |
| 1178 |
|
| 1179 |
/* At this point we either have a OPCODE_ERR_CHECK where we couldn't |
| 1180 |
* extract a valid ip and pid OR some other opcode */ |
| 1181 |
psmi_handle_error(ep_err, PSM_EPID_NETWORK_ERROR, |
| 1182 |
"Received out-of-context %s message(s) from a stray process " |
| 1183 |
"running on LID 0x%x ptype=0x%x subop=0x%x elapsed=%.3fs", |
| 1184 |
pkt_type, (int) psm_epid_nid(epid), ptype, opcode, t_elapsed); |
| 1185 |
|
| 1186 |
return 0; /* Always skip this packet unless the above call was a noreturn |
| 1187 |
* call */ |
| 1188 |
} |
| 1189 |
|
| 1190 |
/* get the error string as a number and a string */ |
| 1191 |
static void rhf_errnum_string(char *msg, size_t msglen, long err) |
| 1192 |
{ |
| 1193 |
int len; |
| 1194 |
char *errmsg; |
| 1195 |
|
| 1196 |
len = snprintf(msg, msglen, "RHFerror %lx: ", err); |
| 1197 |
if(len > 0 && len < msglen) { |
| 1198 |
errmsg = msg + len; |
| 1199 |
msglen -= len; |
| 1200 |
} |
| 1201 |
else |
| 1202 |
errmsg = msg; |
| 1203 |
*errmsg = 0; |
| 1204 |
ips_proto_get_rhf_errstring(err, errmsg, msglen); |
| 1205 |
} |
| 1206 |
|
| 1207 |
/* |
| 1208 |
* Error handling |
| 1209 |
*/ |
| 1210 |
int __recvpath |
| 1211 |
ips_proto_process_packet_error(struct ips_recvhdrq_event *rcv_ev) |
| 1212 |
{ |
| 1213 |
struct ips_proto *proto = rcv_ev->proto; |
| 1214 |
int pkt_verbose_err = infinipath_debug & __IPATH_PKTDBG; |
| 1215 |
int tiderr = rcv_ev->error_flags & INFINIPATH_RHF_H_TIDERR; |
| 1216 |
int tf_seqerr = rcv_ev->error_flags & INFINIPATH_RHF_H_TFSEQERR; |
| 1217 |
int tf_generr = rcv_ev->error_flags & INFINIPATH_RHF_H_TFGENERR; |
| 1218 |
int data_err = rcv_ev->error_flags & |
| 1219 |
(INFINIPATH_RHF_H_ICRCERR | INFINIPATH_RHF_H_VCRCERR | |
| 1220 |
INFINIPATH_RHF_H_PARITYERR | INFINIPATH_RHF_H_LENERR | |
| 1221 |
INFINIPATH_RHF_H_MTUERR | INFINIPATH_RHF_H_IHDRERR | |
| 1222 |
INFINIPATH_RHF_H_IBERR); |
| 1223 |
char pktmsg[128]; |
| 1224 |
|
| 1225 |
*pktmsg = 0; |
| 1226 |
/* |
| 1227 |
* Tid errors on eager pkts mean we get a headerq overflow, perfectly |
| 1228 |
* safe. Tid errors on expected or other packets means trouble. |
| 1229 |
*/ |
| 1230 |
if (tiderr && rcv_ev->ptype == RCVHQ_RCV_TYPE_EAGER) { |
| 1231 |
struct ips_message_header *p_hdr = rcv_ev->p_hdr; |
| 1232 |
|
| 1233 |
|
| 1234 |
/* Payload dropped - Determine flow for this header and see if |
| 1235 |
* we need to generate a NAK. |
| 1236 |
* |
| 1237 |
* ALL PACKET DROPS IN THIS CATEGORY CAN BE FLAGGED AS DROPPED DUE TO |
| 1238 |
* CONGESTION AS THE EAGER BUFFER IS FULL. |
| 1239 |
* |
| 1240 |
* Possible eager packet type: |
| 1241 |
* |
| 1242 |
* Ctrl Message - ignore |
| 1243 |
* MQ message - Can get flow and see if we need to NAK. |
| 1244 |
* AM message - Can get flow and see if we need to NAK. |
| 1245 |
*/ |
| 1246 |
|
| 1247 |
proto->stats.hdr_overflow++; |
| 1248 |
if (data_err) |
| 1249 |
return 0; |
| 1250 |
|
| 1251 |
switch(p_hdr->sub_opcode) { |
| 1252 |
case OPCODE_SEQ_MQ_HDR: |
| 1253 |
case OPCODE_SEQ_MQ_CTRL: |
| 1254 |
case OPCODE_AM_REQUEST: |
| 1255 |
case OPCODE_AM_REQUEST_NOREPLY: |
| 1256 |
case OPCODE_AM_REPLY: |
| 1257 |
{ |
| 1258 |
ptl_epaddr_flow_t flowid = ips_proto_flowid(p_hdr); |
| 1259 |
struct ips_epstate_entry *epstaddr; |
| 1260 |
struct ips_flow *flow; |
| 1261 |
psmi_seqnum_t sequence_num; |
| 1262 |
int16_t diff; |
| 1263 |
|
| 1264 |
/* Obtain ipsaddr for packet */ |
| 1265 |
epstaddr = ips_epstate_lookup(rcv_ev->recvq->epstate, |
| 1266 |
rcv_ev->p_hdr->commidx + |
| 1267 |
INFINIPATH_KPF_RESERVED_BITS(p_hdr->iph.pkt_flags)); |
| 1268 |
if_pf (epstaddr == NULL || epstaddr->epid != rcv_ev->epid) |
| 1269 |
return 0; /* Unknown packet - drop */ |
| 1270 |
|
| 1271 |
rcv_ev->ipsaddr = epstaddr->ipsaddr; |
| 1272 |
flow = &rcv_ev->ipsaddr->flows[flowid]; |
| 1273 |
sequence_num.val = __be32_to_cpu(p_hdr->bth[2]); |
| 1274 |
diff = (int16_t) (sequence_num.pkt - flow->recv_seq_num.pkt); |
| 1275 |
|
| 1276 |
if (diff >= 0 && !(flow->flags & IPS_FLOW_FLAG_NAK_SEND)) { |
| 1277 |
/* Mark flow as congested and attempt to generate NAK */ |
| 1278 |
flow->flags |= IPS_FLOW_FLAG_GEN_BECN; |
| 1279 |
rcv_ev->ipsaddr->stats.congestion_pkts++; |
| 1280 |
flow->last_seq_num = sequence_num; |
| 1281 |
|
| 1282 |
flow->flags |= IPS_FLOW_FLAG_NAK_SEND; |
| 1283 |
flow->cca_ooo_pkts = 0; |
| 1284 |
ips_proto_send_nak((struct ips_recvhdrq *) rcv_ev->recvq, flow); |
| 1285 |
} |
| 1286 |
|
| 1287 |
/* Safe to process ACKs from header */ |
| 1288 |
ips_proto_process_ack(rcv_ev); |
| 1289 |
} |
| 1290 |
break; |
| 1291 |
default: |
| 1292 |
break; |
| 1293 |
} |
| 1294 |
} |
| 1295 |
else if (tiderr) /* tid error, but not on an eager pkt */ |
| 1296 |
{ |
| 1297 |
psm_ep_t ep_err = PSMI_EP_LOGEVENT; |
| 1298 |
int many_tids = 0, many_epids = 0; |
| 1299 |
uint32_t context_tid_off = |
| 1300 |
__le32_to_cpu(rcv_ev->p_hdr->iph.ver_context_tid_offset); |
| 1301 |
uint16_t tid, offset; |
| 1302 |
uint64_t t_now = get_cycles(); |
| 1303 |
|
| 1304 |
proto->tiderr_cnt++; |
| 1305 |
|
| 1306 |
/* Whether and how we will be logging this event */ |
| 1307 |
if (proto->tiderr_max > 0 && proto->tiderr_cnt >= proto->tiderr_max) |
| 1308 |
ep_err = PSMI_EP_NORETURN; |
| 1309 |
else if (proto->tiderr_warn_interval != UINT64_MAX && |
| 1310 |
proto->tiderr_tnext <= t_now) |
| 1311 |
proto->tiderr_tnext = get_cycles() + proto->tiderr_warn_interval; |
| 1312 |
else |
| 1313 |
ep_err = NULL; |
| 1314 |
|
| 1315 |
if (ep_err != NULL) { |
| 1316 |
if (proto->tiderr_context_tid_off != context_tid_off) { /* many tids */ |
| 1317 |
if (proto->tiderr_context_tid_off != 0) |
| 1318 |
many_tids = 1; |
| 1319 |
proto->tiderr_context_tid_off = context_tid_off; |
| 1320 |
} |
| 1321 |
|
| 1322 |
if (proto->tiderr_epid != rcv_ev->epid) { /* many epids */ |
| 1323 |
if (proto->tiderr_epid != 0) |
| 1324 |
many_epids = 1; |
| 1325 |
proto->tiderr_epid = rcv_ev->epid; |
| 1326 |
} |
| 1327 |
|
| 1328 |
rhf_errnum_string(pktmsg, sizeof(pktmsg), rcv_ev->error_flags); |
| 1329 |
|
| 1330 |
tid = (context_tid_off >> INFINIPATH_I_TID_SHIFT) & |
| 1331 |
INFINIPATH_I_TID_MASK; |
| 1332 |
offset = (context_tid_off>>INFINIPATH_I_OFFSET_SHIFT) & |
| 1333 |
INFINIPATH_I_OFFSET_MASK; |
| 1334 |
|
| 1335 |
psmi_handle_error(ep_err, PSM_EP_DEVICE_FAILURE, |
| 1336 |
"%s with tid=%d,offset=%d,count=%d " |
| 1337 |
"from %s%s %s %s", |
| 1338 |
many_tids ? "Multiple TID Errors" : "TID Error", |
| 1339 |
tid, offset, proto->tiderr_cnt, |
| 1340 |
psmi_epaddr_get_name(rcv_ev->epid), |
| 1341 |
many_epids ? " (and other hosts)" : "", |
| 1342 |
pktmsg, ep_err == PSMI_EP_NORETURN ? |
| 1343 |
"(Terminating...)" : ""); |
| 1344 |
} |
| 1345 |
|
| 1346 |
if (proto->protoexp && rcv_ev->ptype == RCVHQ_RCV_TYPE_EXPECTED) |
| 1347 |
ips_protoexp_handle_tiderr(rcv_ev); |
| 1348 |
} |
| 1349 |
else if (tf_generr) |
| 1350 |
ips_protoexp_handle_tf_generr(rcv_ev); |
| 1351 |
else if (tf_seqerr) |
| 1352 |
ips_protoexp_handle_tf_seqerr(rcv_ev); |
| 1353 |
else if (data_err) { |
| 1354 |
uint8_t op_code = __be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 24 & 0xFF; |
| 1355 |
|
| 1356 |
if (!pkt_verbose_err) { |
| 1357 |
rhf_errnum_string(pktmsg, sizeof(pktmsg), rcv_ev->error_flags); |
| 1358 |
_IPATH_DBG("Error %s pkt type opcode 0x%x at hd=0x%x %s\n", |
| 1359 |
(rcv_ev->ptype == RCVHQ_RCV_TYPE_EAGER) ? "eager" : |
| 1360 |
(rcv_ev->ptype == RCVHQ_RCV_TYPE_EXPECTED) ? "expected" : |
| 1361 |
(rcv_ev->ptype == RCVHQ_RCV_TYPE_NON_KD) ? "non-kd" : |
| 1362 |
"<error>", |
| 1363 |
op_code, rcv_ev->recvq->state->hdrq_head, pktmsg); |
| 1364 |
} |
| 1365 |
|
| 1366 |
if (proto->protoexp && rcv_ev->ptype == RCVHQ_RCV_TYPE_EXPECTED) |
| 1367 |
ips_protoexp_handle_data_err(rcv_ev); |
| 1368 |
} |
| 1369 |
else { /* not a tid or data error -- some other error */ |
| 1370 |
uint8_t op_code = __be32_to_cpu(rcv_ev->p_hdr->bth[0]) >> 24 & 0xFF; |
| 1371 |
|
| 1372 |
if (!pkt_verbose_err) |
| 1373 |
rhf_errnum_string(pktmsg, sizeof(pktmsg), rcv_ev->error_flags); |
| 1374 |
|
| 1375 |
/* else RHFerr decode printed below */ |
| 1376 |
_IPATH_DBG("Error pkt type 0x%x opcode 0x%x at hd=0x%x %s\n", |
| 1377 |
rcv_ev->ptype, op_code, rcv_ev->recvq->state->hdrq_head, pktmsg); |
| 1378 |
} |
| 1379 |
if (pkt_verbose_err) { |
| 1380 |
if(!*pktmsg) |
| 1381 |
rhf_errnum_string(pktmsg, sizeof(pktmsg), rcv_ev->error_flags); |
| 1382 |
ips_proto_show_header(rcv_ev->p_hdr, pktmsg); |
| 1383 |
} |
| 1384 |
|
| 1385 |
return 0; |
| 1386 |
} |
| 1387 |
|
| 1388 |
/* |
| 1389 |
* Only valid packets make it to this point. |
| 1390 |
*/ |
| 1391 |
int __recvpath |
| 1392 |
ips_proto_process_packet_inner(struct ips_recvhdrq_event *rcv_ev) |
| 1393 |
{ |
| 1394 |
struct ips_message_header *p_hdr = rcv_ev->p_hdr; |
| 1395 |
ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr; |
| 1396 |
int ret = IPS_RECVHDRQ_CONTINUE; |
| 1397 |
|
| 1398 |
/* NOTE: Fault injection will currently not work with hardware suppression |
| 1399 |
* on QLE73XX. See TODO below for reason why as we currently do not update |
| 1400 |
* the hardware tidflow table if FI is dropping the packet. |
| 1401 |
* |
| 1402 |
* TODO: We need to look into the packet before dropping it and |
| 1403 |
* if it's an expected packet AND we have hardware suppression then we |
| 1404 |
* need to update the hardware tidflow table and the associated tidrecvc |
| 1405 |
* state to fake having received a packet uptil some point in the window |
| 1406 |
* defined by the loss rate. This way the subsequent err chk will be NAKd |
| 1407 |
* and we can resync the flow with the sender. |
| 1408 |
* |
| 1409 |
* Note: For real errors the hardware generates seq/gen errors which are |
| 1410 |
* handled appropriately by the protocol. |
| 1411 |
*/ |
| 1412 |
|
| 1413 |
if_pf (PSMI_FAULTINJ_ENABLED()) { |
| 1414 |
PSMI_FAULTINJ_STATIC_DECL(fi_recv, "recvlost", 1, IPS_FAULTINJ_RECVLOST); |
| 1415 |
if (psmi_faultinj_is_fault(fi_recv)) |
| 1416 |
return ret; |
| 1417 |
} |
| 1418 |
|
| 1419 |
switch (rcv_ev->ptype) { |
| 1420 |
case RCVHQ_RCV_TYPE_EAGER: |
| 1421 |
#if 0 |
| 1422 |
_IPATH_VDBG("got packet from %d with opcode=%x, seqno=%d\n", |
| 1423 |
p_hdr->commidx, |
| 1424 |
p_hdr->sub_opcode, |
| 1425 |
__be32_to_cpu(p_hdr->bth[2])); |
| 1426 |
#endif |
| 1427 |
|
| 1428 |
switch ( p_hdr->sub_opcode ) { |
| 1429 |
case OPCODE_SEQ_MQ_HDR: |
| 1430 |
case OPCODE_SEQ_MQ_CTRL: |
| 1431 |
ret = _process_mq(rcv_ev); |
| 1432 |
break; |
| 1433 |
|
| 1434 |
case OPCODE_ACK: |
| 1435 |
ips_proto_process_ack(rcv_ev); |
| 1436 |
break; |
| 1437 |
|
| 1438 |
case OPCODE_NAK: |
| 1439 |
_process_nak(rcv_ev); |
| 1440 |
break; |
| 1441 |
|
| 1442 |
case OPCODE_AM_REQUEST: |
| 1443 |
case OPCODE_AM_REQUEST_NOREPLY: |
| 1444 |
case OPCODE_AM_REPLY: |
| 1445 |
ret = ips_proto_am(rcv_ev); |
| 1446 |
break; |
| 1447 |
case OPCODE_FLOW_CCA_BECN: |
| 1448 |
{ |
| 1449 |
struct ips_proto *proto = ipsaddr->proto; |
| 1450 |
struct ips_flow *flow = NULL; |
| 1451 |
psm_protocol_type_t protocol; |
| 1452 |
ptl_epaddr_flow_t flowid; |
| 1453 |
|
| 1454 |
protocol = IPS_FLOWID_GET_PROTO(p_hdr->flowid); |
| 1455 |
flowid = IPS_FLOWID_GET_INDEX(p_hdr->flowid); |
| 1456 |
psmi_assert_always(protocol == PSM_PROTOCOL_GO_BACK_N); |
| 1457 |
flow = &ipsaddr->flows[flowid]; |
| 1458 |
|
| 1459 |
if ((flow->path->epr_ccti + |
| 1460 |
proto->cace[flow->path->epr_sl].ccti_increase) <= |
| 1461 |
proto->ccti_limit) { |
| 1462 |
ips_cca_adjust_rate(flow->path, |
| 1463 |
proto->cace[flow->path->epr_sl].ccti_increase); |
| 1464 |
/* Clear congestion event */ |
| 1465 |
rcv_ev->is_congested &= ~IPS_RECV_EVENT_BECN; |
| 1466 |
} |
| 1467 |
} |
| 1468 |
break; |
| 1469 |
|
| 1470 |
case OPCODE_ERR_CHK: |
| 1471 |
case OPCODE_ERR_CHK_OLD: |
| 1472 |
_process_err_chk((struct ips_recvhdrq *) rcv_ev->recvq, |
| 1473 |
ipsaddr, p_hdr); |
| 1474 |
/* Ignore FECN bit since this is the control path */ |
| 1475 |
rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN; |
| 1476 |
break; |
| 1477 |
|
| 1478 |
case OPCODE_ERR_CHK_GEN: |
| 1479 |
_process_err_chk_gen(ipsaddr, p_hdr); |
| 1480 |
/* Ignore FECN bit since this is the control path */ |
| 1481 |
rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN; |
| 1482 |
break; |
| 1483 |
|
| 1484 |
case OPCODE_ERR_CHK_PLS: /* skip for now */ |
| 1485 |
break; |
| 1486 |
|
| 1487 |
case OPCODE_ERR_CHK_BAD: |
| 1488 |
_process_err_chk_bad(ipsaddr, p_hdr); |
| 1489 |
break; |
| 1490 |
|
| 1491 |
case OPCODE_TIDS_GRANT: |
| 1492 |
ips_protoexp_tid_grant(rcv_ev); |
| 1493 |
/* Ignore FECN bit since this is the control path */ |
| 1494 |
rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN; |
| 1495 |
break; |
| 1496 |
|
| 1497 |
case OPCODE_TIDS_GRANT_ACK: |
| 1498 |
ips_protoexp_tid_grant_ack(rcv_ev); |
| 1499 |
/* Ignore FECN bit since this is the control path */ |
| 1500 |
rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN; |
| 1501 |
break; |
| 1502 |
|
| 1503 |
case OPCODE_TIDS_RELEASE: |
| 1504 |
ret = ips_protoexp_tid_release(rcv_ev); |
| 1505 |
/* Ignore FECN bit since this is the control path */ |
| 1506 |
rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN; |
| 1507 |
break; |
| 1508 |
|
| 1509 |
case OPCODE_TIDS_RELEASE_CONFIRM: |
| 1510 |
ips_protoexp_tid_release_ack(rcv_ev); |
| 1511 |
/* Ignore FECN bit since this is the control path */ |
| 1512 |
rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN; |
| 1513 |
break; |
| 1514 |
|
| 1515 |
case OPCODE_SEQ_MQ_EXPTID: |
| 1516 |
ips_protoexp_data(rcv_ev); |
| 1517 |
break; |
| 1518 |
|
| 1519 |
case OPCODE_SEQ_MQ_EXPTID_UNALIGNED: |
| 1520 |
ips_protoexp_recv_unaligned_data(rcv_ev); |
| 1521 |
break; |
| 1522 |
|
| 1523 |
case OPCODE_CONNECT_REQUEST: |
| 1524 |
case OPCODE_CONNECT_REPLY: |
| 1525 |
case OPCODE_DISCONNECT_REQUEST: |
| 1526 |
case OPCODE_DISCONNECT_REPLY: |
| 1527 |
_process_connect(rcv_ev); |
| 1528 |
break; |
| 1529 |
|
| 1530 |
default: /* skip unsupported opcodes */ |
| 1531 |
_process_unknown_opcode(rcv_ev->proto, p_hdr); |
| 1532 |
break; |
| 1533 |
} /* switch (op_code) */ |
| 1534 |
break; |
| 1535 |
|
| 1536 |
case RCVHQ_RCV_TYPE_EXPECTED: |
| 1537 |
ips_protoexp_data(rcv_ev); |
| 1538 |
break; |
| 1539 |
|
| 1540 |
default: /* unknown frame type */ |
| 1541 |
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, |
| 1542 |
"Unknown frame type %x", rcv_ev->ptype); |
| 1543 |
break; |
| 1544 |
} /* switch (ptype) */ |
| 1545 |
|
| 1546 |
return ret; |
| 1547 |
} |