Debian Codegraph @

ptl_ips/ips_proto_help.h

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #ifndef _IPS_PROTO_HELP_H
36 #define _IPS_PROTO_HELP_H
37
38 #include "ips_recvhdrq.h"
39 #include "ips_proto.h"
40 #include "ipserror.h"
41 #include "psm_mq_internal.h" // psmi_mq_handle_tiny_envelope
42 #include "ptl_ips.h"
43 #include "ips_epstate.h"
44
45 /* Some tunable compile-time options */
46 #define IPS_TINY_PROCESS_MQTINY 1 /* whether mq processing of tiny pkts is
47 done separately from non-tiny packets */
48
49 PSMI_ALWAYS_INLINE(
50 uint8_t
51 ips_flow_gen_ackflags(ips_scb_t *scb, struct ips_flow *flow))
52 {
53 uint32_t diff = (flow->protocol == PSM_PROTOCOL_TIDFLOW) ?
54 (flow->xmit_seq_num.seq - flow->xmit_ack_num.seq) :
55 (flow->xmit_seq_num.pkt - flow->xmit_ack_num.pkt);
56
57 /*
58 * This is currently disabled pending more experimentation. The goal
59 * is to eventually use the FLAG_INTR to tighten the control loop
60 * between two endpoints.
61 */
62 #if 0
63 /* At every 64, request ack w/ interrupt */
64 if ((diff & 0x3f) == 0)
65 scb->flags |= IPS_SEND_FLAG_ACK_REQ |
66 (flow->ipsaddr->flags & SESS_FLAG_HAS_RCVTHREAD) ?
67 IPS_SEND_FLAG_INTR : 0;
68 /* At every 16, request ack */
69 else
70 #endif
71 if (((diff & flow->ack_interval) == 0) || (flow->credits == 1))
72 scb->flags |= IPS_SEND_FLAG_ACK_REQ;
73
74 /* Bottom 8 bits wind up in protocol header fields, other bits
75 * control other aspects of packet composition */
76 return (uint8_t) (scb->flags & IPS_SEND_FLAG_PROTO_OPTS);
77 }
78
79 PSMI_ALWAYS_INLINE(
80 ptl_epaddr_flow_t ips_proto_flowid(struct ips_message_header *p_hdr))
81 {
82 ptl_epaddr_flow_t flowidx = IPS_FLOWID2INDEX(p_hdr->flowid);
83 psmi_assert(flowidx < EP_FLOW_LAST);
84 return flowidx;
85 }
86
87 PSMI_ALWAYS_INLINE(
88 void ips_kdeth_cksum(struct ips_message_header *p_hdr))
89 {
90 /* Compute KDETH checksum */
91 p_hdr->iph.chksum = __cpu_to_le16(
92 (uint16_t) IPATH_LRH_BTH +
93 (uint16_t) (__be16_to_cpu(p_hdr->lrh[2])) -
94 (uint16_t) ((__le32_to_cpu(p_hdr->iph.ver_context_tid_offset)>>16) &
95 LOWER_16_BITS) -
96 (uint16_t) (__le32_to_cpu(p_hdr->iph.ver_context_tid_offset) &
97 LOWER_16_BITS) -
98 (uint16_t) __le16_to_cpu(p_hdr->iph.pkt_flags));
99 }
100
101 PSMI_ALWAYS_INLINE(
102 int ips_do_cksum(struct ips_proto *proto,
103 struct ips_message_header *p_hdr,
104 void *payload,
105 uint32_t paylen,
106 uint32_t *cksum))
107 {
108
109 if_pf ((proto->flags & IPS_PROTO_FLAG_CKSUM) &&
110 (((__le32_to_cpu(p_hdr->iph.ver_context_tid_offset) >> INFINIPATH_I_TID_SHIFT) & INFINIPATH_I_TID_MASK) == IPATH_EAGER_TID_ID) && (p_hdr->mqhdr != MQ_MSG_DATA_BLK) && (p_hdr->mqhdr != MQ_MSG_DATA_REQ_BLK)) {
111
112 uint16_t paywords;
113
114 /* Update the payload words in header */
115 paywords = (sizeof(struct ips_message_header) +
116 paylen + PSM_CRC_SIZE_IN_BYTES) >> BYTE2WORD_SHIFT;
117 p_hdr->lrh[2] = __cpu_to_be16(paywords + SIZE_OF_CRC);
118
119 /* Need to regenerate KDETH checksum after updating payload length */
120 ips_kdeth_cksum(p_hdr);
121
122 *cksum = 0xffffffff;
123
124 /* Checksum header */
125 *cksum = ips_crc_calculate(sizeof(struct ips_message_header),
126 (uint8_t*) p_hdr, *cksum);
127
128 /* Checksum payload (if any) */
129 if (paylen) {
130 psmi_assert_always(payload);
131 *cksum = ips_crc_calculate(paylen, (uint8_t*) payload,
132 *cksum);
133 }
134 }
135
136 return 0;
137 }
138
139 /* Get pbc static rate value for flow for a given message length */
140 PSMI_ALWAYS_INLINE(
141 uint32_t ips_proto_pbc_static_rate(struct ips_flow *flow, uint32_t msgLen))
142 {
143 uint32_t rate = 0;
144
145 /* The PBC rate is based on which HCA type as QLE73XX/QLE72XX have different
146 * mechanism for static rate control. QLE71XX does not even have static
147 * rate control capability.
148 */
149
150 switch(flow->epinfo->ep_hca_type) {
151 case PSMI_HCA_TYPE_QLE73XX:
152 {
153
154 /* Rate = IPD * Time to transmit the packet. The rate value is
155 * programmed into the PBC which counts down at a rate of 500 MHz the
156 * TXE to IBC interface speed (Section 7.8.1). Since time to transmit
157 * depends on our local link speed we need to convert that into the
158 * clock frequency of the TXE in 500 MHz units. To transfer a message of
159 * MSgLen bytes for various local link rates we obtain:
160 *
161 * Link Rate (LinWidth * LinkSpeed) Cycle Count
162 * SDR (10 Gbit/sec) (MsgLen >> 1)
163 * DDR (20 Gbit/sec) (MsgLen >> 2)
164 * QDR (40 Gbit/sec) (MsgLen >> 3)
165 */
166 static uint8_t qle73xx_rate_divisor[IBTA_RATE_120_GBPS + 1] = {
167 [IBTA_RATE_2_5_GBPS] = 0,
168 [IBTA_RATE_5_GBPS] = 0,
169 [IBTA_RATE_10_GBPS] = 1,
170 [IBTA_RATE_20_GBPS] = 2,
171 [IBTA_RATE_30_GBPS] = 2,
172 [IBTA_RATE_40_GBPS] = 3
173 };
174
175 uint32_t time_to_send = (msgLen >>
176 qle73xx_rate_divisor[flow->epinfo->ep_link_rate]);
177 /* IBTA CCA additionally has a shift_field for finer grained control
178 * of IPD (This is bit [14:15] in the CCT entry. For static rate control
179 * this value is always so.
180 */
181 rate = (time_to_send >> flow->path->epr_cca_divisor) *
182 (flow->path->epr_active_ipd);
183
184 /* For QLE73XX the max rate is 0x3FF*/
185 rate = min(rate, 0x3FFF);
186 }
187 break;
188 case PSMI_HCA_TYPE_QLE72XX:
189 /* TODO_CCA: Implement for QLE72XX to take into account the PREVIOUS
190 * messages IPD for this flow/path.
191 */
192 rate = 0;
193 break;
194 default:
195 rate = 0;
196 }
197
198 return rate;
199 }
200
201 /* This is only used for SDMA cases; pbc is really a pointer to
202 * struct ips_pbc_header * or the equivalent un-named structure
203 * in ips_scb */
204 PSMI_ALWAYS_INLINE(
205 void ips_proto_pbc_update(struct ips_proto *proto,
206 struct ips_flow *flow, uint32_t isCtrlMsg,
207 union ipath_pbc *pbc, uint32_t hdrlen,
208 void *payload, uint32_t paylen))
209 {
210 struct ips_spio *ctrl = proto->spioc;
211 struct ips_message_header *p_hdr = (struct ips_message_header*) &pbc[1];
212 int vl = (__be16_to_cpu(p_hdr->lrh[0]) >> LRH_VL_SHIFT) & 0xf;
213 uint32_t static_rate = 0;
214
215 if_pf (!isCtrlMsg && flow->path->epr_active_ipd)
216 static_rate = ips_proto_pbc_static_rate(flow, hdrlen + paylen);
217
218 pbc->qword = 0ULL;
219 pbc->length = __cpu_to_le16( ((hdrlen + paylen) >> 2) + 1);
220 if (ctrl->portnum > 1)
221 pbc->pbcflags |= __cpu_to_le32(vl << __PBC_VLSHIFT |
222 __PBC_IBPORT |
223 static_rate);
224 else
225 pbc->pbcflags |= __cpu_to_le32(vl << __PBC_VLSHIFT |
226 static_rate);
227
228 return;
229 }
230
231 /*
232 * Helpers to extract header information
233 */
234 /* With QLE73XX/QLE72XX, we put context 16 in src_context_ext */
235 #define IPS_HEADER_SRCCONTEXT_GET(msg_hdr) \
236 (((msg_hdr)->src_context) | ((msg_hdr)->src_context_ext<<4))
237
238 #define IPS_HEADER_SRCCONTEXT_SET(msg_hdr,context) do { \
239 (msg_hdr)->src_context = (context) & 0xf; \
240 (msg_hdr)->src_context_ext = (context>>4) & 0x3; \
241 } while (0)
242
243 PSMI_ALWAYS_INLINE(
244 uint32_t ips_proto_dest_context_from_header(struct ips_proto *proto,
245 struct ips_message_header *p_hdr))
246 {
247 uint16_t hca_type;
248 uint32_t dest_context;
249
250 hca_type = PSMI_EPID_GET_HCATYPE(proto->ep->epid);
251
252 dest_context =
253 (__le32_to_cpu(p_hdr->iph.ver_context_tid_offset) >> INFINIPATH_I_CONTEXT_SHIFT) & INFINIPATH_I_CONTEXT_MASK;
254 switch(hca_type) {
255 case PSMI_HCA_TYPE_QLE73XX:
256 dest_context |= ((__be32_to_cpu(p_hdr->bth[1]) & 1) << 4);
257 break;
258 case PSMI_HCA_TYPE_QLE72XX:
259 /* Context 16 is special cased on QLE72XX */
260 dest_context |= ((__be32_to_cpu(p_hdr->bth[1]) & 1) << 4);
261 if (dest_context == 0x1f)
262 dest_context = 16;
263 break;
264 case PSMI_HCA_TYPE_QLE71XX:
265 default:
266 /* This is a no-op. */
267 break;
268 }
269
270 return dest_context;
271 }
272
273 PSMI_ALWAYS_INLINE(
274 void ips_proto_hdr(ips_scb_t *scb,
275 struct ips_epinfo *epinfo,
276 struct ips_epinfo_remote *epr,
277 struct ips_flow *flow,
278 uint32_t paywords,
279 uint32_t extra_bytes,
280 uint16_t kpf_flags,
281 uint8_t flags))
282 {
283 struct ips_message_header *p_hdr = &scb->ips_lrh;
284
285 /*
286 * This scb has been used by this connection last time,
287 * so some of the header fields are already set.
288 */
289 if (scb->flow == flow && scb->epaddr == flow->ipsaddr) {
290 p_hdr->bth[2] = __cpu_to_be32(flow->xmit_seq_num.psn);
291 p_hdr->flags = flags;
292 p_hdr->ack_seq_num = flow->recv_seq_num.psn;
293
294 /* check if extra bytes is changed */
295 if (scb->extra_bytes != extra_bytes) {
296 p_hdr->bth[0] =
297 __cpu_to_be32((IPATH_OPCODE_USER1 << BTH_OPCODE_SHIFT) +
298 (extra_bytes << BTH_EXTRA_BYTE_SHIFT) +
299 flow->path->epr_pkey);
300 scb->extra_bytes = extra_bytes;
301 }
302
303 /* If header is exactly the same */
304 if (scb->tid == IPATH_EAGER_TID_ID &&
305 scb->pkt_flags == kpf_flags &&
306 scb->payload_bytes == scb->payload_size) {
307 return;
308 }
309
310 /* context, version, and TID are already known to be in range, no
311 * masking needed; offset in low INFINIPATH_I_OFFSET_MASK bits */
312 p_hdr->iph.ver_context_tid_offset = __cpu_to_le32(
313 (IPS_PROTO_VERSION << INFINIPATH_I_VERS_SHIFT) +
314 (epr->epr_pkt_context << INFINIPATH_I_CONTEXT_SHIFT) +
315 (scb->tid << INFINIPATH_I_TID_SHIFT) +
316 (scb->offset >> 2)); // convert from byte to word offset
317
318 p_hdr->lrh[2] = __cpu_to_be16(paywords + SIZE_OF_CRC);
319 p_hdr->iph.pkt_flags = __cpu_to_le16(kpf_flags);
320
321 ips_kdeth_cksum(p_hdr); // Generate KDETH checksum
322
323 scb->pkt_flags = kpf_flags;
324 scb->payload_bytes = scb->payload_size;
325
326 return;
327 }
328
329 p_hdr->lrh[0] =
330 __cpu_to_be16(IPATH_LRH_BTH |
331 (flow->sl << 4) | /* SL for flow */
332 /* VL for flow */ (flow->path->proto->sl2vl[flow->sl] << LRH_VL_SHIFT));
333 p_hdr->lrh[1] = flow->path->epr_dlid;
334 p_hdr->lrh[2] = __cpu_to_be16(paywords + SIZE_OF_CRC);
335 p_hdr->lrh[3] = flow->path->epr_slid;
336
337 p_hdr->bth[0] =
338 __cpu_to_be32((IPATH_OPCODE_USER1 << BTH_OPCODE_SHIFT) +
339 (extra_bytes << BTH_EXTRA_BYTE_SHIFT) +
340 flow->path->epr_pkey);
341 p_hdr->bth[1] = __cpu_to_be32(epr->epr_qp);
342 p_hdr->bth[2] = __cpu_to_be32(flow->xmit_seq_num.psn);
343 p_hdr->commidx = (uint16_t) epr->epr_commidx_to;
344
345 /* context, version, and TID are already known to be in range, no
346 * masking needed; offset in low INFINIPATH_I_OFFSET_MASK bits */
347 p_hdr->iph.ver_context_tid_offset = __cpu_to_le32(
348 (IPS_PROTO_VERSION << INFINIPATH_I_VERS_SHIFT) +
349 (epr->epr_pkt_context << INFINIPATH_I_CONTEXT_SHIFT) +
350 (scb->tid << INFINIPATH_I_TID_SHIFT) +
351 (scb->offset >> 2)); // convert from byte to word offset
352 p_hdr->iph.pkt_flags = __cpu_to_le16(kpf_flags);
353
354 ips_kdeth_cksum(p_hdr); // Generate KDETH checksum
355
356 p_hdr->flags = flags;
357 p_hdr->flowid = flow->flowid;
358 p_hdr->ack_seq_num = flow->recv_seq_num.psn;
359 IPS_HEADER_SRCCONTEXT_SET(p_hdr, epinfo->ep_context);
360 p_hdr->src_subcontext = epinfo->ep_subcontext;
361 p_hdr->dst_subcontext = epr->epr_subcontext;
362
363 scb->extra_bytes = extra_bytes;
364 scb->pkt_flags = kpf_flags;
365 scb->payload_bytes = scb->payload_size;
366 scb->flow = flow;
367 scb->epaddr = flow->ipsaddr;
368
369 return;
370 }
371
372 /*
373 * Assumes that the following fields are already set in scb:
374 * payload
375 * payload_size
376 * flags
377 */
378 PSMI_INLINE(
379 void
380 ips_scb_prepare_flow_inner(ips_scb_t *scb,
381 struct ips_epinfo *epinfo,
382 struct ips_epinfo_remote *epr,
383 struct ips_flow *flow))
384 {
385 uint32_t extra_bytes;
386 uint32_t tot_paywords;
387 uint16_t pkt_flags = IPS_EPSTATE_COMMIDX_PACK(epr->epr_commidx_to);
388
389 extra_bytes = scb->payload_size & 3;
390 if (extra_bytes) {
391 extra_bytes = 4 - extra_bytes;
392 scb->payload_size += extra_bytes;
393 }
394 tot_paywords = (sizeof(struct ips_message_header) + scb->payload_size)
395 >> BYTE2WORD_SHIFT;
396 pkt_flags |= (scb->flags & IPS_SEND_FLAG_INTR) ? INFINIPATH_KPF_INTR : 0;
397 pkt_flags |= (scb->flags & IPS_SEND_FLAG_HDR_SUPPRESS) ?
398 INFINIPATH_KPF_HDRSUPP : 0;
399
400 ips_proto_hdr(scb, epinfo, epr, flow,
401 tot_paywords, extra_bytes,
402 pkt_flags, ips_flow_gen_ackflags(scb, flow));
403
404 scb->ack_timeout = flow->path->epr_timeout_ack;
405 scb->abs_timeout = TIMEOUT_INFINITE;
406 scb->flags |= IPS_SEND_FLAG_PENDING;
407
408 if (flow->protocol == PSM_PROTOCOL_TIDFLOW) {
409 flow->xmit_seq_num.seq += scb->nfrag;
410 scb->seq_num = flow->xmit_seq_num;
411 scb->seq_num.seq--;
412 } else {
413 flow->xmit_seq_num.pkt += scb->nfrag;
414 scb->seq_num = flow->xmit_seq_num;
415 scb->seq_num.pkt--;
416 }
417
418 return;
419 }
420
421 PSMI_ALWAYS_INLINE(
422 psm_epid_t
423 ips_epid_from_phdr(const uint16_t lmc_mask,
424 const struct ips_message_header *p_hdr))
425 {
426 uint16_t lid = __be16_to_cpu(p_hdr->lrh[3]) & lmc_mask;
427 uint16_t context = (uint16_t) IPS_HEADER_SRCCONTEXT_GET(p_hdr);
428 uint16_t subcontext = (uint16_t) p_hdr->src_subcontext;
429
430 return PSMI_EPID_PACK(lid, context, subcontext);
431 }
432
433 PSMI_ALWAYS_INLINE(
434 void
435 ips_epaddr_stats_send(struct ptl_epaddr *ptladdr, uint8_t msgtype))
436 {
437 switch (msgtype) {
438 case OPCODE_ACK:
439 break;
440 case OPCODE_TIDS_GRANT:
441 ptladdr->stats.tids_grant_send++;
442 break;
443 case OPCODE_ERR_CHK:
444 case OPCODE_ERR_CHK_GEN:
445 ptladdr->stats.err_chk_send++;
446 break;
447 case OPCODE_NAK:
448 ptladdr->stats.nak_send++;
449 break;
450 case OPCODE_CONNECT_REQUEST:
451 ptladdr->stats.connect_req++;
452 break;
453 case OPCODE_DISCONNECT_REQUEST:
454 ptladdr->stats.disconnect_req++;
455 break;
456 default:
457 break;
458 }
459 return;
460 }
461
462 /*
463 * Exported there solely for inlining is_expected_or_nak and mq_tiny handling
464 */
465 extern
466 psm_error_t ips_proto_send_ctrl_message(struct ips_flow *flow,
467 uint8_t message_type,
468 uint32_t *msg_queue_mask,
469 void *payload);
470
471 PSMI_ALWAYS_INLINE(
472 void
473 ips_proto_send_ack(struct ips_recvhdrq *recvq, struct ips_flow *flow))
474 {
475 if_pt (recvq->proto->flags & IPS_PROTO_FLAG_COALESCE_ACKS) {
476 if (flow->flags & IPS_FLOW_FLAG_PENDING_NAK) {
477 flow->flags &= ~IPS_FLOW_FLAG_PENDING_NAK; /* ACK clears NAK */
478 }
479 else if (!(flow->flags & IPS_FLOW_FLAG_PENDING_ACK)) {
480 SLIST_INSERT_HEAD(&recvq->pending_acks, flow, next);
481 }
482
483 flow->flags |= IPS_FLOW_FLAG_PENDING_ACK;
484 }
485 else {
486 /* Coalesced ACKs disabled. Send ACK immediately */
487 ips_proto_send_ctrl_message(flow, OPCODE_ACK,
488 &flow->ipsaddr->ctrl_msg_queued, NULL);
489 }
490 }
491
492 PSMI_ALWAYS_INLINE(
493 void
494 ips_proto_send_nak(struct ips_recvhdrq *recvq, struct ips_flow *flow))
495 {
496 if_pt (recvq->proto->flags & IPS_PROTO_FLAG_COALESCE_ACKS) {
497 if (flow->flags & IPS_FLOW_FLAG_PENDING_ACK) {
498 flow->flags &= ~IPS_FLOW_FLAG_PENDING_ACK; /* NAK clears ACK */
499 }
500 else if (!(flow->flags & IPS_FLOW_FLAG_PENDING_NAK)) {
501 SLIST_INSERT_HEAD(&recvq->pending_acks, flow, next);
502 }
503
504 flow->flags |= IPS_FLOW_FLAG_PENDING_NAK;
505 }
506 else {
507 /* Coalesced ACKs disabled. Send NAK immediately */
508 ips_proto_send_ctrl_message(flow, OPCODE_NAK,
509 &flow->ipsaddr->ctrl_msg_queued, NULL);
510 }
511 }
512
513 /* return 1 if packet is next expected in flow
514 * return 0 if packet is not next expected in flow (and nak packet).
515 */
516 PSMI_ALWAYS_INLINE(
517 int
518 ips_proto_is_expected_or_nak(struct ips_recvhdrq_event *rcv_ev))
519 {
520 ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr;
521 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
522 ptl_epaddr_flow_t flowid = ips_proto_flowid(p_hdr);
523 struct ips_flow *flow = &ipsaddr->flows[flowid];
524 psmi_seqnum_t sequence_num;
525
526 psmi_assert((flowid == EP_FLOW_GO_BACK_N_PIO) ||
527 (flowid == EP_FLOW_GO_BACK_N_DMA) ||
528 (flowid == EP_FLOW_GO_BACK_N_AM_REQ) ||
529 (flowid == EP_FLOW_GO_BACK_N_AM_RSP)
530 );
531
532 /* If packet faced congestion generate BECN in NAK. */
533 if_pf ((rcv_ev->is_congested & IPS_RECV_EVENT_FECN) &&
534 ((flow->cca_ooo_pkts & 0xf) == 0)) {
535 /* Generate a BECN for every 16th OOO packet marked with a FECN. */
536 flow->flags |= IPS_FLOW_FLAG_GEN_BECN;
537 flow->cca_ooo_pkts++;
538 ipsaddr->stats.congestion_pkts++;
539 rcv_ev->is_congested &= ~IPS_RECV_EVENT_FECN; /* Clear FECN event */
540 }
541
542 sequence_num.val = __be32_to_cpu(p_hdr->bth[2]);
543 if_pf (flow->recv_seq_num.pkt != sequence_num.pkt) {
544 int16_t diff = (int16_t) (sequence_num.pkt - flow->last_seq_num.pkt);
545
546 if (diff < 0)
547 return 0;
548
549 flow->cca_ooo_pkts = diff;
550 if (flow->cca_ooo_pkts > flow->ack_interval) {
551 ipsaddr->stats.congestion_pkts++;
552 flow->flags |= IPS_FLOW_FLAG_GEN_BECN;
553 _IPATH_CCADBG("BECN Generation. Expected: %d, Got: %d.\n", flow->recv_seq_num.pkt, sequence_num.pkt);
554 }
555 flow->last_seq_num = sequence_num;
556
557 if (!(flow->flags & IPS_FLOW_FLAG_NAK_SEND)) {
558 /* Queue/Send NAK to peer */
559 ips_proto_send_nak((struct ips_recvhdrq *) rcv_ev->recvq, flow);
560 flow->flags |= IPS_FLOW_FLAG_NAK_SEND;
561 flow->cca_ooo_pkts = 0;
562 }
563 else if (flow->flags & IPS_FLOW_FLAG_GEN_BECN) {
564 /* Send Control message to throttle flow. Will clear flow flag and
565 * reset cca_ooo_pkts.
566 */
567 ips_proto_send_ctrl_message(flow, OPCODE_FLOW_CCA_BECN,
568 &flow->ipsaddr->ctrl_msg_queued,
569 NULL);
570 }
571
572 return 0;
573 }
574 else {
575 flow->flags &= ~IPS_FLOW_FLAG_NAK_SEND;
576
577 flow->last_seq_num = sequence_num;
578 flow->recv_seq_num.pkt += 1;
579 flow->cca_ooo_pkts = 0;
580 return 1;
581 }
582 }
583
584 /*
585 * Return value:
586 * 1: in order message;
587 * 0: out of order, no touch;
588 * -1: out of order, buffered in outoforder queue.
589 */
590 PSMI_ALWAYS_INLINE(
591 int
592 ips_proto_check_msg_order(psm_epaddr_t epaddr,
593 struct ips_flow *flow, struct ips_message_header *p_hdr))
594 {
595 uint16_t msg_seqnum = (uint16_t)(flow->last_seq_num.msg +
596 ((p_hdr->ack_seq_num>>8)&0xff00));
597
598 if (msg_seqnum != epaddr->mctxt_master->mctxt_recv_seqnum) {
599 flow->msg_ooo_toggle = !flow->msg_ooo_toggle;
600
601 if (flow->msg_ooo_toggle) {
602 flow->recv_seq_num.pkt -= 1;
603 flow->msg_ooo_seqnum = msg_seqnum;
604 return 0;
605 }
606
607 psmi_assert(msg_seqnum == flow->msg_ooo_seqnum);
608 return -1;
609 }
610
611 flow->msg_ooo_toggle = 0;
612 epaddr->mctxt_master->mctxt_recv_seqnum++;
613 return 1;
614 }
615
616 #if IPS_TINY_PROCESS_MQTINY
617 PSMI_ALWAYS_INLINE(
618 int
619 ips_proto_process_mq_tiny(const struct ips_recvhdrq_event *rcv_ev))
620 {
621 ips_epaddr_t *ipsaddr = rcv_ev->ipsaddr;
622 psm_epaddr_t epaddr = ipsaddr->epaddr;
623 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
624 ptl_epaddr_flow_t flowid = ips_proto_flowid(p_hdr);
625 struct ips_flow *flow = &ipsaddr->flows[flowid];
626 int ret = IPS_RECVHDRQ_CONTINUE;
627
628 if (ips_proto_is_expected_or_nak((struct ips_recvhdrq_event*) rcv_ev)) {
629 ret = ips_proto_check_msg_order(epaddr, flow, p_hdr);
630 if (ret == 0) return IPS_RECVHDRQ_OOO;
631 if (ret == -1) {
632 psmi_mq_handle_envelope_outoforder(ipsaddr->proto->mq,
633 (uint16_t) p_hdr->mqhdr,
634 epaddr, flow->msg_ooo_seqnum,
635 p_hdr->data[0].u64, /* tag */
636 epaddr->xmit_egrlong, /* place hold only */
637 (uint32_t) p_hdr->hdr_dlen,
638 (void *) &p_hdr->data[1],
639 (uint32_t) p_hdr->hdr_dlen);
640 ret = IPS_RECVHDRQ_BREAK;
641 } else {
642 psmi_mq_handle_tiny_envelope(
643 ipsaddr->proto->mq,
644 epaddr, p_hdr->data[0].u64, /* tag */
645 (void *) &p_hdr->data[1],
646 (uint32_t) p_hdr->hdr_dlen);
647 if (epaddr->mctxt_master->outoforder_c) {
648 psmi_mq_handle_outoforder_queue(epaddr->mctxt_master);
649 }
650 ret = IPS_RECVHDRQ_CONTINUE;
651 }
652 if ((p_hdr->flags & IPS_SEND_FLAG_ACK_REQ) ||
653 (flow->flags & IPS_FLOW_FLAG_GEN_BECN))
654 ips_proto_send_ack((struct ips_recvhdrq *) rcv_ev->recvq, flow);
655 }
656
657 ips_proto_process_ack((struct ips_recvhdrq_event *) rcv_ev);
658 return ret;
659 }
660 #endif
661
662 PSMI_INLINE(
663 int
664 ips_proto_process_packet(const struct ips_recvhdrq_event *rcv_ev))
665 {
666 #if IPS_TINY_PROCESS_MQTINY
667 if (rcv_ev->p_hdr->sub_opcode == OPCODE_SEQ_MQ_HDR) {
668 psmi_assert(rcv_ev->ptype == RCVHQ_RCV_TYPE_EAGER);
669 return ips_proto_process_mq_tiny(rcv_ev);
670 }
671 else
672 #endif
673 return ips_proto_process_packet_inner((struct ips_recvhdrq_event *) rcv_ev);
674 }
675
676 #if PSMI_PLOCK_DISABLED
677 #define ips_ptladdr_lock(ipsaddr) \
678 if (((ipsaddr)->flags & SESS_FLAG_LOCK_SESS)) \
679 pthread_mutex_lock(&(ipsaddr)->sesslock)
680
681 #define ips_ptladdr_unlock(ipsaddr) \
682 if (((ipsaddr)->flags & SESS_FLAG_LOCK_SESS)) \
683 pthread_mutex_unlock(&(ipsaddr)->sesslock)
684 #else
685 #define ips_ptladdr_lock(ipsaddr)
686 #define ips_ptladdr_unlock(ipsaddr)
687 #endif
688
689 /*
690 * Breaks header encapsulation but needed in mq sends so we can pay
691 * "near-equal" attention to putting sends on the wire and servicing the
692 * receive queue.
693 */
694
695 PSMI_ALWAYS_INLINE(
696 psm_error_t
697 ips_recv_progress_if_busy(ptl_t *ptl, psm_error_t err))
698 {
699 if (err == PSM_EP_NO_RESOURCES) {
700 ptl->ctl->ep_poll(ptl, 0);
701 return PSM_OK;
702 }
703 else
704 return err;
705 }
706
707 /* Find next lowest power of a two for a 32 bit number*/
708 PSMI_ALWAYS_INLINE(
709 unsigned int
710 ips_next_low_pow2(unsigned int v))
711 {
712
713 const unsigned int b[] = {0x2, 0xC, 0xF0, 0xFF00, 0xFFFF0000};
714 const unsigned int S[] = {1, 2, 4, 8, 16};
715 register unsigned int r = 1;
716 int i;
717
718 for (i = 4; i >= 0; i--)
719 {
720 if (v & b[i])
721 {
722 v >>= S[i];
723 r <<= S[i];
724 }
725 }
726
727 return r;
728 }
729
730 PSMI_ALWAYS_INLINE(
731 ips_path_rec_t *ips_select_path(struct ips_proto *proto,
732 ips_path_type_t path_type,
733 ips_epaddr_t *ipsaddr))
734 {
735 uint32_t path_idx;
736
737 if (proto->flags & IPS_PROTO_FLAG_PPOLICY_ADAPTIVE) {
738 /* If dispersive routes are configured then select the routes in round
739 * robin order. We may want to use congestion information to select the
740 * least lightly loaded path.
741 */
742 path_idx = ipsaddr->epr.epr_next_path[path_type];
743 if (++ipsaddr->epr.epr_next_path[path_type] >=
744 ipsaddr->epr.epr_num_paths[path_type])
745 ipsaddr->epr.epr_next_path[path_type] = 0;
746 }
747 else if (proto->flags & IPS_PROTO_FLAG_PPOLICY_STATIC_DST)
748 path_idx = /* Key on destination context */
749 ipsaddr->epr.epr_context % ipsaddr->epr.epr_num_paths[path_type];
750 else if (proto->flags & IPS_PROTO_FLAG_PPOLICY_STATIC_SRC)
751 path_idx = /* Key off src context */
752 ipsaddr->proto->ep->context.base_info.spi_context % ipsaddr->epr.epr_num_paths[path_type];
753 else /* Base LID routed - Default in Infinipath 2.5 (Oct 09). */
754 path_idx = 0;
755
756 return ipsaddr->epr.epr_path[path_type][path_idx];
757 }
758
759 #endif /* _IPS_PROTO_HELP_H */