Debian Codegraph @

ptl_ips/ips_proto_mq.c

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "psm_user.h"
36 #include "ipserror.h"
37 #include "ips_proto.h"
38 #include "ips_proto_internal.h"
39
40 #define MQ_NUM_MTUS(size,mtu) (((size) + (mtu) - 1) / (mtu))
41 #define MQ_EGRLONG_ENABLE_MULTIFLOW 0
42
43 PSMI_NEVER_INLINE(
44 ips_scb_t * __sendpath
45 ips_poll_scb(struct ips_proto *proto,
46 int npkts, int len, uint32_t flags, int istiny))
47 {
48 ips_scb_t *scb = NULL;
49 psmi_assert(npkts > 0);
50 psm_error_t err;
51
52 proto->stats.scb_egr_unavail_cnt++;
53
54 PSMI_BLOCKUNTIL(proto->ep,err,
55 ((scb = istiny ?
56 ips_scbctrl_alloc_tiny(&proto->scbc_egr) :
57 ips_scbctrl_alloc(&proto->scbc_egr, npkts, len, flags)) != NULL));
58 psmi_assert(scb != NULL);
59 return scb;
60 }
61
62 PSMI_ALWAYS_INLINE(
63 ips_scb_t *
64 mq_alloc_tiny(struct ips_proto *proto))
65 {
66 ips_scb_t* scb = ips_scbctrl_alloc_tiny(&proto->scbc_egr);
67 // common case should branch right through
68 if_pt (scb != NULL)
69 return scb;
70 else
71 return ips_poll_scb(proto, 1, 0, 0, 1);
72 }
73
74 PSMI_ALWAYS_INLINE(
75 ips_scb_t *
76 mq_alloc_pkts(struct ips_proto *proto, int npkts, int len, uint32_t flags))
77 {
78 psmi_assert(npkts > 0);
79 ips_scb_t* scb = ips_scbctrl_alloc(&proto->scbc_egr, npkts, len, flags);
80 if_pt (scb != NULL) {
81 return scb;
82 }
83 else {
84 return ips_poll_scb(proto, npkts, len, flags, 0 /* not tiny scb */);
85 }
86 }
87
88 static
89 int __recvpath
90 ips_proto_mq_eager_complete(void *reqp, uint32_t nbytes)
91 {
92 psm_mq_req_t req = (psm_mq_req_t)reqp;
93
94 req->send_msgoff += nbytes;
95 if (req->send_msgoff == req->send_msglen) {
96 req->state = MQ_STATE_COMPLETE;
97 mq_qq_append(&req->mq->completed_q, req);
98 }
99 return IPS_RECVHDRQ_CONTINUE;
100 }
101
102 static
103 int __recvpath
104 ips_proto_mq_rv_complete(void *reqp)
105 {
106 psm_mq_req_t req = (psm_mq_req_t) reqp;
107 psmi_mq_handle_rts_complete(req);
108
109 return IPS_RECVHDRQ_CONTINUE;
110 }
111
112 static
113 void __recvpath
114 ips_proto_mq_rv_complete_exp(void *reqp)
115 {
116 ips_proto_mq_rv_complete(reqp);
117 return;
118 }
119
120 extern psm_error_t ips_ptl_poll(ptl_t *ptl, int _ignored);
121
122 /*
123 * Mechanism to capture PIO-ing or DMA-ing the MQ message envelope
124 *
125 * Recoverable errors:
126 * PSM_OK: If PIO, envelope is sent.
127 * If DMA, all queued up packets on flow were flushed.
128 *
129 * Recoverable errors converted to PSM_OK just before return:
130 * PSM_OK_NO_PROGRESS: DMA-only, flushed 1 but not all queued packets.
131 * PSM_EP_NO_RESOURCES:
132 * If PIO, no pio available or cable currently pulled.
133 * If DMA, can be that no scb's available to handle unaligned packets
134 * or writev returned a recoverable error (no mem for
135 * descriptors, dma interrupted or no space left in dma queue).
136 *
137 * Unrecoverable errors (PIO or DMA).
138 * PSM_EP_DEVICE_FAILURE: Unexpected error calling writev(), chip failure,
139 * rxe/txe parity error.
140 * PSM_EP_NO_NETWORK: No network, no lid, ...
141 */
142 PSMI_ALWAYS_INLINE(
143 psm_error_t
144 ips_mq_send_envelope(struct ips_proto *proto, psm_epaddr_t mepaddr,
145 ips_epaddr_t *ipsaddr, struct ips_scb *scb, int do_flush))
146 {
147 psm_error_t err = PSM_OK;
148 struct ips_flow *flow = &ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO];
149
150 if_pf (proto->flags & IPS_PROTO_FLAG_MQ_ENVELOPE_SDMA) {
151 flow = &ipsaddr->flows[EP_FLOW_GO_BACK_N_DMA];
152
153 if_pt (ips_scb_length(scb)) /* For DMA envelope need local completion */
154 ips_scb_flags(scb) |= IPS_SEND_FLAG_WAIT_SDMA;
155 }
156
157 flow->xmit_seq_num.msg = mepaddr->mctxt_send_seqnum&0xff;
158 flow->recv_seq_num.msg = (mepaddr->mctxt_send_seqnum>>8)&0xff;
159 mepaddr->mctxt_send_seqnum++;
160
161 flow->fn.xfer.enqueue(flow, scb);
162
163 if ((flow->transfer == PSM_TRANSFER_PIO) ||
164 (flow->transfer == PSM_TRANSFER_DMA && do_flush))
165 err = flow->fn.xfer.flush(flow, NULL);
166
167 if (do_flush)
168 err = ips_recv_progress_if_busy(ipsaddr->ptl, err);
169
170 PSMI_BLOCKUNTIL(proto->ep,err, (scb->flags&IPS_SEND_FLAG_PENDING) == 0);
171
172 /* As per the PSM error model (or lack thereof), PSM clients expect to see
173 * only PSM_OK as a recoverable error */
174 if (err == PSM_EP_NO_RESOURCES || err == PSM_OK_NO_PROGRESS)
175 err = PSM_OK;
176 return err;
177 }
178
179 /*
180 * We don't use message striping for middle message protocol,
181 * Tests on sandy-bridge two HCAs show lower bandwidth if
182 * message striping is used.
183 */
184 void __sendpath
185 ips_mq_send_payload(psm_epaddr_t epaddr, psmi_egrid_t egrid,
186 void *ubuf, uint32_t len, uint32_t offset,
187 psm_mq_req_t req, uint32_t flags)
188 {
189 psm_error_t err;
190
191 ips_scb_t *scb;
192 uintptr_t buf = (uintptr_t) ubuf;
193 uint32_t nbytes_left = len;
194 uint32_t pktlen, frag_size;
195 ips_epaddr_t *ipsaddr;
196 struct ips_proto *proto;
197 int is_blocking = !!(req == NULL);
198 ptl_epaddr_flow_t flowid =
199 (flags & IPS_PROTO_FLAG_MQ_EAGER_SDMA) ?
200 EP_FLOW_GO_BACK_N_DMA : EP_FLOW_GO_BACK_N_PIO;
201 struct ips_flow *flow;
202
203 psmi_assert(len > 0);
204 ipsaddr = epaddr->ptladdr;
205 proto = ipsaddr->proto;
206 flow = &ipsaddr->flows[flowid];
207 frag_size = flow->frag_size;
208
209 if (!(flags & IPS_PROTO_FLAG_MQ_EAGER_SDMA)) goto spio;
210
211 psmi_assert(req != NULL);
212 pktlen = len;
213 /* The payload size is limited by the pbc.length field which is 16 bits in
214 * DWORD, including both message header and payload. This translates to
215 * less than 256K payload. So 128K is used. */
216 if (pktlen > 131072) pktlen = 131072;
217
218 do {
219 scb = mq_alloc_pkts(proto, 1, 0, 0);
220 psmi_assert(scb != NULL);
221
222 #if 0
223 /* turn on to use single frag-size packet */
224 pktlen = min(frag_size, nbytes_left);
225 #else
226 pktlen = min(pktlen, nbytes_left);
227 #endif
228 ips_scb_length(scb) = pktlen;
229 ips_scb_mqhdr(scb) = MQ_MSG_DATA_BLK;
230 ips_scb_mqparam(scb).u32w0 = egrid.egr_data;
231 ips_scb_mqparam(scb).u32w1 = offset;
232 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
233 ips_scb_buffer(scb) = (void *) buf;
234
235 buf += pktlen;
236 offset += pktlen;
237 nbytes_left -= pktlen;
238
239 if (nbytes_left == 0) {
240 ips_scb_cb(scb) = ips_proto_mq_eager_complete;
241 ips_scb_cb_param(scb) = req;
242 ips_scb_flags(scb) |= IPS_SEND_FLAG_ACK_REQ;
243 } else {
244 req->send_msgoff += pktlen;
245 }
246
247 scb->nfrag = (pktlen + frag_size - 1) / frag_size;
248 scb->frag_size = frag_size;
249
250 /* attach checksum if enabled, this matches what is done for tid-sdma */
251 if (proto->flags & IPS_PROTO_FLAG_CKSUM && !nbytes_left) {
252 uint32_t cksum = 0xffffffff;
253 cksum = ips_crc_calculate(len, (uint8_t *)(buf-len), cksum);
254 scb->ips_lrh.data[0].u32w0 = cksum;
255 scb->ips_lrh.data[0].u32w1 = offset - len;
256 }
257
258 flow->fn.xfer.enqueue(flow, scb);
259
260 ips_scb_flags(scb) |= IPS_SEND_FLAG_WAIT_SDMA;
261
262 if (nbytes_left == 0) {
263 err = flow->fn.xfer.flush(flow, NULL);
264 if (err == PSM_EP_NO_RESOURCES || err == PSM_OK_NO_PROGRESS) {
265 err = ips_recv_progress_if_busy
266 (ipsaddr->ptl, PSM_EP_NO_RESOURCES);
267 }
268 }
269
270 } while (nbytes_left);
271
272 return;
273
274 spio:
275 do {
276 /*
277 * Each flow/proto uses its own scb. If a scb from one proto is
278 * used by another proto, there is a teardown problem, where
279 * a proto deallocates the scb still in use by another proto.
280 */
281 pktlen = min(frag_size, nbytes_left);
282 scb = mq_alloc_pkts(proto, 1, pktlen, is_blocking ? IPS_SCB_FLAG_ADD_BUFFER : 0);
283 psmi_assert(scb != NULL);
284
285 ips_scb_length(scb) = pktlen;
286 ips_scb_mqhdr(scb) = MQ_MSG_DATA;
287 ips_scb_mqparam(scb).u32w0 = egrid.egr_data;
288 ips_scb_mqparam(scb).u32w1 = offset;
289 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
290
291 _IPATH_VDBG("payload=%p, thislen=%d, frag_size=%d, nbytes_left=%d\n",
292 (void *) buf, pktlen, frag_size, nbytes_left);
293 if (!is_blocking) /* non-blocking, send from user's buffer */
294 ips_scb_buffer(scb) = (void *) buf;
295 else /* blocking, copy to bounce buffer */
296 psmi_mq_mtucpy(ips_scb_buffer(scb), (void *) buf, pktlen);
297
298 buf += pktlen;
299 offset += pktlen;
300 nbytes_left -= pktlen;
301
302 if (nbytes_left == 0) { /* last packet */
303 if (!is_blocking) {
304 /* non-blocking mode, need completion */
305 ips_scb_cb(scb) = ips_proto_mq_eager_complete;
306 ips_scb_cb_param(scb) = req;
307 }
308 ips_scb_flags(scb) |= IPS_SEND_FLAG_ACK_REQ;
309 } else {
310 if (!is_blocking) {
311 req->send_msgoff += pktlen;
312 }
313 }
314
315 flow->fn.xfer.enqueue(flow, scb);
316
317 /* we need to flush the pending queue */
318 err = flow->fn.xfer.flush(flow, NULL);
319 err = ips_recv_progress_if_busy(ipsaddr->ptl, err);
320
321 } while (nbytes_left);
322
323 return;
324 }
325
326
327 PSMI_ALWAYS_INLINE(
328 void
329 ips_shortcpy(void* vdest, const void* vsrc, uint32_t nchars)
330 )
331 {
332 #ifdef __MIC__
333 memcpy(vdest, vsrc, nchars);
334 #else
335 unsigned char *dest = vdest;
336 const unsigned char *src = vsrc;
337
338 if(nchars>>2)
339 ipath_dwordcpy((uint32_t*)dest, (uint32_t*)src, nchars>>2);
340 dest += (nchars>>2)<<2;
341 src += (nchars>>2)<<2;
342 switch (nchars&0x03) {
343 case 3: *dest++ = *src++;
344 case 2: *dest++ = *src++;
345 case 1: *dest++ = *src++;
346 }
347 #endif
348 return;
349 }
350
351 static __sendpath
352 psm_error_t
353 ips_ptl_mq_rndv(psm_mq_req_t req, psm_epaddr_t mepaddr, ips_epaddr_t *ipsaddr,
354 const void *buf, uint32_t len)
355 {
356 ips_scb_t *scb;
357 psm_error_t err = PSM_OK;
358 struct ips_proto *proto = ipsaddr->proto;
359
360 req->buf = (void *) buf;
361 req->buf_len = len;
362 req->send_msglen = len;
363 req->send_msgoff = 0;
364 req->recv_msgoff = 0;
365 req->rts_peer = ipsaddr->epaddr;
366
367 scb = mq_alloc_tiny(proto);
368
369 /* If the expected tid protocol is active, use it or else resort to
370 * eager-based r-v. */
371 if (proto->protoexp != NULL)
372 ips_scb_mqhdr(scb) = req->type & MQE_TYPE_WAITING ?
373 MQ_MSG_RTS_WAIT : MQ_MSG_RTS;
374 else
375 ips_scb_mqhdr(scb) = MQ_MSG_RTS_EGR;
376
377 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
378 ips_scb_flags(scb) |= IPS_SEND_FLAG_ACK_REQ;
379
380 ips_scb_uwords(scb)[0].u64 = req->tag;
381 ips_scb_uwords(scb)[1].u32w0 = psmi_mpool_get_obj_index(req);
382 ips_scb_uwords(scb)[1].u32w1 = len;
383
384 memset(&req->tid_grant, 0, sizeof(req->tid_grant));
385 if ((err = ips_mq_send_envelope(proto, mepaddr, ipsaddr, scb, PSMI_TRUE)))
386 goto fail;
387
388 /* Assume that we already put a few rndv requests in flight. This helps
389 * for bibw microbenchmarks and doesn't hurt the 'blocking' case since
390 * we're going to poll anyway */
391 psmi_poll_internal(ipsaddr->epaddr->ep, 1);
392
393 fail:
394 _IPATH_VDBG("[rndv][%s->%s][b=%p][m=%d][t=%"PRIx64"][req=%p/%d]: %s\n",
395 psmi_epaddr_get_name(proto->ep->epid),
396 psmi_epaddr_get_name(ipsaddr->epaddr->epid), buf, len, req->tag, req,
397 psmi_mpool_get_obj_index(req),
398 psm_error_get_string(err));
399
400 return err;
401 }
402
403 psm_error_t __sendpath
404 ips_proto_mq_isend(psm_mq_t mq, psm_epaddr_t mepaddr, uint32_t flags,
405 uint64_t tag, const void *ubuf, uint32_t len, void *context,
406 psm_mq_req_t *req_o)
407 {
408 uint8_t *buf = (uint8_t *) ubuf;
409 uint32_t pktlen = 0;
410 ips_scb_t *scb;
411 psm_epaddr_t epaddr = mepaddr->mctxt_current;
412 ips_epaddr_t *ipsaddr = epaddr->ptladdr;
413 struct ips_proto *proto = ipsaddr->proto;
414 uint32_t pad_write_bytes;
415 psm_error_t err = PSM_OK;
416 psm_mq_req_t req = psmi_mq_req_alloc(mq, MQE_TYPE_SEND);
417 if_pf (req == NULL)
418 return PSM_NO_MEMORY;
419
420 mepaddr->mctxt_current = epaddr->mctxt_next;
421 req->send_msglen = len;
422 req->tag = tag;
423 req->context = context;
424
425 if (!flags && len <= MQ_IPATH_THRESH_TINY) {
426 scb = mq_alloc_tiny(proto);
427 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_HDR;
428 ips_scb_hdr_dlen(scb) = len;
429 ips_scb_mqhdr(scb) = MQ_MSG_TINY;
430 ips_scb_mqtag(scb) = tag;
431 mq_copy_tiny((uint32_t *)&ips_scb_mqparam(scb), (uint32_t *)buf, len);
432 err = ips_mq_send_envelope(proto, mepaddr, ipsaddr, scb, PSMI_TRUE);
433 /* We can mark this op complete since all the data is now copied
434 * into an SCB that remains live until it is remotely acked */
435 req->state = MQ_STATE_COMPLETE;
436 mq_qq_append(&mq->completed_q, req);
437 _IPATH_VDBG("[itiny][%s->%s][b=%p][m=%d][t=%"PRIx64"][req=%p]\n",
438 psmi_epaddr_get_name(mq->ep->epid),
439 psmi_epaddr_get_name(epaddr->epid), buf, len, tag, req);
440 *req_o = req;
441 mq->stats.tx_num++;
442 mq->stats.tx_eager_num++;
443 mq->stats.tx_eager_bytes += len;
444 return err;
445 }
446 else if (flags & PSM_MQ_FLAG_SENDSYNC) {/* skip eager accounting below */
447 err = ips_ptl_mq_rndv(req, mepaddr, ipsaddr, ubuf, len);
448 *req_o = req;
449 return err;
450 }
451 else if (len <= ipsaddr->epr.epr_piosize) {
452 uint32_t cksum_len = (proto->flags & IPS_PROTO_FLAG_CKSUM) ?
453 PSM_CRC_SIZE_IN_BYTES : 0;
454
455 pad_write_bytes = ((PSM_CACHE_LINE_BYTES -
456 ((len + cksum_len) & (PSM_CACHE_LINE_BYTES-1))) &
457 (PSM_CACHE_LINE_BYTES-1));
458
459 if_pf ((pad_write_bytes + len) > ipsaddr->epr.epr_piosize)
460 pad_write_bytes = 0;
461 scb = mq_alloc_pkts(proto, 1, (len + pad_write_bytes),
462 IPS_SCB_FLAG_ADD_BUFFER);
463 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
464 ips_scb_hdr_dlen(scb) = pad_write_bytes;
465 ips_scb_length(scb) = len + pad_write_bytes;
466 ips_scb_mqhdr(scb) = MQ_MSG_SHORT;
467 ips_scb_mqtag(scb) = tag;
468 ips_shortcpy (ips_scb_buffer(scb), buf, len);
469 err = ips_mq_send_envelope(proto, mepaddr, ipsaddr, scb, PSMI_TRUE);
470 req->state = MQ_STATE_COMPLETE;
471 mq_qq_append(&mq->completed_q, req);
472 _IPATH_VDBG("[ishrt][%s->%s][b=%p][m=%d][t=%"PRIx64"][req=%p]\n",
473 psmi_epaddr_get_name(mq->ep->epid),
474 psmi_epaddr_get_name(epaddr->epid), buf, len, tag, req);
475 }
476 else if (len <= mq->ipath_thresh_rv) {
477 uint32_t proto_flags = proto->flags & IPS_PROTO_FLAG_MQ_MASK;
478 psmi_egrid_t egrid;
479
480 scb = mq_alloc_pkts(proto, 1, 0, 0);
481 /* directly send from user's buffer */
482 ips_scb_buffer(scb) = buf;
483
484 if (len < proto->iovec_thresh_eager) {
485 if (len <= 2 * ipsaddr->epr.epr_piosize) {
486 // split into 2 packets and round second down to dword multiple
487 pktlen = len - (((len >> 1) + 3) & ~0x3);
488 }
489 else {
490 pktlen = min(len, ipsaddr->epr.epr_piosize);
491 }
492 proto_flags &= ~IPS_PROTO_FLAG_MQ_EAGER_SDMA;
493
494 /*
495 * since following packets are sent on the same flow,
496 * we only wait for completion for the last packet
497 */
498 req->send_msgoff = pktlen;
499 }
500 else {
501 psmi_assert(proto_flags & IPS_PROTO_FLAG_MQ_EAGER_SDMA);
502 /* send the unaligned bytes only, this is required by sdma. */
503 pktlen = (uint32_t)((uintptr_t)buf & 0x3);
504 if (pktlen) pktlen = 4 - pktlen;
505
506 /* send from user buffer, need completion */
507 req->send_msgoff = 0;
508 if (pktlen) {
509 ips_scb_flags(scb) |= IPS_SEND_FLAG_ACK_REQ;
510 ips_scb_cb(scb) = ips_proto_mq_eager_complete;
511 ips_scb_cb_param(scb) = req;
512 }
513 }
514 psmi_assert(pktlen <= ipsaddr->epr.epr_piosize);
515
516 ips_scb_length(scb) = pktlen;
517 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
518 ips_scb_mqhdr(scb) = MQ_MSG_LONG;
519 ips_scb_mqtag(scb) = tag;
520 ips_scb_mqparam(scb).u32w1 = len;
521
522 /* We need a new eager long message number */
523 egrid.egr_data = ips_scb_mqparam(scb).u32w0 =
524 mepaddr->xmit_egrlong.egr_data;
525 mepaddr->xmit_egrlong.egr_msgno++;
526
527 /* Send the envelope but don't flush if writev is enabled */
528 err = ips_mq_send_envelope(proto, mepaddr, ipsaddr, scb, PSMI_FALSE);
529 ips_mq_send_payload(epaddr, egrid,
530 buf+pktlen, len-pktlen, pktlen, req,
531 proto_flags);
532
533 _IPATH_VDBG("[ilong][%s->%s][b=%p][l=%d][m=%d][t=%"PRIx64"][req=%p]\n",
534 psmi_epaddr_get_name(mq->ep->epid),
535 psmi_epaddr_get_name(epaddr->epid), buf, pktlen, len, tag, req);
536 }
537 else { /* skip eager accounting below */
538 err = ips_ptl_mq_rndv(req, mepaddr, ipsaddr, ubuf, len);
539 *req_o = req;
540 return err;
541 }
542
543 *req_o = req;
544 mq->stats.tx_num++;
545 mq->stats.tx_eager_num++;
546 mq->stats.tx_eager_bytes += len;
547
548 return err;
549 }
550
551 __sendpath
552 psm_error_t
553 ips_proto_mq_send(psm_mq_t mq, psm_epaddr_t mepaddr, uint32_t flags,
554 uint64_t tag, const void *ubuf, uint32_t len)
555 {
556 uint8_t *buf = (uint8_t *) ubuf;
557 uint32_t pktlen;
558 ips_scb_t *scb;
559 psm_epaddr_t epaddr = mepaddr->mctxt_current;
560 ips_epaddr_t *ipsaddr = epaddr->ptladdr;
561 uint32_t pad_write_bytes;
562 psm_error_t err = PSM_OK;
563 struct ips_proto *proto = ipsaddr->proto;
564
565 mepaddr->mctxt_current = epaddr->mctxt_next;
566
567 if (flags == 0 && len <= MQ_IPATH_THRESH_TINY) {
568 scb = mq_alloc_tiny(proto);
569 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_HDR;
570 ips_scb_hdr_dlen(scb) = len;
571 ips_scb_mqhdr(scb) = MQ_MSG_TINY;
572 ips_scb_mqtag(scb) = tag;
573
574 mq_copy_tiny((uint32_t *)&ips_scb_mqparam(scb), (uint32_t *)buf, len);
575 err = ips_mq_send_envelope(proto, mepaddr, ipsaddr, scb, PSMI_TRUE);
576 _IPATH_VDBG("[tiny][%s->%s][b=%p][m=%d][t=%"PRIx64"]\n",
577 psmi_epaddr_get_name(mq->ep->epid),
578 psmi_epaddr_get_name(epaddr->epid), buf, len, tag);
579 mq->stats.tx_num++;
580 mq->stats.tx_eager_num++;
581 mq->stats.tx_eager_bytes += len;
582 return err;
583 }
584 else if ((flags & PSM_MQ_FLAG_SENDSYNC)) {
585 goto do_rendezvous;
586 }
587 else if (len <= ipsaddr->epr.epr_piosize) {
588 uint32_t cksum_len = (proto->flags & IPS_PROTO_FLAG_CKSUM) ?
589 PSM_CRC_SIZE_IN_BYTES : 0;
590
591 pad_write_bytes = ((PSM_CACHE_LINE_BYTES -
592 ((len + cksum_len) & (PSM_CACHE_LINE_BYTES-1))) &
593 (PSM_CACHE_LINE_BYTES-1));
594
595 if_pf ((pad_write_bytes + len) > ipsaddr->epr.epr_piosize)
596 pad_write_bytes = 0;
597
598 scb = mq_alloc_pkts(proto, 1, (len + pad_write_bytes),
599 IPS_SCB_FLAG_ADD_BUFFER);
600 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
601 ips_scb_hdr_dlen(scb) = pad_write_bytes;
602 ips_scb_length(scb) = len + pad_write_bytes;
603 ips_scb_mqhdr(scb) = MQ_MSG_SHORT;
604 ips_scb_mqtag(scb) = tag;
605
606 ips_shortcpy (ips_scb_buffer(scb), buf, len);
607 err = ips_mq_send_envelope(proto, mepaddr, ipsaddr, scb, PSMI_TRUE);
608 _IPATH_VDBG("[shrt][%s->%s][b=%p][m=%d][t=%"PRIx64"]\n",
609 psmi_epaddr_get_name(mq->ep->epid),
610 psmi_epaddr_get_name(epaddr->epid), buf, len, tag);
611 }
612 else if (len <= mq->ipath_thresh_rv) {
613 uint32_t proto_flags = proto->flags & IPS_PROTO_FLAG_MQ_MASK;
614 psmi_egrid_t egrid;
615 psm_mq_req_t req = NULL;
616
617 if (len < proto->iovec_thresh_eager_blocking) {
618 if (len <= 2 * ipsaddr->epr.epr_piosize) {
619 // split into 2 packets and round second down to dword multiple
620 pktlen = len - (((len >> 1) + 3) & ~0x3);
621 }
622 else {
623 pktlen = min(len, ipsaddr->epr.epr_piosize);
624 }
625 proto_flags &= ~IPS_PROTO_FLAG_MQ_EAGER_SDMA;
626
627 scb = mq_alloc_pkts(proto, 1, pktlen, IPS_SCB_FLAG_ADD_BUFFER);
628 /* In blocking mode, copy to scb bounce buffer */
629 ips_shortcpy (ips_scb_buffer(scb), buf, pktlen);
630 }
631 else {
632 psmi_assert(proto_flags & IPS_PROTO_FLAG_MQ_EAGER_SDMA);
633 /* send the unaligned bytes only, this is required by sdma. */
634 pktlen = (uint32_t)((uintptr_t)buf & 0x3);
635 if (pktlen) pktlen = 4 - pktlen;
636
637 /* Block until we can get a req */
638 PSMI_BLOCKUNTIL(mq->ep, err,
639 (req = psmi_mq_req_alloc(mq, MQE_TYPE_SEND)));
640 req->type |= MQE_TYPE_WAITING;
641 req->send_msglen = len;
642 req->tag = tag;
643
644 scb = mq_alloc_pkts(proto, 1, 0, 0);
645 /* directly send from user's buffer */
646 ips_scb_buffer(scb) = buf;
647
648 /* send from user buffer, need completion */
649 req->send_msgoff = 0;
650 if (pktlen) {
651 ips_scb_flags(scb) |= IPS_SEND_FLAG_ACK_REQ;
652 ips_scb_cb(scb) = ips_proto_mq_eager_complete;
653 ips_scb_cb_param(scb) = req;
654 }
655 }
656 psmi_assert(pktlen <= ipsaddr->epr.epr_piosize);
657
658 ips_scb_length(scb) = pktlen;
659 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
660 ips_scb_mqhdr(scb) = MQ_MSG_LONG;
661 ips_scb_mqtag(scb) = tag;
662 ips_scb_mqparam(scb).u32w1 = len;
663
664 /* We need a new eager long message number */
665 egrid.egr_data = ips_scb_mqparam(scb).u32w0 =
666 mepaddr->xmit_egrlong.egr_data;
667 mepaddr->xmit_egrlong.egr_msgno++;
668
669 /* Send the envelope but don't flush if writev is enabled */
670 err = ips_mq_send_envelope(proto, mepaddr, ipsaddr, scb, PSMI_FALSE);
671 ips_mq_send_payload(epaddr, egrid,
672 buf+pktlen, len-pktlen, pktlen, req,
673 proto_flags);
674 if (req) psmi_mq_wait_internal(&req);
675
676 _IPATH_VDBG("[long][%s->%s][b=%p][l=%d][m=%d][t=%"PRIx64"]\n",
677 psmi_epaddr_get_name(mq->ep->epid),
678 psmi_epaddr_get_name(epaddr->epid), buf, pktlen, len, tag);
679 }
680 else {
681 psm_mq_req_t req;
682 do_rendezvous:
683 /* Block until we can get a req */
684 PSMI_BLOCKUNTIL(mq->ep, err,
685 (req = psmi_mq_req_alloc(mq, MQE_TYPE_SEND)));
686 req->type |= MQE_TYPE_WAITING;
687 req->tag = tag;
688 err = ips_ptl_mq_rndv(req, mepaddr, ipsaddr, ubuf, len);
689 if (err != PSM_OK)
690 return err;
691 psmi_mq_wait_internal(&req);
692 return err; /* skip accounting, done separately at completion time */
693 }
694
695 mq->stats.tx_num++;
696 mq->stats.tx_eager_num++;
697 mq->stats.tx_eager_bytes += len;
698
699 return err;
700 }
701
702 static
703 psm_error_t __recvpath
704 ips_proto_mq_rts_match_callback(psm_mq_req_t req, int was_posted)
705 {
706 psm_epaddr_t epaddr = req->rts_peer;
707 ips_epaddr_t *ipsaddr = epaddr->ptladdr;
708 struct ips_proto *proto = ipsaddr->proto;
709
710 /* We have a match.
711 *
712 * If we're doing eager-based r-v, just send back the sreq and length and
713 * have the sender complete the send.
714 *
715 */
716 if (proto->protoexp == NULL) { /* only eager-based r-v so far */
717 struct ips_pend_sends *pends = &proto->pend_sends;
718 struct ips_pend_sreq *sreq = psmi_mpool_get(proto->pend_sends_pool);
719 psmi_assert(sreq != NULL);
720 if (sreq == NULL) return PSM_NO_MEMORY;
721 sreq->type = IPS_PENDSEND_EAGER_REQ;
722 sreq->req = req;
723
724 STAILQ_INSERT_TAIL(&pends->pendq, sreq, next);
725 psmi_timer_request(proto->timerq, &pends->timer, PSMI_TIMER_PRIO_1);
726 }
727 else {
728 ips_protoexp_tid_get_from_token(
729 proto->protoexp, req->buf, req->recv_msglen, epaddr,
730 req->rts_reqidx_peer,
731 req->type & MQE_TYPE_WAITING_PEER ? IPS_PROTOEXP_TIDGET_PEERWAIT : 0,
732 ips_proto_mq_rv_complete_exp, req);
733 }
734
735 _IPATH_VDBG("req=%p, dest=%p, len=%d, recv_msglen=%d, stok=%p, expected=%s\n",
736 req, req->buf, req->buf_len, req->recv_msglen,
737 req->ptl_req_ptr, was_posted ? "YES" : "NO");
738
739 return PSM_OK;
740 }
741
742 psm_error_t __recvpath
743 ips_proto_mq_push_eager_req(struct ips_proto *proto, psm_mq_req_t req)
744 {
745 ips_scb_t *scb;
746 ptl_arg_t *args;
747 ips_epaddr_t *ipsaddr;
748 struct ips_flow *flow;
749
750 scb = ips_scbctrl_alloc(&proto->scbc_egr, 1, 0, 0);
751 if (scb == NULL)
752 return PSM_OK_NO_PROGRESS;
753
754 args = (ptl_arg_t *) ips_scb_uwords(scb);
755
756 args[0].u32w0 = req->rts_reqidx_peer;
757 args[0].u32w1 = psmi_mpool_get_obj_index(req);
758 args[1].u32w0 = req->recv_msglen;
759 req->egrid.egr_data = args[0].u32w1;
760
761 ipsaddr = req->rts_peer->ptladdr;
762 flow = &ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO];
763 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
764 ips_scb_mqhdr (scb) = MQ_MSG_CTS_EGR;
765
766 if (req->recv_msglen == 0) {
767 ips_proto_mq_rv_complete(req);
768 }
769
770 flow->fn.xfer.enqueue(flow, scb);
771 flow->fn.xfer.flush(flow, NULL);
772
773 return PSM_OK;
774 }
775
776 psm_error_t __recvpath
777 ips_proto_mq_push_eager_data(struct ips_proto *proto, psm_mq_req_t req)
778 {
779 uintptr_t buf = (uintptr_t) req->buf;
780 ips_epaddr_t *ipsaddr = req->rts_peer->ptladdr;
781 uint32_t nbytes_this;
782 uint32_t nbytes_left = req->send_msglen - req->recv_msgoff;
783 uint16_t frag_size;
784 struct ips_flow *flow;
785 ips_scb_t *scb;
786
787 psmi_assert(nbytes_left > 0);
788
789 if (!(proto->flags & IPS_PROTO_FLAG_MQ_EAGER_SDMA)) goto spio;
790
791 flow = &ipsaddr->flows[EP_FLOW_GO_BACK_N_DMA];
792 frag_size = flow->frag_size;
793 nbytes_this = 131072/8;
794 while (nbytes_left > 0) {
795 scb = ips_scbctrl_alloc(proto->scbc_rv, 1, 0, 0);
796 if (scb == NULL)
797 return PSM_OK_NO_PROGRESS;
798
799 #if 0
800 /* turn on to use single frag-size packet */
801 nbytes_this = min(frag_size, nbytes_left);
802 #else
803 nbytes_this = min(nbytes_this, nbytes_left);
804 #endif
805
806 ips_scb_length(scb) = nbytes_this;
807 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
808 ips_scb_mqhdr (scb) = MQ_MSG_DATA_REQ_BLK;
809 ips_scb_buffer(scb) = (void *)(buf + req->recv_msgoff);
810 ips_scb_mqparam(scb).u32w0 = req->rts_reqidx_peer;
811 ips_scb_mqparam(scb).u32w1 = req->recv_msgoff;
812
813 if (nbytes_left == nbytes_this) {
814 ips_scb_cb(scb) = ips_proto_mq_eager_complete;
815 ips_scb_cb_param(scb) = req;
816 } else {
817 req->send_msgoff += nbytes_this;
818 }
819
820 scb->nfrag = (nbytes_this + frag_size - 1) / frag_size;
821 scb->frag_size = frag_size;
822
823 /* attach checksum if enabled, this matches what is done for tid-sdma */
824 if (proto->flags&IPS_PROTO_FLAG_CKSUM && nbytes_left==nbytes_this) {
825 uint32_t cksum = 0xffffffff;
826 cksum = ips_crc_calculate(req->send_msglen, req->buf, cksum);
827 scb->ips_lrh.data[0].u32w0 = cksum;
828 }
829
830 ips_scb_flags(scb) |= IPS_SEND_FLAG_ACK_REQ;
831 ips_scb_flags(scb) |= IPS_SEND_FLAG_WAIT_SDMA;
832 SLIST_NEXT(scb, next) = NULL;
833
834 flow->fn.xfer.enqueue(flow, scb);
835 flow->fn.xfer.flush(flow, NULL);
836
837 nbytes_left -= nbytes_this;
838 req->recv_msgoff += nbytes_this;
839 }
840
841 return PSM_OK;
842
843 spio:
844 flow = &ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO];
845 frag_size = flow->frag_size;
846 while (nbytes_left > 0) {
847 scb = ips_scbctrl_alloc(proto->scbc_rv, 1, 0, 0);
848 if (scb == NULL)
849 return PSM_OK_NO_PROGRESS;
850
851 nbytes_this = min(nbytes_left, frag_size);
852 ips_scb_length(scb) = nbytes_this;
853 ips_scb_subopcode(scb) = OPCODE_SEQ_MQ_CTRL;
854 ips_scb_mqhdr (scb) = MQ_MSG_DATA_REQ;
855 ips_scb_buffer(scb) = (void *)(buf + req->recv_msgoff);
856 ips_scb_mqparam(scb).u32w0 = req->rts_reqidx_peer;
857 ips_scb_mqparam(scb).u32w1 = req->recv_msgoff;
858
859 ips_scb_cb(scb) = ips_proto_mq_eager_complete;
860 ips_scb_cb_param(scb) = req;
861 if (nbytes_left == nbytes_this) {
862 ips_scb_flags(scb) |= IPS_SEND_FLAG_ACK_REQ;
863 }
864 #if 0
865 _IPATH_INFO("send req %p, off %d/%d, len %d, last=%s\n",
866 req, req->send_msgoff, req->send_msglen, nbytes_this,
867 nbytes_left == nbytes_this ? "YES" : "NO");
868 #endif
869 SLIST_NEXT(scb, next) = NULL;
870
871 flow->fn.xfer.enqueue(flow, scb);
872 flow->fn.xfer.flush(flow, NULL);
873
874 nbytes_left -= nbytes_this;
875 req->recv_msgoff += nbytes_this;
876 }
877
878 return PSM_OK;
879 }
880
881 int __recvpath
882 ips_proto_mq_handle_cts(struct ips_proto *proto, ptl_arg_t *args)
883 {
884 psm_mq_req_t req;
885 psm_mq_t mq = proto->ep->mq;
886 uint32_t reqidx, reqidx_peer;
887 struct ips_pend_sreq *sreq;
888 uint32_t msglen;
889
890 reqidx = args[0].u32w0;
891 reqidx_peer = args[0].u32w1;
892 msglen = args[1].u32w0;
893
894 req = psmi_mpool_find_obj_by_index(mq->sreq_pool, reqidx);
895 psmi_assert(req != NULL);
896 if (req == NULL) return IPS_RECVHDRQ_BREAK;
897
898 if (msglen == 0) {
899 ips_proto_mq_rv_complete(req);
900 return IPS_RECVHDRQ_CONTINUE;
901 }
902
903 sreq = psmi_mpool_get(proto->pend_sends_pool);
904 psmi_assert(sreq != NULL);
905 if (sreq == NULL) return IPS_RECVHDRQ_BREAK;
906 sreq->type = IPS_PENDSEND_EAGER_DATA;
907 sreq->req = req;
908 req->rts_reqidx_peer = reqidx_peer;
909 req->send_msglen = msglen;
910 req->send_msgoff = 0;
911 STAILQ_INSERT_TAIL(&proto->pend_sends.pendq, sreq, next);
912 /* Make sure it's processed by timer */
913 psmi_timer_request(proto->timerq, &proto->pend_sends.timer,
914 PSMI_TIMER_PRIO_1);
915
916 /* XXX Optimization here: If the 'req' is blocking in the MPI sense, we
917 * could choose to break out of the progress loop and make progress on it
918 * ASAP instead of continuing to process the receive queue */
919 return IPS_RECVHDRQ_CONTINUE;
920 }
921
922 int __recvpath
923 ips_proto_mq_handle_rts_envelope(psm_mq_t mq, int mode, psm_epaddr_t epaddr,
924 uint64_t tag, uint32_t reqidx_peer,
925 uint32_t msglen)
926 {
927 psm_mq_req_t req;
928 _IPATH_VDBG("tag=%llx reqidx_peer=%d, msglen=%d\n",
929 (long long) tag, reqidx_peer, msglen);
930 int rc = psmi_mq_handle_rts(mq, tag, 0, msglen, epaddr,
931 ips_proto_mq_rts_match_callback, &req);
932 req->rts_reqidx_peer = reqidx_peer;
933 if (mode == MQ_MSG_RTS_WAIT)
934 req->type |= MQE_TYPE_WAITING_PEER;
935
936 if (rc == MQ_RET_MATCH_OK) {
937 ips_proto_mq_rts_match_callback(req, 1);
938 /* XXX if blocking, break out of progress loop */
939 }
940
941 /* If no match, will be called when send actually matches */
942 return IPS_RECVHDRQ_CONTINUE;
943 }
944
945 int __recvpath
946 ips_proto_mq_handle_rts_envelope_outoforder(psm_mq_t mq, int mode,
947 psm_epaddr_t peer, uint16_t msg_seqnum,
948 uint64_t tag, uint32_t reqidx_peer,
949 uint32_t msglen)
950 {
951 psm_mq_req_t req;
952 _IPATH_VDBG("tag=%llx reqidx_peer=%d, msglen=%d\n",
953 (long long) tag, reqidx_peer, msglen);
954 psmi_mq_handle_rts_outoforder(mq, tag, 0, msglen,
955 peer, msg_seqnum,
956 ips_proto_mq_rts_match_callback, &req);
957 req->rts_reqidx_peer = reqidx_peer;
958 if (mode == MQ_MSG_RTS_WAIT)
959 req->type |= MQE_TYPE_WAITING_PEER;
960
961 /* If no match, will be called when send actually matches */
962 return IPS_RECVHDRQ_CONTINUE;
963 }
964