Debian Codegraph @

ptl_ips/ips_proto_am.c

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "psm_user.h"
36 #include "psm_am.h"
37 #include "psm_am_internal.h"
38 #include "ips_proto.h"
39 #include "ips_proto_internal.h"
40
41 #define IPS_AMFLAG_ISTINY 1
42
43 struct ips_am_token {
44 struct psmi_am_token tok;
45
46 /* ptl-specific token stuff */
47 struct ips_proto_am *proto_am;
48 };
49
50 psm_error_t
51 ips_proto_am_init(struct ips_proto *proto,
52 int num_of_send_bufs, int num_of_send_desc,
53 uint32_t imm_size, struct ips_proto_am *proto_am)
54 {
55 psm_error_t err = PSM_OK;
56 int send_buf_size = proto->scb_bufsize;
57
58 proto_am->proto = proto;
59 proto_am->scbc_request = &proto->scbc_egr;
60
61 if ((err = ips_scbctrl_init(&proto->ep->context, num_of_send_desc,
62 num_of_send_bufs, imm_size, send_buf_size,
63 NULL, NULL, &proto_am->scbc_reply)))
64 goto fail;
65 fail:
66 return err;
67 }
68
69 psm_error_t
70 ips_proto_am_fini(struct ips_proto_am *proto_am)
71 {
72 return PSM_OK;
73 }
74
75 static
76 psm_error_t
77 am_short_reqrep(struct ips_proto_am *proto_am, ips_scb_t *scb,
78 struct ptl_epaddr *ipsaddr,
79 psm_amarg_t *args, int nargs, uint8_t sub_opcode,
80 void *src, size_t len, int flags, int pad_bytes)
81
82 {
83 int i, hdr_qwords = PSM_AM_HDR_QWORDS;
84 ptl_epaddr_flow_t flowid = ((sub_opcode == OPCODE_AM_REQUEST) ||
85 (sub_opcode == OPCODE_AM_REQUEST_NOREPLY)) ?
86 EP_FLOW_GO_BACK_N_AM_REQ : EP_FLOW_GO_BACK_N_AM_RSP;
87 struct ips_flow *flow = &ipsaddr->flows[flowid];
88
89 _IPATH_VDBG("%s src=%p len=%d, nargs=%d\n",
90 ((sub_opcode == OPCODE_AM_REQUEST) ||
91 (sub_opcode == OPCODE_AM_REQUEST_NOREPLY)) ? "req" : "rep",
92 src, (int) len, nargs);
93
94 if (nargs == 1) { /* fastpath */
95 scb->ips_lrh.data[0].u64w0 = args[0].u64w0;
96 hdr_qwords--;
97 }
98 else if (nargs > 1) {
99 /* Easily unrollable but leave as is in case we can increase qwords
100 * on the chip in the near future */
101 for (i = 0; i < PSM_AM_HDR_QWORDS; i++, hdr_qwords--)
102 scb->ips_lrh.data[i].u64w0 = args[i].u64w0;
103
104 if (nargs > PSM_AM_HDR_QWORDS) {
105 /* Slow case -- we don't have iovec and not enough space in the
106 * message header, so we have to copy the user's arguments even if
107 * the payload is marked ASYNC */
108 uintptr_t bufp = (uintptr_t) scb->payload;
109 psmi_mq_mtucpy((void *) bufp, &args[PSM_AM_HDR_QWORDS],
110 sizeof(psm_amarg_t) * (nargs - PSM_AM_HDR_QWORDS));
111 bufp += sizeof(psm_amarg_t) * (nargs - PSM_AM_HDR_QWORDS);
112 scb->payload_size = sizeof(psm_amarg_t) * (nargs-PSM_AM_HDR_QWORDS);
113 if (src != NULL && len > 0) {
114 psmi_mq_mtucpy((void *) bufp, src, len);
115 scb->payload_size += len;
116 }
117 scb->payload_size += pad_bytes;
118 scb->ips_lrh.hdr_dlen = pad_bytes;
119 goto send_scb;
120 }
121 }
122
123 /*
124 * If small enough, try to stuff the message in a header only
125 */
126 if (len <= (hdr_qwords<<3)) { /* can handle len == 0 */
127 psmi_mq_mtucpy(&scb->ips_lrh.data[PSM_AM_HDR_QWORDS-hdr_qwords], src, len);
128 scb->payload_size = 0;
129 scb->ips_lrh.hdr_dlen = len;
130 scb->ips_lrh.amhdr_flags |= IPS_AMFLAG_ISTINY;
131 }
132 else { /* Whatever's left requires a separate payload */
133 if (scb->payload == NULL) { /* Just attach the buffer */
134 scb->payload = src;
135 }
136 else { /* May need to re-xmit user data, keep it around */
137 psmi_mq_mtucpy(scb->payload, src, len);
138 }
139 scb->payload_size = len + pad_bytes;
140 scb->ips_lrh.hdr_dlen = pad_bytes;
141 }
142
143 send_scb:
144 scb->ips_lrh.sub_opcode = sub_opcode;
145 flow->fn.xfer.enqueue(flow, scb);
146 flow->fn.xfer.flush(flow, NULL);
147 return PSM_OK;
148 }
149
150 static inline int
151 calculate_pad_bytes (struct ips_proto_am *proto_am, int nargs, size_t len)
152 {
153 if ((nargs <= PSM_AM_HDR_QWORDS) &&
154 (len <= ((PSM_AM_HDR_QWORDS - nargs) << 3)))
155 return 0;
156 else {
157 size_t arg_overflow = (nargs > PSM_AM_HDR_QWORDS) ?
158 (sizeof(psm_amarg_t) * (nargs - PSM_AM_HDR_QWORDS)) : 0;
159 size_t cache_aligned_len = (len + arg_overflow + PSM_CACHE_LINE_BYTES-1) &
160 ~(PSM_CACHE_LINE_BYTES - 1);
161 if (cache_aligned_len <= proto_am->proto->scb_bufsize)
162 return cache_aligned_len - (len + arg_overflow);
163 else
164 return 0;
165 }
166 }
167
168 static inline
169 void
170 ips_am_scb_init(ips_scb_t *scb, uint8_t handler, int nargs,
171 int pad_bytes,
172 psm_am_completion_fn_t completion_fn,
173 void *completion_ctxt)
174 {
175 scb->completion_am = completion_fn;
176 scb->cb_param = completion_ctxt;
177 scb->ips_lrh.amhdr_hidx = handler;
178 scb->ips_lrh.hdr_dlen = pad_bytes;
179 scb->ips_lrh.amhdr_nargs = nargs;
180 scb->ips_lrh.amhdr_flags = 0;
181 if (completion_fn)
182 scb->flags |= IPS_SEND_FLAG_ACK_REQ;
183 return;
184 }
185
186 psm_error_t
187 ips_am_short_request(psm_epaddr_t epaddr,
188 psm_handler_t handler, psm_amarg_t *args, int nargs,
189 void *src, size_t len, int flags,
190 psm_am_completion_fn_t completion_fn,
191 void *completion_ctxt)
192 {
193 struct ips_proto_am *proto_am = &epaddr->ptl->proto.proto_am;
194 psm_error_t err;
195 ips_scb_t *scb;
196 int pad_bytes = calculate_pad_bytes(proto_am, nargs, len);
197 int payload_sz = (nargs << 3) + pad_bytes;
198
199 if_pt (!(flags & PSM_AM_FLAG_ASYNC))
200 payload_sz += len;
201
202 if (payload_sz > (PSM_AM_HDR_QWORDS << 3)) {
203 /* Payload can't fit in header - allocate buffer to carry data */
204 int arg_sz = (nargs > PSM_AM_HDR_QWORDS) ?
205 ((nargs - PSM_AM_HDR_QWORDS) << 3) : 0;
206
207 /* len + pad_bytes + overflow_args */
208 PSMI_BLOCKUNTIL(epaddr->ep,err,
209 ((scb = ips_scbctrl_alloc(proto_am->scbc_request, 1,
210 len + pad_bytes + arg_sz,
211 IPS_SCB_FLAG_ADD_BUFFER)) != NULL));
212 }
213 else {
214 PSMI_BLOCKUNTIL(epaddr->ep,err,
215 ((scb = ips_scbctrl_alloc_tiny(proto_am->scbc_request)) != NULL));
216 }
217
218 psmi_assert_always(scb != NULL);
219 ips_am_scb_init(scb, handler, nargs, pad_bytes,
220 completion_fn, completion_ctxt);
221
222 return am_short_reqrep(proto_am, scb, epaddr->ptladdr, args, nargs,
223 (flags & PSM_AM_FLAG_NOREPLY) ?
224 OPCODE_AM_REQUEST_NOREPLY : OPCODE_AM_REQUEST,
225 src, len, flags, pad_bytes);
226 }
227
228 psm_error_t
229 ips_am_short_reply(psm_am_token_t tok,
230 psm_handler_t handler, psm_amarg_t *args, int nargs,
231 void *src, size_t len, int flags,
232 psm_am_completion_fn_t completion_fn,
233 void *completion_ctxt)
234 {
235 ips_scb_t *scb;
236 struct ips_am_token *token = (struct ips_am_token *) tok;
237 struct ips_proto_am *proto_am = token->proto_am;
238 struct ptl_epaddr *ipsaddr = token->tok.epaddr_from->ptladdr;
239 int scb_flags = 0;
240 int pad_bytes = calculate_pad_bytes(proto_am, nargs, len);
241
242 if (!token->tok.can_reply) {
243 /* Trying to reply for an AM request that did not expect a reply */
244 _IPATH_ERROR("Invalid AM reply for request!");
245 return PSM_AM_INVALID_REPLY;
246 }
247
248 psmi_assert_always(ips_scbctrl_avail(&proto_am->scbc_reply));
249
250 if ((nargs<<3) + len <= (PSM_AM_HDR_QWORDS<<3)) {
251 psmi_assert_always(pad_bytes == 0);
252 scb = ips_scbctrl_alloc_tiny(&proto_am->scbc_reply);
253 }
254 else {
255 int payload_sz = (nargs << 3) + pad_bytes;
256
257 payload_sz += (flags & PSM_AM_FLAG_ASYNC) ? 0 : len;
258 scb_flags |= (payload_sz > (PSM_AM_HDR_QWORDS << 3)) ?
259 IPS_SCB_FLAG_ADD_BUFFER : 0;
260
261 scb = ips_scbctrl_alloc(&proto_am->scbc_reply, 1, payload_sz, scb_flags);
262 }
263
264 psmi_assert_always(scb != NULL);
265 ips_am_scb_init(scb, handler, nargs, pad_bytes,
266 completion_fn, completion_ctxt);
267 am_short_reqrep(proto_am, scb, ipsaddr, args, nargs, OPCODE_AM_REPLY,
268 src, len, flags, pad_bytes);
269 return PSM_OK;
270 }
271
272 /* Prepares and runs a handler from a receive event. */
273 static int
274 ips_am_run_handler(struct ips_am_token *tok,
275 const struct ips_recvhdrq_event *rcv_ev)
276 {
277 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
278 struct ips_proto_am *proto_am = &rcv_ev->proto->proto_am;
279 psm_am_handler_fn_t hfn;
280
281 int nargs = p_hdr->amhdr_nargs;
282 tok->tok.flags = p_hdr->amhdr_flags;
283 tok->tok.epaddr_from = rcv_ev->ipsaddr->epaddr;
284 tok->tok.can_reply = (p_hdr->sub_opcode == OPCODE_AM_REQUEST);
285 tok->proto_am = proto_am;
286
287 hfn = psm_am_get_handler_function(rcv_ev->proto->ep,
288 p_hdr->amhdr_hidx);
289 _IPATH_VDBG("amhdr_len=%d, amhdr_flags=%x, amhdr_nargs=%d, p_hdr=%p\n",
290 p_hdr->hdr_dlen, p_hdr->amhdr_flags, p_hdr->amhdr_nargs, p_hdr);
291
292 /* Fast path: everything fits only in a header */
293 if (tok->tok.flags & IPS_AMFLAG_ISTINY) {
294 return hfn(tok, tok->tok.epaddr_from,
295 (psm_amarg_t *) &p_hdr->data[0].u64, nargs,
296 &p_hdr->data[nargs].u64, p_hdr->hdr_dlen);
297 }
298 else {
299 /* Arguments and payload may split across header/eager_payload
300 * boundaries. */
301 psm_amarg_t args[8] = {};
302 int i;
303 uint64_t *payload = (uint64_t *) ips_recvhdrq_event_payload(rcv_ev);
304 uint32_t paylen = ips_recvhdrq_event_paylen(rcv_ev);
305 for (i = 0; i < nargs; i++) {
306 if (i < PSM_AM_HDR_QWORDS)
307 args[i].u64 = p_hdr->data[i].u64;
308 else {
309 args[i].u64 = *payload++;
310 paylen -= 8;
311 }
312 }
313
314 paylen -= p_hdr->hdr_dlen;
315 return hfn(tok, tok->tok.epaddr_from, args, nargs, payload, paylen);
316 }
317 }
318
319 int
320 ips_proto_am(struct ips_recvhdrq_event *rcv_ev)
321 {
322 struct ips_am_token token;
323 struct ips_message_header *p_hdr = rcv_ev->p_hdr;
324 struct ptl_epaddr *ipsaddr = rcv_ev->ipsaddr;
325 struct ips_proto_am *proto_am = &rcv_ev->proto->proto_am;
326 ptl_epaddr_flow_t flowid = ips_proto_flowid(p_hdr);
327 struct ips_flow *flow = &ipsaddr->flows[flowid];
328 int ret = IPS_RECVHDRQ_CONTINUE;
329
330 /*
331 * Based on AM request/reply traffic pattern, if we don't have
332 * a reply scb slot then we can't process the request packet,
333 * we just silently drop it. Otherwise, it will be a deadlock.
334 * note: ips_proto_is_expected_or_nak() can not be called in this case.
335 */
336 if (p_hdr->sub_opcode == OPCODE_AM_REQUEST &&
337 !ips_scbctrl_avail(&proto_am->scbc_reply)) {
338 proto_am->amreply_nobufs++;
339 return ret;
340 }
341
342 if (ips_proto_is_expected_or_nak((struct ips_recvhdrq_event*) rcv_ev)) {
343 /* run handler */
344 if (ips_am_run_handler(&token, rcv_ev))
345 ret = IPS_RECVHDRQ_BREAK;
346
347 /* Look if the handler replied, if it didn't, ack the request */
348 if ((p_hdr->flags & IPS_SEND_FLAG_ACK_REQ) ||
349 (flow->flags & IPS_FLOW_FLAG_GEN_BECN))
350 ips_proto_send_ack((struct ips_recvhdrq *) rcv_ev->recvq, flow);
351 }
352
353 ips_proto_process_ack(rcv_ev);
354 return ret;
355 }