Debian Codegraph @

ptl_ips/ips_scb.c

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "psm_user.h"
36 #include "ips_proto.h"
37 #include "ips_scb.h"
38
39 psm_error_t
40 ips_scbctrl_init(const psmi_context_t *context,
41 uint32_t numscb, uint32_t numbufs,
42 uint32_t imm_size, uint32_t bufsize,
43 ips_scbctrl_avail_callback_fn_t scb_avail_callback,
44 void *scb_avail_context,
45 struct ips_scbctrl *scbc)
46 {
47 int i;
48 struct ips_scb *scb;
49 size_t scb_size;
50 size_t alloc_sz;
51 uintptr_t base, imm_base;
52 psm_ep_t ep = context->ep;
53 //scbc->context = context;
54 psm_error_t err = PSM_OK;
55
56 psmi_assert_always(numscb > 0);
57 scbc->sbuf_num = scbc->sbuf_num_cur = numbufs;
58 SLIST_INIT(&scbc->sbuf_free);
59 scbc->sbuf_buf_size = bufsize;
60 scbc->sbuf_buf_base = NULL;
61 scbc->sbuf_buf_alloc = NULL;
62 scbc->sbuf_buf_last = NULL;
63
64 /* send buffers are not mandatory but when allocating them, make sure they
65 * are on a page boundary */
66 if (numbufs > 0) {
67 struct ips_scbbuf *sbuf;
68 int redzone = PSM_VALGRIND_REDZONE_SZ;
69
70 /* If the allocation requested is a page and we have redzones we have
71 * to allocate 2 pages so we end up using a redzone of 2048 bytes.
72 *
73 * if the allocation is not 4096, we relax that requirement and keep
74 * the redzones PSM_VALGRIND_REDZONE_SZ
75 */
76 if (redzone > 0 && bufsize % PSMI_PAGESIZE == 0)
77 redzone = PSMI_PAGESIZE / 2;
78 bufsize += 2 * redzone;
79 bufsize = PSMI_ALIGNUP(bufsize, 64);
80
81 alloc_sz = numbufs * bufsize + redzone + PSMI_PAGESIZE;
82 scbc->sbuf_buf_alloc =
83 psmi_calloc(ep, NETWORK_BUFFERS, 1, alloc_sz);
84 if (scbc->sbuf_buf_alloc == NULL) {
85 err = PSM_NO_MEMORY;
86 goto fail;
87 }
88 base = (uintptr_t)scbc->sbuf_buf_alloc;
89 base = PSMI_ALIGNUP(base + redzone, PSMI_PAGESIZE);
90 scbc->sbuf_buf_base = (void *)base;
91 scbc->sbuf_buf_last = (void *)(base + bufsize * (numbufs-1));
92 _IPATH_VDBG("sendbufs=%d, (redzone=%d|size=%d|redzone=%d),base=[%p..%p)\n",
93 numbufs, redzone, bufsize-2*redzone, redzone,
94 (void *) scbc->sbuf_buf_base, (void *) scbc->sbuf_buf_last);
95
96 for (i = 0; i < numbufs; i++) {
97 sbuf = (struct ips_scbbuf *) (base + bufsize * i);
98 SLIST_NEXT(sbuf, next) = NULL;
99 SLIST_INSERT_HEAD(&scbc->sbuf_free, sbuf, next);
100 }
101
102 VALGRIND_CREATE_MEMPOOL(scbc->sbuf_buf_alloc,
103 0,
104 /* Should be undefined but we stuff a next
105 * pointer in the buffer */
106 PSM_VALGRIND_MEM_DEFINED);
107 }
108
109 imm_base = 0;
110 scbc->scb_imm_size = imm_size;
111 if (scbc->scb_imm_size) {
112 scbc->scb_imm_size = PSMI_ALIGNUP(imm_size, 64);
113 alloc_sz = numscb * scbc->scb_imm_size + 64;
114 scbc->scb_imm_buf =
115 psmi_calloc(ep, NETWORK_BUFFERS, 1, alloc_sz);
116 if (scbc->scb_imm_buf == NULL) {
117 err = PSM_NO_MEMORY;
118 goto fail;
119 }
120 imm_base = PSMI_ALIGNUP(scbc->scb_imm_buf, 64);
121 }
122 else
123 scbc->scb_imm_buf = NULL;
124
125 scbc->scb_num = scbc->scb_num_cur = numscb;
126 SLIST_INIT(&scbc->scb_free);
127 scb_size = sizeof(struct ips_scb) + 2*PSM_VALGRIND_REDZONE_SZ;
128 scb_size = PSMI_ALIGNUP(scb_size, 64);
129 alloc_sz = numscb * scb_size + PSM_VALGRIND_REDZONE_SZ + 64;
130 scbc->scb_base = (void *)
131 psmi_calloc(ep, NETWORK_BUFFERS, 1, alloc_sz);
132 if (scbc->scb_base == NULL) {
133 err = PSM_NO_MEMORY;
134 goto fail;
135 }
136 base = (uintptr_t)scbc->scb_base;
137 base = PSMI_ALIGNUP(base + PSM_VALGRIND_REDZONE_SZ, 64);
138 for (i = 0; i < numscb; i++) {
139 scb = (struct ips_scb *)(base + i * scb_size);
140 scb->scbc = scbc;
141 if (scbc->scb_imm_buf)
142 scb->imm_payload = (void*)(imm_base + (i * scbc->scb_imm_size));
143 else
144 scb->imm_payload = NULL;
145
146 SLIST_INSERT_HEAD(&scbc->scb_free, scb, next);
147 }
148 scbc->scb_avail_callback = scb_avail_callback;
149 scbc->scb_avail_context = scb_avail_context;
150
151 /* It would be nice to mark the scb as undefined but we pre-initialize the
152 * "next" pointer and valgrind would see this as a violation.
153 */
154 VALGRIND_CREATE_MEMPOOL(scbc, PSM_VALGRIND_REDZONE_SZ,
155 PSM_VALGRIND_MEM_DEFINED);
156
157 fail:
158 return err;
159 }
160
161 psm_error_t
162 ips_scbctrl_fini(struct ips_scbctrl *scbc)
163 {
164 if (scbc->scb_base != NULL) {
165 psmi_free(scbc->scb_base);
166 VALGRIND_DESTROY_MEMPOOL(scbc);
167 }
168 if (scbc->sbuf_buf_alloc) {
169 VALGRIND_DESTROY_MEMPOOL(scbc->sbuf_buf_alloc);
170 psmi_free(scbc->sbuf_buf_alloc);
171 }
172 return PSM_OK;
173 }
174
175 int
176 ips_scbctrl_bufalloc(ips_scb_t *scb)
177 {
178 struct ips_scbctrl *scbc = scb->scbc;
179
180 psmi_assert_always(scbc->sbuf_num > 0);
181 psmi_assert_always(!((scb->payload >= scbc->sbuf_buf_base) &&
182 (scb->payload <= scbc->sbuf_buf_last)));
183 if (SLIST_EMPTY(&scbc->sbuf_free))
184 return 0;
185 else {
186 psmi_assert(scbc->sbuf_num_cur);
187 scb->payload = SLIST_FIRST(&scbc->sbuf_free);
188 scb->payload_size = scbc->sbuf_buf_size;
189 scbc->sbuf_num_cur--;
190
191 /* If under memory pressure request ACK for packet to reclaim
192 * credits.
193 */
194 if (scbc->sbuf_num_cur < (scbc->sbuf_num >> 1))
195 scb->flags |= IPS_SEND_FLAG_ACK_REQ;
196
197 VALGRIND_MEMPOOL_ALLOC(scbc->sbuf_buf_alloc, scb->payload,
198 scb->payload_size);
199 SLIST_REMOVE_HEAD(&scbc->sbuf_free, next);
200 return 1;
201 }
202 }
203
204 int
205 ips_scbctrl_avail(struct ips_scbctrl *scbc)
206 {
207 return (!SLIST_EMPTY(&scbc->scb_free) && scbc->sbuf_num_cur > 0);
208 }
209
210 ips_scb_t *
211 ips_scbctrl_alloc(struct ips_scbctrl *scbc, int scbnum, int len, uint32_t flags)
212 {
213 ips_scb_t *scb, *scb_head = NULL;
214
215 psmi_assert(flags & IPS_SCB_FLAG_ADD_BUFFER ? (scbc->sbuf_num>0) : 1);
216
217 while (scbnum--) {
218 if (SLIST_EMPTY(&scbc->scb_free))
219 break;
220 scb = SLIST_FIRST(&scbc->scb_free);
221 scb->flags = 0; /* Need to set this here as bufalloc may request
222 * an ACK under memory pressure
223 */
224 VALGRIND_MEMPOOL_ALLOC(scbc, scb, sizeof(struct ips_scb));
225
226 if (flags & IPS_SCB_FLAG_ADD_BUFFER) {
227 if (len > scbc->scb_imm_size) {
228 if (!ips_scbctrl_bufalloc(scb))
229 break;
230 }
231 else { /* Attach immediate buffer */
232 scb->payload = scb->imm_payload;
233 scb->payload_size = scbc->scb_imm_size;
234 psmi_assert(scb->payload);
235 }
236 }
237 else {
238 scb->payload = NULL;
239 scb->payload_size = 0;
240 }
241
242 scb->tid = IPATH_EAGER_TID_ID;
243 scb->tidsendc = NULL;
244 scb->callback = NULL;
245 scb->ips_lrh.mqhdr = 0;
246 scb->offset = 0;
247 scb->nfrag = 1;
248 scb->frag_size = 0;
249
250 scbc->scb_num_cur--;
251 if (scbc->scb_num_cur < (scbc->scb_num >> 1))
252 scb->flags |= IPS_SEND_FLAG_ACK_REQ;
253
254 SLIST_REMOVE_HEAD(&scbc->scb_free, next);
255 SLIST_NEXT(scb, next) = scb_head;
256 scb_head = scb;
257 }
258 return scb_head;
259 }
260
261 void
262 ips_scbctrl_free(ips_scb_t *scb)
263 {
264 struct ips_scbctrl *scbc = scb->scbc;
265 if (scbc->sbuf_num && (scb->payload >= scbc->sbuf_buf_base) &&
266 (scb->payload <= scbc->sbuf_buf_last)) {
267 scbc->sbuf_num_cur++;
268 SLIST_INSERT_HEAD(&scbc->sbuf_free, scb->sbuf, next);
269 VALGRIND_MEMPOOL_FREE(scbc->sbuf_buf_alloc, scb->payload);
270 }
271
272 scb->payload = NULL;
273 scb->tidsendc = NULL;
274 scb->payload_size = 0;
275 scbc->scb_num_cur++;
276 if (SLIST_EMPTY(&scbc->scb_free)) {
277 SLIST_INSERT_HEAD(&scbc->scb_free, scb, next);
278 if (scbc->scb_avail_callback != NULL)
279 scbc->scb_avail_callback(scbc, scbc->scb_avail_context);
280 }
281 else
282 SLIST_INSERT_HEAD(&scbc->scb_free, scb, next);
283
284 VALGRIND_MEMPOOL_FREE(scbc, scb);
285 return;
286 }
287
288 ips_scb_t *
289 ips_scbctrl_alloc_tiny(struct ips_scbctrl *scbc)
290 {
291 ips_scb_t *scb;
292 if (SLIST_EMPTY(&scbc->scb_free))
293 return NULL;
294 scb = SLIST_FIRST(&scbc->scb_free);
295
296 VALGRIND_MEMPOOL_ALLOC(scbc, scb, sizeof(struct ips_scb));
297 SLIST_REMOVE_HEAD(&scbc->scb_free, next);
298 SLIST_NEXT(scb, next) = NULL;
299
300 scb->payload = NULL;
301 scb->payload_size = 0;
302 scb->flags = 0;
303 scb->tid = IPATH_EAGER_TID_ID;
304 scb->tidsendc = NULL;
305 scb->callback = NULL;
306 scb->nfrag = 1;
307 scb->frag_size = 0;
308
309 scbc->scb_num_cur--;
310 if (scbc->scb_num_cur < (scbc->scb_num >> 1))
311 scb->flags |= IPS_SEND_FLAG_ACK_REQ;
312 return scb;
313 }
314