Debian Codegraph @

ptl_ips/ptl_rcvthread.c

raw
1 /*
2 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
3 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <sys/poll.h>
35
36 #include "ptl_ips.h"
37 #include "ips_proto.h"
38 #include "ips_proto_internal.h"
39 #include "ips_recvhdrq.h"
40
41 /* All in milliseconds */
42 #define RCVTHREAD_TO_MIN_FREQ 10 /* min of 10 polls per sec */
43 #define RCVTHREAD_TO_MAX_FREQ 100 /* max of 100 polls per sec */
44 #define RCVTHREAD_TO_SHIFT 1
45
46 struct ptl_rcvthread;
47
48 static void * ips_ptl_pollintr(void *recvthreadc);
49 static psm_error_t rcvthread_initstats(ptl_t *ptl);
50 static psm_error_t rcvthread_initsched(struct ptl_rcvthread *rcvc);
51
52 struct ptl_rcvthread {
53 const psmi_context_t *context;
54 const ptl_t *ptl;
55 struct ips_recvhdrq *recvq;
56
57 pthread_t hdrq_threadid;
58 uint64_t t_start_cyc;
59 int pipefd[2];
60
61 /* stats and some for scheduling */
62 uint64_t pollcnt;
63 uint64_t pollcnt_to;
64 uint64_t pollcyc;
65 uint64_t pollok;
66
67 /* For scheduling interrupt thread */
68 int timeout_period_min;
69 int timeout_period_max;
70 int timeout_shift;
71 uint64_t pollok_last;
72 uint64_t pollcnt_last;
73 uint32_t last_timeout;
74 };
75
76 /*
77 * The receive thread knows about the ptl interface, so it can muck with it
78 * directly.
79 */
80 psm_error_t
81 ips_ptl_rcvthread_init(ptl_t *ptl, struct ips_recvhdrq *recvq)
82 {
83 psm_error_t err = PSM_OK;
84 struct ptl_rcvthread *rcvc;
85
86 ptl->rcvthread =
87 psmi_calloc(ptl->ep, UNDEFINED, 1, sizeof(struct ptl_rcvthread));
88 if (ptl->rcvthread == NULL) {
89 err = PSM_NO_MEMORY;
90 goto fail;
91 }
92 rcvc = ptl->rcvthread;
93
94 rcvc->recvq = recvq;
95 rcvc->ptl = ptl;
96 rcvc->context = ptl->context;
97 rcvc->t_start_cyc = get_cycles();
98
99 if (ptl->runtime_flags & PSMI_RUNTIME_RCVTHREAD) {
100
101 if ((err = rcvthread_initsched(rcvc)))
102 goto fail;
103
104 /* Create a pipe so we can synchronously terminate the thread */
105 if (pipe(rcvc->pipefd) != 0) {
106 err = psmi_handle_error(ptl->ep, PSM_EP_DEVICE_FAILURE,
107 "Cannot create a pipe for receive thread: %s\n",
108 strerror(errno));
109 goto fail;
110 }
111
112 if (pthread_create(&rcvc->hdrq_threadid, NULL,
113 ips_ptl_pollintr, ptl->rcvthread))
114 {
115 close(rcvc->pipefd[0]);
116 close(rcvc->pipefd[1]);
117 err = psmi_handle_error(ptl->ep, PSM_EP_DEVICE_FAILURE,
118 "Cannot start receive thread: %s\n", strerror(errno));
119 goto fail;
120 }
121
122 }
123
124 if ((err = rcvthread_initstats(ptl)))
125 goto fail;
126
127 fail:
128 return err;
129 }
130
131 psm_error_t
132 ips_ptl_rcvthread_fini(ptl_t *ptl)
133 {
134 struct ptl_rcvthread *rcvc = (struct ptl_rcvthread *) ptl->rcvthread;
135 uint64_t t_now;
136 double t_cancel_us;
137 psm_error_t err = PSM_OK;
138
139 PSMI_PLOCK_ASSERT();
140
141 if (ptl->rcvthread == NULL)
142 return err;
143
144 if (ptl->runtime_flags & PSMI_RUNTIME_RCVTHREAD) {
145 t_now = get_cycles();
146
147 /* Disable interrupts then kill the receive thread */
148 if (psmi_context_interrupt_isenabled((psmi_context_t *) ptl->context))
149 if ((err = psmi_context_interrupt_set((psmi_context_t *) ptl->context, 0)))
150 goto fail;
151
152 /* Close the pipe so we can have the thread synchronously exit.
153 On Linux just closing the pipe does not wake up the receive
154 thread.
155 */
156 if (write(rcvc->pipefd[1], (const void*) &t_now,
157 sizeof(uint64_t)) == -1 ||
158 close(rcvc->pipefd[1]) == -1) {
159 _IPATH_VDBG("unable to close pipe to receive thread cleanly\n");
160 }
161 pthread_join(rcvc->hdrq_threadid, NULL);
162 t_cancel_us = (double) cycles_to_nanosecs(get_cycles() - t_now) / 1e3;
163
164 _IPATH_PRDBG("rcvthread poll success %lld/%lld times, "
165 "thread cancelled in %.3f us\n", (long long) rcvc->pollok,
166 (long long) rcvc->pollcnt, t_cancel_us);
167
168 }
169
170 psmi_free(ptl->rcvthread);
171
172 fail:
173 return err;
174 }
175
176 psm_error_t
177 rcvthread_initsched(struct ptl_rcvthread *rcvc)
178 {
179 union psmi_envvar_val env_to;
180 char buf[192];
181 char *rcv_freq = buf;
182 int no_timeout = 0;
183 int tvals[3] = { RCVTHREAD_TO_MIN_FREQ,
184 RCVTHREAD_TO_MAX_FREQ,
185 RCVTHREAD_TO_SHIFT };
186 snprintf(buf, sizeof buf - 1, "%d:%d:%d", RCVTHREAD_TO_MIN_FREQ,
187 RCVTHREAD_TO_MAX_FREQ, RCVTHREAD_TO_SHIFT);
188 buf[sizeof buf - 1] = '\0';
189
190 if (!psmi_getenv("PSM_RCVTHREAD_FREQ",
191 "Thread timeouts (per sec) <min_freq[:max_freq[:shift_freq]]>",
192 PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_STR,
193 (union psmi_envvar_val) rcv_freq, &env_to))
194 {
195 /* not using default values */
196 int nparsed = psmi_parse_str_tuples(env_to.e_str, 3, tvals);
197 int invalid = 0;
198
199 if (nparsed < 1 || (nparsed > 0 && tvals[0] == 0) ||
200 (nparsed > 1 && tvals[1] == 0))
201 {
202 no_timeout = 1;
203 }
204 else {
205 if (nparsed > 0 && tvals[0] > 1000)
206 invalid = 1;
207 if (nparsed > 1 && (tvals[1] > 1000 || tvals[1] < tvals[0]))
208 invalid = 1;
209 if (nparsed > 2 && tvals[2] > 10)
210 invalid = 1;
211 }
212
213 if (invalid) {
214 _IPATH_INFO("Overriding invalid request for RcvThread frequency"
215 " settings of %s to be <%d:%d:%d>\n",
216 env_to.e_str, RCVTHREAD_TO_MIN_FREQ,
217 RCVTHREAD_TO_MAX_FREQ, RCVTHREAD_TO_SHIFT);
218 tvals[0] = RCVTHREAD_TO_MIN_FREQ;
219 tvals[1] = RCVTHREAD_TO_MAX_FREQ;
220 tvals[2] = RCVTHREAD_TO_SHIFT;
221 }
222 }
223
224 if (no_timeout) {
225 rcvc->last_timeout = -1;
226 _IPATH_PRDBG("PSM_RCVTHREAD_FREQ set to only interrupt "
227 "(no timeouts)\n");
228 }
229 else {
230 /* Convert freq to period in microseconds (for poll()) */
231 rcvc->timeout_period_max = 1000 / tvals[0];
232 rcvc->timeout_period_min = 1000 / tvals[1];
233 rcvc->timeout_shift = tvals[2];
234 /* Start in the middle of min and max */
235 rcvc->last_timeout = (rcvc->timeout_period_min +
236 rcvc->timeout_period_max) / 2;
237 _IPATH_PRDBG("PSM_RCVTHREAD_FREQ converted to period "
238 "min=%dms,max=%dms,shift=%d\n",
239 rcvc->timeout_period_min, rcvc->timeout_period_max,
240 rcvc->timeout_shift);
241 }
242 return PSM_OK;
243 }
244
245 static
246 int
247 rcvthread_next_timeout(struct ptl_rcvthread *rcvc)
248 {
249 uint64_t pollok_diff = rcvc->pollok - rcvc->pollok_last;
250
251 if (pollok_diff > 0) {
252 if (rcvc->last_timeout > rcvc->timeout_period_min)
253 /* By default, be less aggressive, but there's a more aggressive
254 * alternative if need be */
255 #if 1
256 rcvc->last_timeout >>= rcvc->timeout_shift;
257 #else
258 rcvc->last_timeout = rcvc->timeout_period_min;
259 #endif
260 }
261 else { /* we had less progress */
262 if (rcvc->last_timeout < rcvc->timeout_period_max)
263 rcvc->last_timeout <<= rcvc->timeout_shift;
264 }
265
266 rcvc->pollok_last = rcvc->pollok;
267 rcvc->pollcnt_last = rcvc->pollcnt;
268 return (int) rcvc->last_timeout;
269 }
270
271 extern int ips_in_rcvthread;
272
273 /*
274 * Receiver thread support.
275 *
276 * By default, polling in the driver asks the chip to generate an interrupt on
277 * every packet. When the driver supports POLLURG we can switch the poll mode
278 * to one that requests interrupts only for packets that contain an urgent bit
279 * (and optionally enable interrupts for hdrq overflow events). When poll
280 * returns an event, we *try* to make progress on the receive queue but simply
281 * go back to sleep if we notice that the main thread is already making
282 * progress.
283 */
284 static
285 void *
286 ips_ptl_pollintr(void *rcvthreadc)
287 {
288 struct ptl_rcvthread *rcvc = (struct ptl_rcvthread *) rcvthreadc;
289 struct ips_recvhdrq *recvq = rcvc->recvq;
290 psmi_context_t *context = (psmi_context_t *) rcvc->context;
291 int fd_dev = context->fd;
292 int fd_pipe = rcvc->pipefd[0];
293 psm_ep_t ep = context->ep;
294 struct pollfd pfd[2];
295 int ret;
296 int next_timeout = rcvc->last_timeout;
297 uint64_t t_cyc;
298 psm_error_t err;
299
300 /* No reason to have many of these, keep this as a backup in case the
301 * recvhdrq init function is misused */
302 psmi_assert_always((recvq->runtime_flags & PSMI_RUNTIME_RCVTHREAD));
303
304 /* Switch driver to a mode where it can interrupt on urgent packets */
305 if (psmi_context_interrupt_set((psmi_context_t *)
306 rcvc->context, 1) == PSM_EP_NO_RESOURCES) {
307 _IPATH_PRDBG("ipath_poll_type feature not present in driver, turning "
308 "off internal progress thread\n");
309 return NULL;
310 }
311
312 _IPATH_PRDBG("Enabled communication thread on URG packets\n");
313
314 while (1) {
315 pfd[0].fd = fd_dev;
316 pfd[0].events = POLLIN;
317 pfd[0].revents = 0;
318 pfd[1].fd = fd_pipe;
319 pfd[1].events = POLLIN;
320 pfd[1].revents = 0;
321
322 ret = poll(pfd, 2, next_timeout);
323 t_cyc = get_cycles();
324
325 if_pf (ret < 0) {
326 if (errno == EINTR)
327 _IPATH_DBG("got signal, keep polling\n");
328 else
329 psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
330 "Receive thread poll() error: %s", strerror(errno));
331 }
332 else if (pfd[1].revents) {
333 /* Any type of event on this fd means exit, should be POLLHUP */
334 _IPATH_DBG("close thread: revents=0x%x\n", pfd[1].revents);
335 close(fd_pipe);
336 break;
337 }
338 else {
339 rcvc->pollcnt++;
340
341 if (ret == 0 || pfd[0].revents & (POLLIN | POLLERR)) {
342 if (PSMI_PLOCK_DISABLED) {
343 /* We do this check without acquiring the lock, no sense to
344 * adding the overhead and it doesn't matter if we're
345 * wrong. */
346 if (ips_recvhdrq_isempty(recvq))
347 continue;
348 if (!ips_recvhdrq_trylock(recvq))
349 continue;
350 err = ips_recvhdrq_progress(recvq);
351 if (err == PSM_OK)
352 rcvc->pollok++;
353 else
354 rcvc->pollcyc += get_cycles() - t_cyc;
355 ips_recvhdrq_unlock(recvq);
356 }
357 else if (!PSMI_PLOCK_TRY()) {
358 /* If we time out, we service shm and ipath. If not, we
359 * assume to have received an ipath interrupt and service
360 * only ipath.
361 */
362 err = psmi_poll_internal(ep,
363 ret == 0 ? PSMI_TRUE : PSMI_FALSE);
364
365 if (err == PSM_OK) {
366 rcvc->pollok++;
367 /*
368 if (rcvc->pollok % 1000 == 0 && rcvc->pollok >= 1000)
369 _IPATH_INFO("pollok = %lld\n", (unsigned long long)rcvc->pollok);
370 */
371 }
372 else
373 rcvc->pollcyc += get_cycles() - t_cyc;
374 PSMI_PUNLOCK();
375 }
376 }
377
378 if (ret == 0) { /* change timeout only on timed out poll */
379 rcvc->pollcnt_to++;
380 next_timeout = rcvthread_next_timeout(rcvc);
381 }
382 }
383 }
384
385 return NULL;
386 }
387
388 static uint64_t
389 rcvthread_stats_pollok(void *context)
390 {
391 struct ptl_rcvthread *rcvc = (struct ptl_rcvthread *) context;
392 double ratio = 0.0;
393 uint64_t ratio_u;
394 if (rcvc->pollcnt > 0)
395 ratio = (double) rcvc->pollok * 100.0 / rcvc->pollcnt;
396 memcpy(&ratio_u, &ratio, sizeof(uint64_t));
397 return ratio_u;
398 }
399
400 static uint64_t
401 rcvthread_stats_pollcyc(void *context)
402 {
403 struct ptl_rcvthread *rcvc = (struct ptl_rcvthread *) context;
404 /* log in milliseconds */
405 return (uint64_t) ((double) cycles_to_nanosecs(rcvc->pollcyc) / 1.0e6);
406 }
407
408 static psm_error_t
409 rcvthread_initstats(ptl_t *ptl)
410 {
411 struct ptl_rcvthread *rcvc = (struct ptl_rcvthread *) ptl->rcvthread;
412 struct psmi_stats_entry entries[] = {
413 PSMI_STATS_DECL("intrthread schedule count",
414 MPSPAWN_STATS_REDUCTION_ALL | MPSPAWN_STATS_SKIP_IF_ZERO,
415 NULL, &rcvc->pollcnt),
416 PSMI_STATS_DECL("intrthread schedule success (%)",
417 MPSPAWN_STATS_REDUCTION_ALL |
418 MPSPAWN_STATS_TYPE_DOUBLE,
419 rcvthread_stats_pollok, NULL),
420 PSMI_STATS_DECL("intrthread timeout count",
421 MPSPAWN_STATS_REDUCTION_ALL | MPSPAWN_STATS_SKIP_IF_ZERO,
422 NULL, &rcvc->pollcnt_to),
423 PSMI_STATS_DECL("intrthread wasted time (ms)",
424 MPSPAWN_STATS_REDUCTION_ALL,
425 rcvthread_stats_pollcyc, NULL)
426 };
427
428 /* If we don't want a thread, make sure we still initialize the counters
429 * but set them to NaN instead */
430 if (!(ptl->runtime_flags & PSMI_RUNTIME_RCVTHREAD)) {
431 int i;
432 static uint64_t ctr_nan = MPSPAWN_NAN;
433 for (i = 0; i < (int) PSMI_STATS_HOWMANY(entries); i++) {
434 entries[i].getfn = NULL;
435 entries[i].u.val = &ctr_nan;
436 }
437 }
438
439 return psmi_stats_register_type(PSMI_STATS_NO_HEADING,
440 PSMI_STATSTYPE_RCVTHREAD,
441 entries,
442 PSMI_STATS_HOWMANY(entries),
443 rcvc);
444 }