Debian Codegraph @

ptl_ips/ptl.c

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 /* This file implements the PSM PTL for ips */
36 #include "psm_user.h"
37 #include "ptl_ips.h"
38 #include "ipserror.h"
39
40 int ips_ptl_recvq_isempty(const struct ptl *ptl);
41
42 #define PSMI_CONTEXT_STATUS_CHECK_INTERVAL_MSECS 250
43
44 static
45 int
46 ips_subcontext_ignore(const struct ips_recvhdrq_event *rcv_ev, uint32_t subcontext)
47 {
48 return IPS_RECVHDRQ_CONTINUE;
49 }
50
51 static
52 int
53 ips_subcontext_process(const struct ips_recvhdrq_event *rcv_ev, uint32_t subcontext)
54 {
55 struct ptl_shared *recvshc = rcv_ev->proto->ptl->recvshc;
56 if_pt (subcontext != recvshc->subcontext &&
57 subcontext < recvshc->subcontext_cnt) {
58 return ips_writehdrq_append(&recvshc->writeq[subcontext], rcv_ev);
59 }
60 else {
61 _IPATH_VDBG("Drop pkt for subcontext %d out of %d (I am %d) : errors 0x%x\n",
62 (int) subcontext, (int) recvshc->subcontext_cnt,
63 (int) recvshc->subcontext, (unsigned) rcv_ev->error_flags);
64 return IPS_RECVHDRQ_BREAK;
65 }
66 }
67
68 static
69 void
70 recvhdrq_hw_params(const psmi_context_t *context,
71 struct ips_recvq_params *hdrq,
72 struct ips_recvq_params *egrq,
73 int is_shared_context, int subcontext)
74 {
75 const struct ipath_base_info *base_info = &context->base_info;
76
77 hdrq->elemcnt = base_info->spi_rcvhdr_cnt;
78 hdrq->elemsz = base_info->spi_rcvhdrent_size;
79
80 egrq->elemsz = base_info->spi_rcv_egrbufsize; /* bytes */
81 egrq->elemcnt = base_info->spi_tidegrcnt; /* words */
82
83 if (!is_shared_context) {
84 volatile uint64_t *uregbase = /* HW registers */
85 (volatile uint64_t *) (uintptr_t) base_info->spi_uregbase;
86 hdrq->base_addr = (uint32_t *)(uintptr_t) base_info->spi_rcvhdr_base;
87 hdrq->head_register = (volatile __le32 *) &uregbase[ur_rcvhdrhead];
88 hdrq->tail_register = (volatile __le32 *) (uintptr_t)
89 base_info->spi_rcvhdr_tailaddr;
90 egrq->base_addr = (void *) (uintptr_t) base_info->spi_rcv_egrbufs;
91 egrq->head_register = (volatile __le32 *)
92 &uregbase[ur_rcvegrindexhead];
93 egrq->tail_register = (volatile __le32 *)
94 &uregbase[ur_rcvegrindextail];
95 }
96 else {
97 /* Subcontexts mimic the HW registers but use different addresses
98 * to avoid cache contention. */
99 volatile uint64_t *subcontext_uregbase;
100 uint32_t *rcv_hdr;
101 void *rcv_egr;
102 unsigned pagesize = getpagesize();
103 unsigned hdrsize, egrsize;
104 unsigned i = pagesize - 1;
105 hdrsize = (base_info->spi_rcvhdr_cnt * sizeof(uint32_t) *
106 base_info->spi_rcvhdrent_size + i) & ~i;
107 egrsize = base_info->spi_rcv_egrbuftotlen;
108 subcontext_uregbase = (uint64_t *)
109 (((uintptr_t) base_info->spi_subctxt_uregbase) +
110 (pagesize * subcontext));
111 rcv_hdr = (uint32_t *)
112 (((uintptr_t) base_info->spi_subctxt_rcvhdr_base +
113 (hdrsize * subcontext)));
114 rcv_egr = (void *)
115 (((uintptr_t) base_info->spi_subctxt_rcvegrbuf +
116 (egrsize * subcontext)));
117 hdrq->base_addr = (uint32_t *) rcv_hdr;
118 hdrq->head_register = (volatile __le32 *)
119 &subcontext_uregbase[ur_rcvhdrhead * 8];
120 hdrq->tail_register = (volatile __le32 *) (uintptr_t)
121 &subcontext_uregbase[ur_rcvhdrtail * 8];
122 egrq->base_addr = rcv_egr;
123 egrq->head_register = (volatile __le32 *)
124 &subcontext_uregbase[ur_rcvegrindexhead * 8];
125 egrq->tail_register = (volatile __le32 *)
126 &subcontext_uregbase[ur_rcvegrindextail * 8];
127 }
128 }
129
130 static psm_error_t shrecvq_init(ptl_t *ptl, const psmi_context_t *context);
131 static psm_error_t shrecvq_fini(ptl_t *ptl);
132
133 static
134 size_t
135 ips_ptl_sizeof(void)
136 {
137 return sizeof(ptl_t);
138 }
139
140 static
141 int
142 ips_ptl_epaddr_stats_num(void)
143 {
144 return sizeof(struct ptl_epaddr_stats) / sizeof (uint64_t);
145 }
146
147 static
148 int
149 ips_ptl_epaddr_stats_init(char **desc, uint16_t *flags)
150 {
151 int num_stats = sizeof(struct ptl_epaddr_stats) / sizeof (uint64_t);
152 int i;
153
154 /* All stats are uint64_t */
155 for (i = 0; i < num_stats; i++)
156 flags[i] = MPSPAWN_STATS_REDUCTION_ALL |
157 MPSPAWN_STATS_SKIP_IF_ZERO;
158
159 desc[0] = "errchecks sent";
160 desc[1] = "errchecks recv";
161 desc[2] = "naks sent";
162 desc[3] = "naks recv";
163 desc[4] = "connect reqs sent";
164 desc[5] = "disconnect reqs sent";
165 desc[6] = "tid grants sent";
166 desc[7] = "tid grants recv";
167 desc[8] = "send rexmit";
168 desc[9] = "congestion packets";
169
170 return num_stats;
171 }
172
173 int
174 ips_ptl_epaddr_stats_get(psm_epaddr_t epaddr, uint64_t *stats_o)
175 {
176 struct ptl_epaddr *ipsaddr = epaddr->ptladdr;
177 int i, num_stats = sizeof(struct ptl_epaddr_stats) / sizeof (uint64_t);
178 uint64_t *stats_i = (uint64_t *) &ipsaddr->stats;
179
180 for (i = 0; i < num_stats; i++)
181 stats_o[i] = stats_i[i];
182
183 return num_stats;
184 }
185
186 static psm_error_t
187 psmi_context_check_status_callback(struct psmi_timer *t, uint64_t current)
188 {
189 struct ptl *ptl = (struct ptl *) t->context;
190 const uint64_t current_count = get_cycles();
191 psm_error_t err;
192
193 err = psmi_context_check_status(ptl->context);
194 psmi_timer_request_always(&ptl->timerq, &ptl->status_timer,
195 current_count + ptl->status_cyc_timeout);
196
197 return err;
198 }
199
200 static
201 psm_error_t
202 ips_ptl_init(const psm_ep_t ep, ptl_t *ptl, ptl_ctl_t *ctl)
203 {
204 psm_error_t err = PSM_OK;
205 uint32_t num_of_send_bufs = ep->ipath_num_sendbufs;
206 uint32_t num_of_send_desc = ep->ipath_num_descriptors;
207 uint32_t imm_size = ep->ipath_imm_size;
208 const psmi_context_t *context = &ep->context;
209 const struct ipath_user_info *user_info = &context->user_info;
210 const int enable_shcontexts = (user_info->spu_subcontext_cnt > 0);
211 const uint64_t current_count = get_cycles();
212
213 /* Preconditions */
214 psmi_assert_always(ep != NULL);
215 psmi_assert_always(ep->epaddr != NULL);
216 psmi_assert_always(ep->epid != 0);
217 psmi_assert_always(ep->ipath_num_sendbufs > 0);
218
219 memset(ptl, 0, sizeof(struct ptl));
220
221 ptl->ep = ep; /* back pointer */
222 ptl->epid = ep->epid; /* cache epid */
223 ptl->epaddr = ep->epaddr; /* cache a copy */
224 ptl->ctl = ctl;
225 ptl->context = context;
226 ptl->runtime_flags = context->runtime_flags;
227
228 memset(ctl, 0, sizeof(*ctl));
229 /* Fill in the control structure */
230 ctl->ptl = ptl;
231 ctl->ep_poll = enable_shcontexts ? ips_ptl_shared_poll : ips_ptl_poll;
232 ctl->ep_connect = ips_ptl_connect;
233 ctl->ep_disconnect = ips_ptl_disconnect;
234 ctl->mq_send = ips_proto_mq_send;
235 ctl->mq_isend = ips_proto_mq_isend;
236
237 ctl->am_short_request = ips_am_short_request;
238 ctl->am_short_reply = ips_am_short_reply;
239
240 ctl->epaddr_stats_num = ips_ptl_epaddr_stats_num;
241 ctl->epaddr_stats_init = ips_ptl_epaddr_stats_init;
242 ctl->epaddr_stats_get = ips_ptl_epaddr_stats_get;
243
244 /*
245 * Runtime flags in 'ptl' are different from runtime flags in 'context'.
246 * In 'context', runtime flags reflect what the driver is capable of.
247 * In 'ptl', runtime flags reflect the features we can or want to use in
248 * the driver's supported runtime flags.
249 */
250
251 /*
252 * This timer is to be used to check the context's status at every
253 * PSMI_CONTEXT_STATUS_CHECK_INTERVAL_MSECS. This is useful to detect when
254 * the link transitions from the DOWN state to the UP state. We can thus
255 * stop aggregating link failure messages once we detect that the link is
256 * up.
257 */
258 psmi_timer_entry_init(&ptl->status_timer,
259 psmi_context_check_status_callback, ptl);
260
261 /* cache the context's status timeout in cycles */
262 ptl->status_cyc_timeout =
263 ms_2_cycles(PSMI_CONTEXT_STATUS_CHECK_INTERVAL_MSECS);
264
265 /*
266 * Retransmissions and pending operations are kept in a timer structure
267 * (queue). The timerq is shared to various internal IPS interfaces so
268 * that they too may schedule events on the timer queue. The timerq is
269 * drained in the progress function.
270 */
271 if ((err = psmi_timer_init(&ptl->timerq)))
272 goto fail;
273
274 /* start the context's status timer */
275 psmi_timer_request_always(&ptl->timerq, &ptl->status_timer,
276 current_count + ptl->status_cyc_timeout);
277
278 /*
279 * Hardware send pio used by eager and control messages.
280 */
281 if ((err = ips_spio_init(context, ptl, &ptl->spioc)))
282 goto fail;
283
284 /*
285 * Epstate maps endpoint ids (epid integers) to ipsaddr (structs). Mappings
286 * are added/removed by the connect portion of the ips protocol and lookup
287 * is made by the receive queue processing component.
288 */
289 if ((err = ips_epstate_init(&ptl->epstate, context)))
290 goto fail;
291
292 /*
293 * Actual ips protocol handling.
294 */
295 if ((err = ips_proto_init(context, ptl, num_of_send_bufs, num_of_send_desc,
296 imm_size, &ptl->timerq, &ptl->epstate,
297 &ptl->spioc, &ptl->proto)))
298 goto fail;
299
300 /*
301 * Hardware receive hdr/egr queue, services incoming packets and issues
302 * callbacks for protocol handling in proto_recv. It uses the epstate
303 * interface to determine if a packet is known or unknown.
304 */
305 if (!enable_shcontexts) {
306 struct ips_recvhdrq_callbacks recvq_callbacks;
307 struct ips_recvq_params hdrq, egrq;
308 recvhdrq_hw_params(context, &hdrq, &egrq, 0, 0);
309 recvq_callbacks.callback_packet_unknown = ips_proto_process_unknown;
310 recvq_callbacks.callback_subcontext = ips_subcontext_ignore;
311 recvq_callbacks.callback_error = ips_proto_process_packet_error;
312 if ((err = ips_recvhdrq_init(context, &ptl->epstate, &ptl->proto,
313 &hdrq, &egrq, &recvq_callbacks,
314 ptl->runtime_flags, 0,
315 &ptl->recvq, &ptl->recvq_state)))
316 goto fail;
317 }
318
319 /*
320 * Software receive hdr/egr queue, used in shared contexts.
321 */
322 if (enable_shcontexts && (err = shrecvq_init(ptl, context)))
323 goto fail;
324
325 /*
326 * Receive thread, always initialized but not necessary creates a
327 * pthread.
328 */
329 if ((err = ips_ptl_rcvthread_init(ptl, &ptl->recvq)))
330 goto fail;
331 fail:
332 return err;
333 }
334
335 static
336 psm_error_t
337 ips_ptl_fini(ptl_t *ptl, int force, uint64_t timeout_in)
338 {
339 const struct ipath_user_info *user_info = &ptl->context->user_info;
340 const int enable_shcontexts = (user_info->spu_subcontext_cnt > 0);
341 psm_error_t err = PSM_OK;
342
343 if ((err = ips_proto_fini(&ptl->proto, force, timeout_in)))
344 goto fail;
345
346 /* We have to cancel the thread after terminating the protocol because
347 * connect/disconnect packets use interrupts and the kernel doesn't
348 * like to have no pollers waiting */
349 if ((err = ips_ptl_rcvthread_fini(ptl)))
350 goto fail;
351
352 if ((err = ips_epstate_fini(&ptl->epstate)))
353 goto fail;
354
355 if ((err = ips_spio_fini(&ptl->spioc)))
356 goto fail;
357
358 if ((err = psmi_timer_fini(&ptl->timerq)))
359 goto fail;
360
361 if (!enable_shcontexts && (err = ips_recvhdrq_fini(&ptl->recvq)))
362 goto fail;
363
364 if (enable_shcontexts && (err = shrecvq_fini(ptl)))
365 goto fail;
366
367 fail:
368 return err;
369 }
370
371 static
372 psm_error_t
373 ips_ptl_optctl(const void *core_obj, int optname,
374 void *optval, uint64_t *optlen, int get)
375 {
376 psm_error_t err = PSM_OK;
377
378 switch(optname) {
379 case PSM_IB_OPT_EP_SL:
380 {
381 /* Core object is psm_epaddr */
382 psm_epaddr_t epaddr = (psm_epaddr_t) core_obj;
383 ips_epaddr_t *ipsaddr = epaddr->ptladdr;
384
385 /* If endpoint does not use IB ignore for set, complain for get */
386 if (epaddr->ptlctl->ep_connect != ips_ptl_connect) {
387 if (get)
388 err = psmi_handle_error(PSMI_EP_LOGEVENT,
389 PSM_PARAM_ERR, "Invalid EP transport");
390 goto exit_fn;
391 }
392
393 /* Sanity check option length */
394 if (*optlen < sizeof(uint8_t)) {
395 err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR,
396 "Option value length error");
397 *optlen = sizeof(unsigned);
398 goto exit_fn;
399 }
400
401 if (get) {
402 /* Get returns the SL for the PIO flow */
403 *((uint8_t *) optval) =
404 (uint8_t) ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO].sl;
405 }
406 else {
407 uint16_t new_sl;
408
409 /* Sanity check if SL is within range */
410 new_sl = (uint16_t) *(uint8_t*) optval;
411 if (new_sl > 15) {
412 err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR,
413 "Invalid SL value %u. 0 <= SL <= 15.",new_sl);
414 goto exit_fn;
415 }
416
417 /* Set new SL for all flows */
418 ipsaddr->flows[EP_FLOW_GO_BACK_N_PIO].sl = new_sl;
419 ipsaddr->flows[EP_FLOW_GO_BACK_N_DMA].sl = new_sl;
420 ipsaddr->flows[EP_FLOW_GO_BACK_N_AM_REQ].sl = new_sl;
421 ipsaddr->flows[EP_FLOW_GO_BACK_N_AM_RSP].sl = new_sl;
422 }
423 }
424 break;
425 case PSM_IB_OPT_DF_SL:
426 {
427 /* Set default SL to be used by an endpoint for all communication */
428 /* Core object is psm_epaddr */
429 psm_ep_t ep = (psm_ep_t) core_obj;
430
431 /* Make sure ep is specified */
432 if (!ep) {
433 err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR,
434 "Invalid PSM Endpoint");
435 goto exit_fn;
436 }
437
438 /* Sanity check option length */
439 if (*optlen < sizeof(uint8_t)) {
440 err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR,
441 "Option value length error");
442 *optlen = sizeof(uint8_t);
443 goto exit_fn;
444 }
445
446 if (get) {
447 *((uint8_t *) optval) = ep->ptl_ips.ptl->proto.epinfo.ep_sl;
448 }
449 else {
450 uint16_t new_sl;
451
452 /* Sanity check if SL is within range */
453 new_sl = (uint16_t) *(uint8_t*) optval;
454 if (new_sl > 15) {
455 err = psmi_handle_error(PSMI_EP_LOGEVENT, PSM_PARAM_ERR,
456 "Invalid SL value %u. 0 <= SL <= 15.",new_sl);
457 goto exit_fn;
458 }
459
460 ep->ptl_ips.ptl->proto.epinfo.ep_sl = (uint8_t) new_sl;
461 }
462 }
463 break;
464 default:
465 err = psmi_handle_error(NULL, PSM_PARAM_ERR, "Unknown PSM_IB option %u.", optname);
466 }
467
468 exit_fn:
469 return err;
470 }
471
472 static
473 psm_error_t
474 ips_ptl_setopt(const void *component_obj, int optname,
475 const void *optval, uint64_t optlen)
476 {
477 return ips_ptl_optctl(component_obj, optname, (void*) optval, &optlen, 0);
478 }
479
480 static
481 psm_error_t
482 ips_ptl_getopt(const void *component_obj, int optname,
483 void *optval, uint64_t *optlen)
484 {
485 return ips_ptl_optctl(component_obj, optname, optval, optlen, 1);
486 }
487
488 psm_error_t __recvpath
489 ips_ptl_poll(ptl_t *ptl, int _ignored)
490 {
491 const uint64_t current_count = get_cycles();
492 const int do_lock = PSMI_PLOCK_DISABLED &&
493 (ptl->runtime_flags & PSMI_RUNTIME_RCVTHREAD);
494 psm_error_t err = PSM_OK_NO_PROGRESS;
495 psm_error_t err2;
496
497 if (!ips_recvhdrq_isempty(&ptl->recvq)) {
498 if (do_lock && !ips_recvhdrq_trylock(&ptl->recvq))
499 return err;
500 err = ips_recvhdrq_progress(&ptl->recvq);
501 if (do_lock)
502 ips_recvhdrq_unlock(&ptl->recvq);
503 if_pf (err > PSM_OK_NO_PROGRESS)
504 return err;
505 err2 = psmi_timer_process_if_expired(&(ptl->timerq), current_count);
506 if (err2 != PSM_OK_NO_PROGRESS)
507 return err2;
508 else
509 return err;
510 }
511
512 /*
513 * Process timer expirations after servicing receive queues (some packets
514 * may have been acked, some requests-to-send may have been queued).
515 *
516 * It's safe to look at the timer without holding the lock because it's not
517 * incorrect to be wrong some of the time.
518 */
519 if (psmi_timer_is_expired(&(ptl->timerq), current_count)) {
520 if (do_lock)
521 ips_recvhdrq_lock(&ptl->recvq);
522 err = psmi_timer_process_expired(&(ptl->timerq), current_count);
523 if (do_lock)
524 ips_recvhdrq_unlock(&ptl->recvq);
525 }
526
527 return err;
528 }
529
530 PSMI_INLINE(
531 int
532 ips_try_lock_shared_context (struct ptl_shared *recvshc))
533 {
534 return pthread_spin_trylock(recvshc->context_lock);
535 }
536
537 PSMI_INLINE(
538 void
539 ips_lock_shared_context (struct ptl_shared *recvshc))
540 {
541 pthread_spin_lock(recvshc->context_lock);
542 }
543
544 PSMI_INLINE(
545 void
546 ips_unlock_shared_context (struct ptl_shared *recvshc))
547 {
548 pthread_spin_unlock(recvshc->context_lock);
549 }
550
551 psm_error_t __recvpath
552 ips_ptl_shared_poll(ptl_t *ptl, int _ignored)
553 {
554 const uint64_t current_count = get_cycles();
555 psm_error_t err = PSM_OK_NO_PROGRESS;
556 psm_error_t err2;
557 struct ptl_shared *recvshc = ptl->recvshc;
558 psmi_assert(recvshc != NULL);
559
560 /* The following header queue checks are speculative (but safe)
561 * until this process has acquired the lock. The idea is to
562 * minimize lock contention due to processes spinning on the
563 * shared context. */
564 if (ips_recvhdrq_isempty(&recvshc->recvq)) {
565 if (!ips_recvhdrq_isempty(&ptl->recvq) &&
566 ips_try_lock_shared_context(recvshc) == 0) {
567 /* check that subcontext is empty while under lock to avoid
568 * re-ordering of incoming packets (since packets from
569 * hardware context will be processed immediately). */
570 if_pt (ips_recvhdrq_isempty(&recvshc->recvq)) {
571 err = ips_recvhdrq_progress(&ptl->recvq);
572 }
573 ips_unlock_shared_context(recvshc);
574 }
575 }
576
577 if_pf (err > PSM_OK_NO_PROGRESS)
578 return err;
579
580 if (!ips_recvhdrq_isempty(&recvshc->recvq)) {
581 err2 = ips_recvhdrq_progress(&recvshc->recvq);
582 if (err2 != PSM_OK_NO_PROGRESS) {
583 err = err2;
584 }
585 }
586
587 if_pf (err > PSM_OK_NO_PROGRESS)
588 return err;
589
590 /*
591 * Process timer expirations after servicing receive queues (some packets
592 * may have been acked, some requests-to-send may have been queued).
593 */
594 err2 = psmi_timer_process_if_expired(&(ptl->timerq), current_count);
595 if (err2 != PSM_OK_NO_PROGRESS)
596 err = err2;
597
598 return err;
599 }
600
601 int __recvpath
602 ips_ptl_recvq_isempty(const ptl_t *ptl)
603 {
604 struct ptl_shared *recvshc = ptl->recvshc;
605
606 if (recvshc != NULL && !ips_recvhdrq_isempty(&recvshc->recvq))
607 return 0;
608 return ips_recvhdrq_isempty(&ptl->recvq);
609 }
610
611 /*
612 * Legacy ips_get_stat -- do nothing.
613 */
614 int ips_get_stat(psm_epaddr_t epaddr, ips_sess_stat * stats)
615 {
616 memset(stats, 0, sizeof (ips_sess_stat));
617 return 0;
618 }
619
620 static
621 psm_error_t
622 shrecvq_init(ptl_t *ptl, const psmi_context_t *context)
623 {
624 const struct ipath_base_info *base_info = &context->base_info;
625 const struct ipath_user_info *user_info = &context->user_info;
626 struct ips_recvhdrq_callbacks recvq_callbacks;
627 struct ips_recvq_params hdrq, egrq;
628 psm_error_t err = PSM_OK;
629 struct ptl_shared *recvshc;
630 int i;
631
632 psmi_assert_always(user_info->spu_subcontext_cnt > 0);
633
634 recvshc = (struct ptl_shared *)
635 psmi_calloc(context->ep, UNDEFINED, 1, sizeof(struct ptl_shared));
636 if (recvshc == NULL) {
637 err = PSM_NO_MEMORY;
638 goto fail;
639 }
640
641 ptl->recvshc = recvshc;
642 recvshc->ptl = ptl;
643
644 /* Initialize recvshc fields */
645 recvshc->subcontext = base_info->spi_subcontext;
646 recvshc->subcontext_cnt = user_info->spu_subcontext_cnt;
647 psmi_assert_always(recvshc->subcontext_cnt <= INFINIPATH_MAX_SUBCONTEXT);
648 psmi_assert_always(recvshc->subcontext < recvshc->subcontext_cnt);
649
650 if ((err = ips_subcontext_ureg_get(ptl, context, recvshc->subcontext_ureg,
651 recvshc->subcontext_cnt)))
652 goto fail;
653 if ((err = ips_subcontext_ureg_initialize(
654 ptl, recvshc->subcontext, recvshc->subcontext_ureg[recvshc->subcontext])))
655 goto fail;
656 recvshc->context_lock = &recvshc->subcontext_ureg[0]->context_lock;
657
658 /* Initialize (shared) hardware context recvq (ptl->recvq) */
659 /* NOTE: uses recvq in ptl structure for shared h/w context */
660 recvhdrq_hw_params(context, &hdrq, &egrq, 0, 0);
661 recvq_callbacks.callback_packet_unknown = ips_proto_process_unknown;
662 recvq_callbacks.callback_subcontext = ips_subcontext_process;
663 recvq_callbacks.callback_error = ips_proto_process_packet_error;
664 if ((err = ips_recvhdrq_init(context, &ptl->epstate, &ptl->proto,
665 &hdrq, &egrq, &recvq_callbacks,
666 ptl->runtime_flags, recvshc->subcontext,
667 &ptl->recvq,
668 &recvshc->subcontext_ureg[0]->recvq_state))) {
669 goto fail;
670 }
671
672 /* Initialize software subcontext (recvshc->recvq). Subcontexts do */
673 /* not require the rcvhdr copy feature. */
674 recvhdrq_hw_params(context, &hdrq, &egrq, 1, recvshc->subcontext);
675 recvq_callbacks.callback_subcontext = ips_subcontext_ignore;
676 if ((err = ips_recvhdrq_init(context, &ptl->epstate, &ptl->proto,
677 &hdrq, &egrq, &recvq_callbacks,
678 ptl->runtime_flags & ~IPATH_RUNTIME_RCVHDR_COPY,
679 recvshc->subcontext,
680 &recvshc->recvq,
681 &recvshc->recvq_state))) {
682 goto fail;
683 }
684
685 /* Initialize each recvshc->writeq for shared contexts */
686 for (i = 0; i < recvshc->subcontext_cnt; i++) {
687 recvhdrq_hw_params(context, &hdrq, &egrq, 1, i);
688 if ((err = ips_writehdrq_init(context, &hdrq, &egrq,
689 &recvshc->writeq[i],
690 &recvshc->subcontext_ureg[i]->writeq_state,
691 ptl->runtime_flags & ~IPATH_RUNTIME_RCVHDR_COPY))) {
692 goto fail;
693 }
694 }
695
696 if (err == PSM_OK)
697 _IPATH_DBG("Context sharing in use: lid %d, context %d, sub-context %d\n",
698 (int) psm_epid_nid(ptl->epid), base_info->spi_context,
699 recvshc->subcontext);
700 fail:
701 return err;
702 }
703
704 static
705 psm_error_t
706 shrecvq_fini(ptl_t *ptl)
707 {
708 psm_error_t err = PSM_OK;
709 int i;
710
711 /* disable my write header queue before deallocation */
712 i = ptl->recvshc->subcontext;
713 ptl->recvshc->subcontext_ureg[i]->writeq_state.enabled = 0;
714
715 if ((err = ips_recvhdrq_fini(&ptl->recvq)))
716 goto fail;
717
718 if ((err = ips_recvhdrq_fini(&ptl->recvshc->recvq)))
719 goto fail;
720
721 for (i = 0; i < ptl->recvshc->subcontext_cnt; i++) {
722 if ((err = ips_writehdrq_fini(&ptl->recvshc->writeq[i]))) {
723 goto fail;
724 }
725 }
726
727 psmi_free(ptl->recvshc);
728
729 fail:
730 return err;
731 }
732
733 psm_error_t
734 ips_ptl_connect(ptl_t *ptl, int numep, const psm_epid_t *array_of_epid,
735 const int *array_of_epid_mask, psm_error_t *array_of_errors,
736 psm_epaddr_t *array_of_epaddr, uint64_t timeout_in)
737 {
738 psm_error_t err;
739 psm_ep_t ep;
740 psm_epid_t *epid_array = NULL;
741 psm_error_t *error_array = NULL;
742 psm_epaddr_t *epaddr_array = NULL;
743 int *mask_array = NULL;
744 int i, count;
745
746 PSMI_PLOCK_ASSERT();
747 err = ips_proto_connect(&ptl->proto, numep, array_of_epid,
748 array_of_epid_mask, array_of_errors,
749 array_of_epaddr, timeout_in);
750 if (err) return err;
751
752 psmi_assert_always(ptl->ep->mctxt_master == ptl->ep);
753 if (ptl->ep->mctxt_next == ptl->ep) return err;
754
755 /* make the additional mutil-context connections. */
756 epid_array = (psm_epid_t *)
757 psmi_malloc(ptl->ep, UNDEFINED, sizeof(psm_epid_t)*numep);
758 mask_array = (int *)
759 psmi_malloc(ptl->ep, UNDEFINED, sizeof(int)*numep);
760 error_array = (psm_error_t *)
761 psmi_malloc(ptl->ep, UNDEFINED, sizeof(psm_error_t)*numep);
762 epaddr_array = (psm_epaddr_t *)
763 psmi_malloc(ptl->ep, UNDEFINED, sizeof(psm_epaddr_t)*numep);
764 if (!epid_array || !mask_array || !error_array || !epaddr_array) {
765 goto fail;
766 }
767
768 count = 0;
769 ep = ptl->ep->mctxt_next;
770 while (ep != ep->mctxt_master) {
771
772 /* Setup the mask array and epid array. */
773 for (i = 0; i < numep; i++) {
774 if (array_of_epid_mask[i]
775 && array_of_errors[i] == PSM_OK
776 && count < array_of_epaddr[i]->mctxt_epcount) {
777 if (ep->gid_hi != array_of_epaddr[i]->mctxt_gidhi[count]) {
778 mask_array[i] = 0;
779 _IPATH_INFO("Subnet ID mismatch, ignore...\n");
780 } else {
781 mask_array[i] = 1;
782 epid_array[i] = array_of_epaddr[i]->mctxt_epid[count];
783 }
784 } else {
785 mask_array[i] = 0;
786 }
787 }
788
789 /* Make the real protocol connections. */
790 err = ips_proto_connect(&ep->ptl_ips.ptl->proto, numep, epid_array,
791 mask_array, error_array,
792 epaddr_array, timeout_in);
793 if (err) goto fail;
794
795 /* Make the epaddr linklist for this peer. */
796 for (i = 0; i < numep; i++) {
797 if (!mask_array[i]) continue;
798
799 /* In rare case, when the peer exits psm_ep_connect()
800 * and sends a message, it is received by this epaddr,
801 * because the epaddr->mctxt_master is still itself (linked
802 * and changed by below macro), epaddr->mctxt_recv_seqnum
803 * is increased, not the master's mctxt_recv_seqnum.
804 * when this happens, we need to apply this mctxt_recv_seqnum
805 * to master's mctxt_recv_seqnum, otherwise, the message
806 * sequence number doesnot match master's mctxt_recv_seqnum,
807 * and causes code hanging.
808 * This case only happens in the last rail of multi-rail.
809 */
810 if (epaddr_array[i]->mctxt_recv_seqnum) {
811 array_of_epaddr[i]->mctxt_recv_seqnum +=
812 epaddr_array[i]->mctxt_recv_seqnum;
813 epaddr_array[i]->mctxt_recv_seqnum = 0;
814 }
815
816 PSM_MCTXT_APPEND(array_of_epaddr[i], epaddr_array[i]);
817
818 /* randomize the rail to start traffic */
819 if ((random()%(count+2)) == 0) {
820 array_of_epaddr[i]->mctxt_current = epaddr_array[i];
821 }
822
823 /* Set the # slave connections so far */
824 array_of_epaddr[i]->mctxt_nsconn++;
825 }
826
827 count++;
828 ep = ep->mctxt_next;
829 }
830
831 fail:
832 if (epid_array) psmi_free(epid_array);
833 if (mask_array) psmi_free(mask_array);
834 if (error_array) psmi_free(error_array);
835 if (epaddr_array) psmi_free(epaddr_array);
836
837 return err;
838 }
839
840 psm_error_t
841 ips_ptl_disconnect(ptl_t *ptl, int force, int numep,
842 const psm_epaddr_t array_of_epaddr[],
843 const int array_of_epaddr_mask[],
844 psm_error_t array_of_errors[], uint64_t timeout_in)
845 {
846 psm_error_t err;
847
848 fprintf(stderr, "Aiee! ips_proto_disconnect() called.\n");
849 PSMI_PLOCK_ASSERT();
850 err = ips_proto_disconnect(&ptl->proto, force, numep, array_of_epaddr,
851 array_of_epaddr_mask, array_of_errors,
852 timeout_in);
853 return err;
854 }
855
856 /* Only symbol we expose out of here */
857 struct ptl_ctl_init
858 psmi_ptl_ips = {
859 ips_ptl_sizeof, ips_ptl_init, ips_ptl_fini, ips_ptl_setopt, ips_ptl_getopt
860 };