Debian Codegraph @

ptl_ips/ips_writehdrq.h

raw
1 /*
2 * Copyright (c) 2013. Intel Corporation. All rights reserved.
3 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #ifndef _IPS_WRITEHDRQ_H
36 #define _IPS_WRITEHDRQ_H
37
38 #include "psm_user.h"
39 #include "ips_recvhdrq.h"
40 #include "ips_recvq.h"
41 #include "psm_mq_internal.h"
42
43 /*
44 * Structure containing state for writehdrq writing. This is logically
45 * part of ips_writehdrq but needs to be separated out for context
46 * sharing so that it can be put in a shared memory page and hence
47 * be available to all processes sharing the port. Generally, do not
48 * put pointers in here since the address map of each process can be
49 * different.
50 */
51 struct ips_writehdrq_state
52 {
53 uint32_t hdrq_rhf_seq; /* last seq */
54 uint32_t enabled; /* enables writing */
55 };
56
57 struct ips_writehdrq
58 {
59 const psmi_context_t *context;
60 struct ips_writehdrq_state *state;
61 struct ips_recvq_params hdrq;
62 uint32_t hdrq_elemlast;
63 uint32_t hdrq_rhf_off; /* rhf offset */
64 uint32_t hdrq_hdr_copysz;
65 struct ips_recvq_params egrq;
66 void **egrq_buftable; /* table of eager idx-to-ptr */
67 uint32_t runtime_flags;
68 };
69
70 psm_error_t
71 ips_writehdrq_init(const psmi_context_t *context,
72 const struct ips_recvq_params *hdrq_params,
73 const struct ips_recvq_params *egrq_params,
74 struct ips_writehdrq *writeq,
75 struct ips_writehdrq_state *state,
76 uint32_t runtime_flags);
77
78 psm_error_t
79 ips_writehdrq_fini(struct ips_writehdrq *writeq);
80
81 PSMI_ALWAYS_INLINE(
82 void
83 ips_writehdrq_write_rhf_atomic(uint32_t *rhf_dest, uint32_t *rhf_src))
84 {
85 #if WORDSIZE == 64
86 /*
87 * In 64-bit mode, we check in init that the rhf will always be 8-byte
88 * aligned
89 */
90 *((uint64_t *)rhf_dest) = *((uint64_t *)rhf_src);
91 #else
92 /*
93 * In 32-bit mode, we ensure that word 0 always gets written before word 1
94 */
95 rhf_dest[0] = rhf_src[0];
96 ips_wmb();
97 rhf_dest[1] = rhf_src[1];
98 #endif
99 return;
100 }
101
102 PSMI_INLINE(
103 int
104 ips_writehdrq_append(struct ips_writehdrq *writeq,
105 const struct ips_recvhdrq_event *rcv_ev))
106 {
107 const uint32_t *rcv_hdr = rcv_ev->rcv_hdr;
108 uint32_t write_hdr_head;
109 uint32_t write_hdr_tail;
110 uint32_t *write_hdr;
111 uint32_t *write_rhf;
112 char *write_payload = NULL;
113 uint32_t next_write_hdr_tail;
114 uint32_t rcv_paylen;
115 union {
116 uint32_t u32[2];
117 uint64_t u64;
118 } rhf;
119 int result = IPS_RECVHDRQ_CONTINUE;
120
121 /* Drop packet if write header queue is disabled */
122 if (!writeq->state->enabled) {
123 result = IPS_RECVHDRQ_BREAK;
124 goto done;
125 }
126
127 write_hdr_head = ips_recvq_head_get(&writeq->hdrq);
128 write_hdr_tail = ips_recvq_tail_get(&writeq->hdrq);
129 write_hdr = writeq->hdrq.base_addr + write_hdr_tail;
130 write_rhf = write_hdr + writeq->hdrq_rhf_off;
131
132 /* Drop packet if write header queue is full */
133 next_write_hdr_tail = write_hdr_tail + writeq->hdrq.elemsz;
134 if (next_write_hdr_tail > writeq->hdrq_elemlast)
135 next_write_hdr_tail = 0;
136 if (next_write_hdr_tail == write_hdr_head) {
137 result = IPS_RECVHDRQ_BREAK;
138 goto done;
139 }
140
141 /*
142 * If NORDMA_TAIL, don't let consumer see RHF until it's ready. We copy
143 * the source rhf and operate on it until we are ready to atomically update
144 * it for the reader.
145 */
146 if (writeq->runtime_flags & IPATH_RUNTIME_NODMA_RTAIL) {
147 write_rhf = &rhf.u32[0];
148 rhf.u64 = *((uint64_t *) rcv_ev->rhf);
149 }
150
151 /* Copy the data if this is an eager packet */
152 rcv_paylen = ips_recvhdrq_event_paylen(rcv_ev);
153 rcv_paylen += (rcv_ev->has_cksum ? PSM_CRC_SIZE_IN_BYTES : 0);
154
155 if (rcv_ev->ptype == RCVHQ_RCV_TYPE_EAGER && rcv_paylen > 0)
156 {
157 uint32_t write_egr_tail = ips_recvq_tail_get(&writeq->egrq);
158 uint32_t next_write_egr_tail;
159
160 /* Drop packet if write eager queue is full */
161 next_write_egr_tail = write_egr_tail + 1;
162 if (next_write_egr_tail >= writeq->egrq.elemcnt)
163 next_write_egr_tail = 0;
164 if (next_write_egr_tail == ips_recvq_head_get(&writeq->egrq)) {
165 /* Copy the header to the subcontext's header queue */
166 psmi_mq_mtucpy(write_hdr, rcv_hdr, writeq->hdrq_hdr_copysz);
167
168 /* Mark header with ETIDERR (eager overflow) */
169 ipath_hdrset_err_flags(write_rhf, INFINIPATH_RHF_H_TIDERR);
170
171 /* Fix up the header with current subcontext eager index */
172 ipath_hdrset_index(write_rhf, write_egr_tail);
173
174 result = IPS_RECVHDRQ_BREAK;
175 }
176 else {
177 if (rcv_paylen) {
178 const char *rcv_payload = ips_recvhdrq_event_payload(rcv_ev);
179
180 /* Use pre-calculated address from look-up table */
181 write_payload = ips_recvq_egr_index_2_ptr(
182 writeq->egrq_buftable, write_egr_tail);
183
184 psmi_mq_mtucpy(write_payload, rcv_payload, rcv_paylen);
185 }
186
187 /* Copy the header to the subcontext's header queue */
188 psmi_mq_mtucpy(write_hdr, rcv_hdr, writeq->hdrq_hdr_copysz);
189
190 /* Fix up the header with the subcontext's eager index */
191 ipath_hdrset_index((uint32_t *) write_rhf, write_egr_tail);
192
193 /* Update the eager buffer tail pointer */
194 ips_recvq_tail_update(&writeq->egrq, next_write_egr_tail);
195 }
196 }
197 else {
198 /* Copy the header to the subcontext's header queue */
199 psmi_mq_mtucpy(write_hdr, rcv_hdr, writeq->hdrq_hdr_copysz);
200
201 /* Copy the value of the current egr tail, handles the
202 * eager-with-no-payload case */
203 if (rcv_ev->ptype == RCVHQ_RCV_TYPE_EAGER)
204 ipath_hdrset_index((uint32_t *) write_rhf,
205 ips_recvq_tail_get(&writeq->egrq));
206 }
207
208 /* Ensure previous writes are visible before writing rhf seq or tail */
209 ips_wmb();
210
211 if (writeq->runtime_flags & IPATH_RUNTIME_NODMA_RTAIL) {
212 /* We accumulated a few changes to the RHF and now want to make it
213 * atomically visible for the reader.
214 */
215 uint32_t rhf_seq = writeq->state->hdrq_rhf_seq;
216 ipath_hdrset_seq((uint32_t *) write_rhf, rhf_seq);
217 if (rhf_seq >= LAST_RHF_SEQNO)
218 writeq->state->hdrq_rhf_seq = 1;
219 else
220 writeq->state->hdrq_rhf_seq = rhf_seq + 1;
221
222 /* Now write the new rhf */
223 ips_writehdrq_write_rhf_atomic(write_hdr + writeq->hdrq_rhf_off, write_rhf);
224 }
225
226 /* The tail must be updated regardless of IPATH_RUNTIME_NODMA_RTAIL
227 * since this tail is also used to keep track of where
228 * ips_writehdrq_append will write to next. For subcontexts there is
229 * no separate shadow copy of the tail. */
230 ips_recvq_tail_update(&writeq->hdrq, next_write_hdr_tail);
231
232 done:
233 return result;
234 }
235
236 #endif /* _IPS_WRITEHDRQ_H */