Debian Codegraph @

rts/opencl/transpose.cl

raw
1 // Start of transpose.cl
2
3 #define GEN_TRANSPOSE_KERNELS(NAME, ELEM_TYPE) \
4 FUTHARK_KERNEL_SIZED(TR_BLOCK_DIM*2, TR_TILE_DIM/TR_ELEMS_PER_THREAD, 1)\
5 void map_transpose_##NAME(SHARED_MEM_PARAM \
6 __global ELEM_TYPE *dst_mem, \
7 int64_t dst_offset, \
8 __global ELEM_TYPE *src_mem, \
9 int64_t src_offset, \
10 int32_t num_arrays, \
11 int32_t x_elems, \
12 int32_t y_elems, \
13 int32_t mulx, \
14 int32_t muly, \
15 int32_t repeat_1, \
16 int32_t repeat_2) { \
17 (void)mulx; (void)muly; \
18 __local ELEM_TYPE* block = (__local ELEM_TYPE*)shared_mem; \
19 int tblock_id_0 = get_tblock_id(0); \
20 int global_id_0 = get_global_id(0); \
21 int tblock_id_1 = get_tblock_id(1); \
22 int global_id_1 = get_global_id(1); \
23 for (int i1 = 0; i1 <= repeat_1; i1++) { \
24 int tblock_id_2 = get_tblock_id(2); \
25 int global_id_2 = get_global_id(2); \
26 for (int i2 = 0; i2 <= repeat_2; i2++) { \
27 if (tblock_id_2 >= num_arrays) { break; } \
28 int32_t our_array_offset = tblock_id_2 * x_elems * y_elems; \
29 int32_t odata_offset = dst_offset + our_array_offset; \
30 int32_t idata_offset = src_offset + our_array_offset; \
31 int32_t x_index = global_id_0; \
32 int32_t y_index = tblock_id_1 * TR_TILE_DIM + get_local_id(1); \
33 if (x_index < x_elems) { \
34 for (int32_t j = 0; j < TR_ELEMS_PER_THREAD; j++) { \
35 int32_t index_i = (y_index + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD)) * x_elems + x_index; \
36 if (y_index + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD) < y_elems) { \
37 block[(get_local_id(1) + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD)) * (TR_TILE_DIM+1) + \
38 get_local_id(0)] = \
39 src_mem[idata_offset + index_i]; \
40 } \
41 } \
42 } \
43 barrier_local(); \
44 x_index = tblock_id_1 * TR_TILE_DIM + get_local_id(0); \
45 y_index = tblock_id_0 * TR_TILE_DIM + get_local_id(1); \
46 if (x_index < y_elems) { \
47 for (int32_t j = 0; j < TR_ELEMS_PER_THREAD; j++) { \
48 int32_t index_out = (y_index + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD)) * y_elems + x_index; \
49 if (y_index + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD) < x_elems) { \
50 dst_mem[(odata_offset + index_out)] = \
51 block[get_local_id(0) * (TR_TILE_DIM+1) + \
52 get_local_id(1) + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD)]; \
53 } \
54 } \
55 } \
56 tblock_id_2 += get_num_tblocks(2); \
57 global_id_2 += get_global_size(2); \
58 } \
59 tblock_id_1 += get_num_tblocks(1); \
60 global_id_1 += get_global_size(1); \
61 } \
62 } \
63 \
64 FUTHARK_KERNEL_SIZED(TR_BLOCK_DIM, TR_BLOCK_DIM, 1) \
65 void map_transpose_##NAME##_low_height(SHARED_MEM_PARAM \
66 __global ELEM_TYPE *dst_mem, \
67 int64_t dst_offset, \
68 __global ELEM_TYPE *src_mem, \
69 int64_t src_offset, \
70 int32_t num_arrays, \
71 int32_t x_elems, \
72 int32_t y_elems, \
73 int32_t mulx, \
74 int32_t muly, \
75 int32_t repeat_1, \
76 int32_t repeat_2) { \
77 __local ELEM_TYPE* block = (__local ELEM_TYPE*)shared_mem; \
78 int tblock_id_0 = get_tblock_id(0); \
79 int global_id_0 = get_global_id(0); \
80 int tblock_id_1 = get_tblock_id(1); \
81 int global_id_1 = get_global_id(1); \
82 for (int i1 = 0; i1 <= repeat_1; i1++) { \
83 int tblock_id_2 = get_tblock_id(2); \
84 int global_id_2 = get_global_id(2); \
85 for (int i2 = 0; i2 <= repeat_2; i2++) { \
86 if (tblock_id_2 >= num_arrays) { break; } \
87 int32_t our_array_offset = tblock_id_2 * x_elems * y_elems; \
88 int32_t odata_offset = dst_offset + our_array_offset; \
89 int32_t idata_offset = src_offset + our_array_offset; \
90 int32_t x_index = \
91 tblock_id_0 * TR_BLOCK_DIM * mulx + \
92 get_local_id(0) + \
93 get_local_id(1)%mulx * TR_BLOCK_DIM; \
94 int32_t y_index = tblock_id_1 * TR_BLOCK_DIM + get_local_id(1)/mulx; \
95 int32_t index_in = y_index * x_elems + x_index; \
96 if (x_index < x_elems && y_index < y_elems) { \
97 block[get_local_id(1) * (TR_BLOCK_DIM+1) + get_local_id(0)] = \
98 src_mem[idata_offset + index_in]; \
99 } \
100 barrier_local(); \
101 x_index = tblock_id_1 * TR_BLOCK_DIM + get_local_id(0)/mulx; \
102 y_index = \
103 tblock_id_0 * TR_BLOCK_DIM * mulx + \
104 get_local_id(1) + \
105 (get_local_id(0)%mulx) * TR_BLOCK_DIM; \
106 int32_t index_out = y_index * y_elems + x_index; \
107 if (x_index < y_elems && y_index < x_elems) { \
108 dst_mem[odata_offset + index_out] = \
109 block[get_local_id(0) * (TR_BLOCK_DIM+1) + get_local_id(1)]; \
110 } \
111 tblock_id_2 += get_num_tblocks(2); \
112 global_id_2 += get_global_size(2); \
113 } \
114 tblock_id_1 += get_num_tblocks(1); \
115 global_id_1 += get_global_size(1); \
116 } \
117 } \
118 \
119 FUTHARK_KERNEL_SIZED(TR_BLOCK_DIM, TR_BLOCK_DIM, 1) \
120 void map_transpose_##NAME##_low_width(SHARED_MEM_PARAM \
121 __global ELEM_TYPE *dst_mem, \
122 int64_t dst_offset, \
123 __global ELEM_TYPE *src_mem, \
124 int64_t src_offset, \
125 int32_t num_arrays, \
126 int32_t x_elems, \
127 int32_t y_elems, \
128 int32_t mulx, \
129 int32_t muly, \
130 int32_t repeat_1, \
131 int32_t repeat_2) { \
132 __local ELEM_TYPE* block = (__local ELEM_TYPE*)shared_mem; \
133 int tblock_id_0 = get_tblock_id(0); \
134 int global_id_0 = get_global_id(0); \
135 int tblock_id_1 = get_tblock_id(1); \
136 int global_id_1 = get_global_id(1); \
137 for (int i1 = 0; i1 <= repeat_1; i1++) { \
138 int tblock_id_2 = get_tblock_id(2); \
139 int global_id_2 = get_global_id(2); \
140 for (int i2 = 0; i2 <= repeat_2; i2++) { \
141 if (tblock_id_2 >= num_arrays) { break; } \
142 int32_t our_array_offset = tblock_id_2 * x_elems * y_elems; \
143 int32_t odata_offset = dst_offset + our_array_offset; \
144 int32_t idata_offset = src_offset + our_array_offset; \
145 int32_t x_index = tblock_id_0 * TR_BLOCK_DIM + get_local_id(0)/muly; \
146 int32_t y_index = \
147 tblock_id_1 * TR_BLOCK_DIM * muly + \
148 get_local_id(1) + (get_local_id(0)%muly) * TR_BLOCK_DIM; \
149 int32_t index_in = y_index * x_elems + x_index; \
150 if (x_index < x_elems && y_index < y_elems) { \
151 block[get_local_id(1) * (TR_BLOCK_DIM+1) + get_local_id(0)] = \
152 src_mem[idata_offset + index_in]; \
153 } \
154 barrier_local(); \
155 x_index = tblock_id_1 * TR_BLOCK_DIM * muly + \
156 get_local_id(0) + (get_local_id(1)%muly) * TR_BLOCK_DIM; \
157 y_index = tblock_id_0 * TR_BLOCK_DIM + get_local_id(1)/muly; \
158 int32_t index_out = y_index * y_elems + x_index; \
159 if (x_index < y_elems && y_index < x_elems) { \
160 dst_mem[odata_offset + index_out] = \
161 block[get_local_id(0) * (TR_BLOCK_DIM+1) + get_local_id(1)]; \
162 } \
163 tblock_id_2 += get_num_tblocks(2); \
164 global_id_2 += get_num_tblocks(2) * get_local_size(2); \
165 } \
166 tblock_id_1 += get_num_tblocks(1); \
167 global_id_1 += get_num_tblocks(1) * get_local_size(1); \
168 } \
169 } \
170 \
171 FUTHARK_KERNEL_SIZED(TR_BLOCK_DIM*TR_BLOCK_DIM, 1, 1) \
172 void map_transpose_##NAME##_small(SHARED_MEM_PARAM \
173 __global ELEM_TYPE *dst_mem, \
174 int64_t dst_offset, \
175 __global ELEM_TYPE *src_mem, \
176 int64_t src_offset, \
177 int32_t num_arrays, \
178 int32_t x_elems, \
179 int32_t y_elems, \
180 int32_t mulx, \
181 int32_t muly, \
182 int32_t repeat_1, \
183 int32_t repeat_2) { \
184 (void)mulx; (void)muly; \
185 __local ELEM_TYPE* block = (__local ELEM_TYPE*)shared_mem; \
186 int tblock_id_0 = get_tblock_id(0); \
187 int global_id_0 = get_global_id(0); \
188 int tblock_id_1 = get_tblock_id(1); \
189 int global_id_1 = get_global_id(1); \
190 for (int i1 = 0; i1 <= repeat_1; i1++) { \
191 int tblock_id_2 = get_tblock_id(2); \
192 int global_id_2 = get_global_id(2); \
193 for (int i2 = 0; i2 <= repeat_2; i2++) { \
194 int32_t our_array_offset = global_id_0/(y_elems * x_elems) * y_elems * x_elems; \
195 int32_t x_index = (global_id_0 % (y_elems * x_elems))/y_elems; \
196 int32_t y_index = global_id_0%y_elems; \
197 int32_t odata_offset = dst_offset + our_array_offset; \
198 int32_t idata_offset = src_offset + our_array_offset; \
199 int32_t index_in = y_index * x_elems + x_index; \
200 int32_t index_out = x_index * y_elems + y_index; \
201 if (global_id_0 < x_elems * y_elems * num_arrays) { \
202 dst_mem[odata_offset + index_out] = src_mem[idata_offset + index_in]; \
203 } \
204 tblock_id_2 += get_num_tblocks(2); \
205 global_id_2 += get_global_size(2); \
206 } \
207 tblock_id_1 += get_num_tblocks(1); \
208 global_id_1 += get_global_size(1); \
209 } \
210 } \
211 \
212 FUTHARK_KERNEL_SIZED(TR_BLOCK_DIM*2, TR_TILE_DIM/TR_ELEMS_PER_THREAD, 1)\
213 void map_transpose_##NAME##_large(SHARED_MEM_PARAM \
214 __global ELEM_TYPE *dst_mem, \
215 int64_t dst_offset, \
216 __global ELEM_TYPE *src_mem, \
217 int64_t src_offset, \
218 int64_t num_arrays, \
219 int64_t x_elems, \
220 int64_t y_elems, \
221 int64_t mulx, \
222 int64_t muly, \
223 int32_t repeat_1, \
224 int32_t repeat_2) { \
225 (void)mulx; (void)muly; \
226 __local ELEM_TYPE* block = (__local ELEM_TYPE*)shared_mem; \
227 int tblock_id_0 = get_tblock_id(0); \
228 int global_id_0 = get_global_id(0); \
229 int tblock_id_1 = get_tblock_id(1); \
230 int global_id_1 = get_global_id(1); \
231 for (int i1 = 0; i1 <= repeat_1; i1++) { \
232 int tblock_id_2 = get_tblock_id(2); \
233 int global_id_2 = get_global_id(2); \
234 for (int i2 = 0; i2 <= repeat_2; i2++) { \
235 if (tblock_id_2 >= num_arrays) { break; } \
236 int64_t our_array_offset = tblock_id_2 * x_elems * y_elems; \
237 int64_t odata_offset = dst_offset + our_array_offset; \
238 int64_t idata_offset = src_offset + our_array_offset; \
239 int64_t x_index = global_id_0; \
240 int64_t y_index = tblock_id_1 * TR_TILE_DIM + get_local_id(1); \
241 if (x_index < x_elems) { \
242 for (int64_t j = 0; j < TR_ELEMS_PER_THREAD; j++) { \
243 int64_t index_i = (y_index + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD)) * x_elems + x_index; \
244 if (y_index + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD) < y_elems) { \
245 block[(get_local_id(1) + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD)) * (TR_TILE_DIM+1) + \
246 get_local_id(0)] = \
247 src_mem[idata_offset + index_i]; \
248 } \
249 } \
250 } \
251 barrier_local(); \
252 x_index = tblock_id_1 * TR_TILE_DIM + get_local_id(0); \
253 y_index = tblock_id_0 * TR_TILE_DIM + get_local_id(1); \
254 if (x_index < y_elems) { \
255 for (int64_t j = 0; j < TR_ELEMS_PER_THREAD; j++) { \
256 int64_t index_out = (y_index + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD)) * y_elems + x_index; \
257 if (y_index + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD) < x_elems) { \
258 dst_mem[(odata_offset + index_out)] = \
259 block[get_local_id(0) * (TR_TILE_DIM+1) + \
260 get_local_id(1) + j * (TR_TILE_DIM/TR_ELEMS_PER_THREAD)]; \
261 } \
262 } \
263 } \
264 tblock_id_2 += get_num_tblocks(2); \
265 global_id_2 += get_global_size(2); \
266 } \
267 tblock_id_1 += get_num_tblocks(1); \
268 global_id_1 += get_global_size(1); \
269 } \
270 } \
271
272 GEN_TRANSPOSE_KERNELS(1b, uint8_t)
273 GEN_TRANSPOSE_KERNELS(2b, uint16_t)
274 GEN_TRANSPOSE_KERNELS(4b, uint32_t)
275 GEN_TRANSPOSE_KERNELS(8b, uint64_t)
276
277 // End of transpose.cl