Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_InnerGemmFixA_Serial_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_INNER_GEMM_FIX_A_SERIAL_IMPL_HPP
17 #define KOKKOSBATCHED_INNER_GEMM_FIX_A_SERIAL_IMPL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosBatched_InnerGemmFixA_Decl.hpp"
23
24 namespace KokkosBatched {
25
26 ///
27 /// Inner kernel (5x5)
28 /// ==================
29
30 template <>
31 template <typename ScalarType, typename ValueType>
32 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<5, 5>::serial_invoke(const ScalarType alpha,
33 const ValueType *KOKKOS_RESTRICT A,
34 const ValueType *KOKKOS_RESTRICT B, const int n,
35 /**/ ValueType *KOKKOS_RESTRICT C) {
36 if (n <= 0) return 0;
37
38 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
39 a_03 = A[0 * _as0 + 3 * _as1], a_04 = A[0 * _as0 + 4 * _as1], a_10 = A[1 * _as0 + 0 * _as1],
40 a_11 = A[1 * _as0 + 1 * _as1], a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1],
41 a_14 = A[1 * _as0 + 4 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1],
42 a_22 = A[2 * _as0 + 2 * _as1], a_23 = A[2 * _as0 + 3 * _as1], a_24 = A[2 * _as0 + 4 * _as1],
43 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1],
44 a_33 = A[3 * _as0 + 3 * _as1], a_34 = A[3 * _as0 + 4 * _as1], a_40 = A[4 * _as0 + 0 * _as1],
45 a_41 = A[4 * _as0 + 1 * _as1], a_42 = A[4 * _as0 + 2 * _as1], a_43 = A[4 * _as0 + 3 * _as1],
46 a_44 = A[4 * _as0 + 4 * _as1];
47
48 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, c_2p, b_3p, c_3p, b_4p, c_4p;
49
50 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ib4 = 4 * _bs0, ic0 = 0 * _cs0,
51 ic1 = 1 * _cs0, ic2 = 2 * _cs0, ic3 = 3 * _cs0, ic4 = 4 * _cs0;
52
53 for (int p = 0; p < n; ++p) {
54 b_0p = B[ib0 + p * _bs1];
55 b_1p = B[ib1 + p * _bs1];
56 b_2p = B[ib2 + p * _bs1];
57 b_3p = B[ib3 + p * _bs1];
58 b_4p = B[ib4 + p * _bs1];
59
60 c_0p = a_00 * b_0p;
61 c_0p += a_01 * b_1p;
62 c_0p += a_02 * b_2p;
63 c_0p += a_03 * b_3p;
64 c_0p += a_04 * b_4p;
65 c_1p = a_10 * b_0p;
66 c_1p += a_11 * b_1p;
67 c_1p += a_12 * b_2p;
68 c_1p += a_13 * b_3p;
69 c_1p += a_14 * b_4p;
70 c_2p = a_20 * b_0p;
71 c_2p += a_21 * b_1p;
72 c_2p += a_22 * b_2p;
73 c_2p += a_23 * b_3p;
74 c_2p += a_24 * b_4p;
75 c_3p = a_30 * b_0p;
76 c_3p += a_31 * b_1p;
77 c_3p += a_32 * b_2p;
78 c_3p += a_33 * b_3p;
79 c_3p += a_34 * b_4p;
80 c_4p = a_40 * b_0p;
81 c_4p += a_41 * b_1p;
82 c_4p += a_42 * b_2p;
83 c_4p += a_43 * b_3p;
84 c_4p += a_44 * b_4p;
85
86 C[ic0 + p * _cs1] += alpha * c_0p;
87 C[ic1 + p * _cs1] += alpha * c_1p;
88 C[ic2 + p * _cs1] += alpha * c_2p;
89 C[ic3 + p * _cs1] += alpha * c_3p;
90 C[ic4 + p * _cs1] += alpha * c_4p;
91 }
92
93 return 0;
94 }
95
96 template <>
97 template <typename ScalarType, typename ValueType>
98 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<5, 4>::serial_invoke(const ScalarType alpha,
99 const ValueType *KOKKOS_RESTRICT A,
100 const ValueType *KOKKOS_RESTRICT B, const int n,
101 /**/ ValueType *KOKKOS_RESTRICT C) {
102 if (n <= 0) return 0;
103
104 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
105 a_03 = A[0 * _as0 + 3 * _as1], a_10 = A[1 * _as0 + 0 * _as1], a_11 = A[1 * _as0 + 1 * _as1],
106 a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1], a_20 = A[2 * _as0 + 0 * _as1],
107 a_21 = A[2 * _as0 + 1 * _as1], a_22 = A[2 * _as0 + 2 * _as1], a_23 = A[2 * _as0 + 3 * _as1],
108 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1],
109 a_33 = A[3 * _as0 + 3 * _as1], a_40 = A[4 * _as0 + 0 * _as1], a_41 = A[4 * _as0 + 1 * _as1],
110 a_42 = A[4 * _as0 + 2 * _as1], a_43 = A[4 * _as0 + 3 * _as1];
111
112 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, c_2p, b_3p, c_3p,
113 /**/ c_4p;
114
115 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0,
116 ic2 = 2 * _cs0, ic3 = 3 * _cs0, ic4 = 4 * _cs0;
117
118 for (int p = 0; p < n; ++p) {
119 b_0p = B[ib0 + p * _bs1];
120 b_1p = B[ib1 + p * _bs1];
121 b_2p = B[ib2 + p * _bs1];
122 b_3p = B[ib3 + p * _bs1];
123
124 c_0p = a_00 * b_0p;
125 c_0p += a_01 * b_1p;
126 c_0p += a_02 * b_2p;
127 c_0p += a_03 * b_3p;
128 c_1p = a_10 * b_0p;
129 c_1p += a_11 * b_1p;
130 c_1p += a_12 * b_2p;
131 c_1p += a_13 * b_3p;
132 c_2p = a_20 * b_0p;
133 c_2p += a_21 * b_1p;
134 c_2p += a_22 * b_2p;
135 c_2p += a_23 * b_3p;
136 c_3p = a_30 * b_0p;
137 c_3p += a_31 * b_1p;
138 c_3p += a_32 * b_2p;
139 c_3p += a_33 * b_3p;
140 c_4p = a_40 * b_0p;
141 c_4p += a_41 * b_1p;
142 c_4p += a_42 * b_2p;
143 c_4p += a_43 * b_3p;
144
145 C[ic0 + p * _cs1] += alpha * c_0p;
146 C[ic1 + p * _cs1] += alpha * c_1p;
147 C[ic2 + p * _cs1] += alpha * c_2p;
148 C[ic3 + p * _cs1] += alpha * c_3p;
149 C[ic4 + p * _cs1] += alpha * c_4p;
150 }
151
152 return 0;
153 }
154
155 template <>
156 template <typename ScalarType, typename ValueType>
157 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<5, 3>::serial_invoke(const ScalarType alpha,
158 const ValueType *KOKKOS_RESTRICT A,
159 const ValueType *KOKKOS_RESTRICT B, const int n,
160 /**/ ValueType *KOKKOS_RESTRICT C) {
161 if (n <= 0) return 0;
162
163 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
164 a_10 = A[1 * _as0 + 0 * _as1], a_11 = A[1 * _as0 + 1 * _as1], a_12 = A[1 * _as0 + 2 * _as1],
165 a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1], a_22 = A[2 * _as0 + 2 * _as1],
166 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1],
167 a_40 = A[4 * _as0 + 0 * _as1], a_41 = A[4 * _as0 + 1 * _as1], a_42 = A[4 * _as0 + 2 * _as1];
168
169 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, c_2p,
170 /**/ c_3p,
171 /**/ c_4p;
172
173 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0, ic2 = 2 * _cs0,
174 ic3 = 3 * _cs0, ic4 = 4 * _cs0;
175
176 for (int p = 0; p < n; ++p) {
177 b_0p = B[ib0 + p * _bs1];
178 b_1p = B[ib1 + p * _bs1];
179 b_2p = B[ib2 + p * _bs1];
180
181 c_0p = a_00 * b_0p;
182 c_0p += a_01 * b_1p;
183 c_0p += a_02 * b_2p;
184 c_1p = a_10 * b_0p;
185 c_1p += a_11 * b_1p;
186 c_1p += a_12 * b_2p;
187 c_2p = a_20 * b_0p;
188 c_2p += a_21 * b_1p;
189 c_2p += a_22 * b_2p;
190 c_3p = a_30 * b_0p;
191 c_3p += a_31 * b_1p;
192 c_3p += a_32 * b_2p;
193 c_4p = a_40 * b_0p;
194 c_4p += a_41 * b_1p;
195 c_4p += a_42 * b_2p;
196
197 C[ic0 + p * _cs1] += alpha * c_0p;
198 C[ic1 + p * _cs1] += alpha * c_1p;
199 C[ic2 + p * _cs1] += alpha * c_2p;
200 C[ic3 + p * _cs1] += alpha * c_3p;
201 C[ic4 + p * _cs1] += alpha * c_4p;
202 }
203
204 return 0;
205 }
206 template <>
207 template <typename ScalarType, typename ValueType>
208 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<5, 2>::serial_invoke(const ScalarType alpha,
209 const ValueType *KOKKOS_RESTRICT A,
210 const ValueType *KOKKOS_RESTRICT B, const int n,
211 /**/ ValueType *KOKKOS_RESTRICT C) {
212 if (n <= 0) return 0;
213
214 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_10 = A[1 * _as0 + 0 * _as1],
215 a_11 = A[1 * _as0 + 1 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1],
216 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_40 = A[4 * _as0 + 0 * _as1],
217 a_41 = A[4 * _as0 + 1 * _as1];
218
219 ValueType b_0p, c_0p, b_1p, c_1p,
220 /**/ c_2p,
221 /**/ c_3p,
222 /**/ c_4p;
223
224 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0, ic2 = 2 * _cs0, ic3 = 3 * _cs0,
225 ic4 = 4 * _cs0;
226
227 for (int p = 0; p < n; ++p) {
228 b_0p = B[ib0 + p * _bs1];
229 b_1p = B[ib1 + p * _bs1];
230
231 c_0p = a_00 * b_0p;
232 c_0p += a_01 * b_1p;
233 c_1p = a_10 * b_0p;
234 c_1p += a_11 * b_1p;
235 c_2p = a_20 * b_0p;
236 c_2p += a_21 * b_1p;
237 c_3p = a_30 * b_0p;
238 c_3p += a_31 * b_1p;
239 c_4p = a_40 * b_0p;
240 c_4p += a_41 * b_1p;
241
242 C[ic0 + p * _cs1] += alpha * c_0p;
243 C[ic1 + p * _cs1] += alpha * c_1p;
244 C[ic2 + p * _cs1] += alpha * c_2p;
245 C[ic3 + p * _cs1] += alpha * c_3p;
246 C[ic4 + p * _cs1] += alpha * c_4p;
247 }
248
249 return 0;
250 }
251 template <>
252 template <typename ScalarType, typename ValueType>
253 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<5, 1>::serial_invoke(const ScalarType alpha,
254 const ValueType *KOKKOS_RESTRICT A,
255 const ValueType *KOKKOS_RESTRICT B, const int n,
256 /**/ ValueType *KOKKOS_RESTRICT C) {
257 if (n <= 0) return 0;
258
259 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_10 = A[1 * _as0 + 0 * _as1], a_20 = A[2 * _as0 + 0 * _as1],
260 a_30 = A[3 * _as0 + 0 * _as1], a_40 = A[4 * _as0 + 0 * _as1];
261
262 ValueType b_0p, c_0p,
263 /**/ c_1p,
264 /**/ c_2p,
265 /**/ c_3p,
266 /**/ c_4p;
267
268 const int ib0 = 0 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0, ic2 = 2 * _cs0, ic3 = 3 * _cs0, ic4 = 4 * _cs0;
269
270 for (int p = 0; p < n; ++p) {
271 b_0p = B[ib0 + p * _bs1];
272
273 c_0p = a_00 * b_0p;
274 c_1p = a_10 * b_0p;
275 c_2p = a_20 * b_0p;
276 c_3p = a_30 * b_0p;
277 c_4p = a_40 * b_0p;
278
279 C[ic0 + p * _cs1] += alpha * c_0p;
280 C[ic1 + p * _cs1] += alpha * c_1p;
281 C[ic2 + p * _cs1] += alpha * c_2p;
282 C[ic3 + p * _cs1] += alpha * c_3p;
283 C[ic4 + p * _cs1] += alpha * c_4p;
284 }
285
286 return 0;
287 }
288
289 template <>
290 template <typename ScalarType, typename ValueType>
291 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<4, 5>::serial_invoke(const ScalarType alpha,
292 const ValueType *KOKKOS_RESTRICT A,
293 const ValueType *KOKKOS_RESTRICT B, const int n,
294 /**/ ValueType *KOKKOS_RESTRICT C) {
295 if (n <= 0) return 0;
296
297 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
298 a_03 = A[0 * _as0 + 3 * _as1], a_04 = A[0 * _as0 + 4 * _as1], a_10 = A[1 * _as0 + 0 * _as1],
299 a_11 = A[1 * _as0 + 1 * _as1], a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1],
300 a_14 = A[1 * _as0 + 4 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1],
301 a_22 = A[2 * _as0 + 2 * _as1], a_23 = A[2 * _as0 + 3 * _as1], a_24 = A[2 * _as0 + 4 * _as1],
302 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1],
303 a_33 = A[3 * _as0 + 3 * _as1], a_34 = A[3 * _as0 + 4 * _as1];
304
305 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, c_2p, b_3p, c_3p, b_4p;
306
307 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ib4 = 4 * _bs0, ic0 = 0 * _cs0,
308 ic1 = 1 * _cs0, ic2 = 2 * _cs0, ic3 = 3 * _cs0;
309
310 for (int p = 0; p < n; ++p) {
311 b_0p = B[ib0 + p * _bs1];
312 b_1p = B[ib1 + p * _bs1];
313 b_2p = B[ib2 + p * _bs1];
314 b_3p = B[ib3 + p * _bs1];
315 b_4p = B[ib4 + p * _bs1];
316
317 c_0p = a_00 * b_0p;
318 c_0p += a_01 * b_1p;
319 c_0p += a_02 * b_2p;
320 c_0p += a_03 * b_3p;
321 c_0p += a_04 * b_4p;
322 c_1p = a_10 * b_0p;
323 c_1p += a_11 * b_1p;
324 c_1p += a_12 * b_2p;
325 c_1p += a_13 * b_3p;
326 c_1p += a_14 * b_4p;
327 c_2p = a_20 * b_0p;
328 c_2p += a_21 * b_1p;
329 c_2p += a_22 * b_2p;
330 c_2p += a_23 * b_3p;
331 c_2p += a_24 * b_4p;
332 c_3p = a_30 * b_0p;
333 c_3p += a_31 * b_1p;
334 c_3p += a_32 * b_2p;
335 c_3p += a_33 * b_3p;
336 c_3p += a_34 * b_4p;
337
338 C[ic0 + p * _cs1] += alpha * c_0p;
339 C[ic1 + p * _cs1] += alpha * c_1p;
340 C[ic2 + p * _cs1] += alpha * c_2p;
341 C[ic3 + p * _cs1] += alpha * c_3p;
342 }
343
344 return 0;
345 }
346
347 template <>
348 template <typename ScalarType, typename ValueType>
349 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<3, 5>::serial_invoke(const ScalarType alpha,
350 const ValueType *KOKKOS_RESTRICT A,
351 const ValueType *KOKKOS_RESTRICT B, const int n,
352 /**/ ValueType *KOKKOS_RESTRICT C) {
353 if (n <= 0) return 0;
354
355 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
356 a_03 = A[0 * _as0 + 3 * _as1], a_04 = A[0 * _as0 + 4 * _as1], a_10 = A[1 * _as0 + 0 * _as1],
357 a_11 = A[1 * _as0 + 1 * _as1], a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1],
358 a_14 = A[1 * _as0 + 4 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1],
359 a_22 = A[2 * _as0 + 2 * _as1], a_23 = A[2 * _as0 + 3 * _as1], a_24 = A[2 * _as0 + 4 * _as1];
360
361 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, c_2p, b_3p, b_4p;
362
363 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ib4 = 4 * _bs0, ic0 = 0 * _cs0,
364 ic1 = 1 * _cs0, ic2 = 2 * _cs0;
365
366 for (int p = 0; p < n; ++p) {
367 b_0p = B[ib0 + p * _bs1];
368 b_1p = B[ib1 + p * _bs1];
369 b_2p = B[ib2 + p * _bs1];
370 b_3p = B[ib3 + p * _bs1];
371 b_4p = B[ib4 + p * _bs1];
372
373 c_0p = a_00 * b_0p;
374 c_0p += a_01 * b_1p;
375 c_0p += a_02 * b_2p;
376 c_0p += a_03 * b_3p;
377 c_0p += a_04 * b_4p;
378 c_1p = a_10 * b_0p;
379 c_1p += a_11 * b_1p;
380 c_1p += a_12 * b_2p;
381 c_1p += a_13 * b_3p;
382 c_1p += a_14 * b_4p;
383 c_2p = a_20 * b_0p;
384 c_2p += a_21 * b_1p;
385 c_2p += a_22 * b_2p;
386 c_2p += a_23 * b_3p;
387 c_2p += a_24 * b_4p;
388
389 C[ic0 + p * _cs1] += alpha * c_0p;
390 C[ic1 + p * _cs1] += alpha * c_1p;
391 C[ic2 + p * _cs1] += alpha * c_2p;
392 }
393
394 return 0;
395 }
396 template <>
397 template <typename ScalarType, typename ValueType>
398 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<2, 5>::serial_invoke(const ScalarType alpha,
399 const ValueType *KOKKOS_RESTRICT A,
400 const ValueType *KOKKOS_RESTRICT B, const int n,
401 /**/ ValueType *KOKKOS_RESTRICT C) {
402 if (n <= 0) return 0;
403
404 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
405 a_03 = A[0 * _as0 + 3 * _as1], a_04 = A[0 * _as0 + 4 * _as1], a_10 = A[1 * _as0 + 0 * _as1],
406 a_11 = A[1 * _as0 + 1 * _as1], a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1],
407 a_14 = A[1 * _as0 + 4 * _as1];
408
409 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, b_3p, b_4p;
410
411 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ib4 = 4 * _bs0, ic0 = 0 * _cs0,
412 ic1 = 1 * _cs0;
413
414 for (int p = 0; p < n; ++p) {
415 b_0p = B[ib0 + p * _bs1];
416 b_1p = B[ib1 + p * _bs1];
417 b_2p = B[ib2 + p * _bs1];
418 b_3p = B[ib3 + p * _bs1];
419 b_4p = B[ib4 + p * _bs1];
420
421 c_0p = a_00 * b_0p;
422 c_0p += a_01 * b_1p;
423 c_0p += a_02 * b_2p;
424 c_0p += a_03 * b_3p;
425 c_0p += a_04 * b_4p;
426 c_1p = a_10 * b_0p;
427 c_1p += a_11 * b_1p;
428 c_1p += a_12 * b_2p;
429 c_1p += a_13 * b_3p;
430 c_1p += a_14 * b_4p;
431
432 C[ic0 + p * _cs1] += alpha * c_0p;
433 C[ic1 + p * _cs1] += alpha * c_1p;
434 }
435
436 return 0;
437 }
438 template <>
439 template <typename ScalarType, typename ValueType>
440 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<1, 5>::serial_invoke(const ScalarType alpha,
441 const ValueType *KOKKOS_RESTRICT A,
442 const ValueType *KOKKOS_RESTRICT B, const int n,
443 /**/ ValueType *KOKKOS_RESTRICT C) {
444 if (n <= 0) return 0;
445
446 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
447 a_03 = A[0 * _as0 + 3 * _as1], a_04 = A[0 * _as0 + 4 * _as1];
448
449 ValueType b_0p, c_0p, b_1p, b_2p, b_3p, b_4p;
450
451 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ib4 = 4 * _bs0, ic0 = 0 * _cs0;
452
453 for (int p = 0; p < n; ++p) {
454 b_0p = B[ib0 + p * _bs1];
455 b_1p = B[ib1 + p * _bs1];
456 b_2p = B[ib2 + p * _bs1];
457 b_3p = B[ib3 + p * _bs1];
458 b_4p = B[ib4 + p * _bs1];
459
460 c_0p = a_00 * b_0p;
461 c_0p += a_01 * b_1p;
462 c_0p += a_02 * b_2p;
463 c_0p += a_03 * b_3p;
464 c_0p += a_04 * b_4p;
465
466 C[ic0 + p * _cs1] += alpha * c_0p;
467 }
468
469 return 0;
470 }
471
472 template <>
473 template <typename ScalarType, typename ValueType>
474 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<5, 5>::serial_invoke(const ScalarType alpha,
475 const ValueType *KOKKOS_RESTRICT A,
476 const ValueType *KOKKOS_RESTRICT B, const int m,
477 const int n, const int k,
478 /**/ ValueType *KOKKOS_RESTRICT C) {
479 if (m <= 0 || n <= 0 || k <= 0) return 0;
480
481 switch (m * 10 + k) {
482 case 54: {
483 InnerGemmFixA<5, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
484 inner.serial_invoke(alpha, A, B, n, C);
485 break;
486 }
487 case 53: {
488 InnerGemmFixA<5, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
489 inner.serial_invoke(alpha, A, B, n, C);
490 break;
491 }
492 case 52: {
493 InnerGemmFixA<5, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
494 inner.serial_invoke(alpha, A, B, n, C);
495 break;
496 }
497 case 51: {
498 InnerGemmFixA<5, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
499 inner.serial_invoke(alpha, A, B, n, C);
500 break;
501 }
502 case 45: {
503 InnerGemmFixA<4, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
504 inner.serial_invoke(alpha, A, B, n, C);
505 break;
506 }
507 case 35: {
508 InnerGemmFixA<3, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
509 inner.serial_invoke(alpha, A, B, n, C);
510 break;
511 }
512 case 25: {
513 InnerGemmFixA<2, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
514 inner.serial_invoke(alpha, A, B, n, C);
515 break;
516 }
517 case 15: {
518 InnerGemmFixA<1, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
519 inner.serial_invoke(alpha, A, B, n, C);
520 break;
521 }
522 default: {
523 if (m < 5 && n < 5) {
524 InnerGemmFixA<2, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
525 for (int i = 0; i < m; i += 2)
526 for (int p = 0; p < k; p += 2)
527 inner.serial_invoke(alpha, A + i * _as0 + p * _as1, B + p * _bs0, (i + 2 > m ? 1 : 2), n,
528 (p + 2 > k ? 1 : 2), C + i * _cs0);
529 } else {
530 Kokkos::abort("InnerGemmFixA<5,5>::serial_invoke, assert failure (m<5 && n<5)");
531 }
532 break;
533 }
534 }
535
536 return 0;
537 }
538
539 ///
540 /// Inner kernel (4x4)
541 /// ==================
542
543 template <>
544 template <typename ScalarType, typename ValueType>
545 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<4, 4>::serial_invoke(const ScalarType alpha,
546 const ValueType *KOKKOS_RESTRICT A,
547 const ValueType *KOKKOS_RESTRICT B, const int n,
548 /**/ ValueType *KOKKOS_RESTRICT C) {
549 if (n <= 0) return 0;
550
551 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
552 a_03 = A[0 * _as0 + 3 * _as1], a_10 = A[1 * _as0 + 0 * _as1], a_11 = A[1 * _as0 + 1 * _as1],
553 a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1], a_20 = A[2 * _as0 + 0 * _as1],
554 a_21 = A[2 * _as0 + 1 * _as1], a_22 = A[2 * _as0 + 2 * _as1], a_23 = A[2 * _as0 + 3 * _as1],
555 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1],
556 a_33 = A[3 * _as0 + 3 * _as1];
557
558 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, c_2p, b_3p, c_3p;
559
560 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0,
561 ic2 = 2 * _cs0, ic3 = 3 * _cs0;
562
563 for (int p = 0; p < n; ++p) {
564 b_0p = B[ib0 + p * _bs1];
565 b_1p = B[ib1 + p * _bs1];
566 b_2p = B[ib2 + p * _bs1];
567 b_3p = B[ib3 + p * _bs1];
568
569 c_0p = a_00 * b_0p;
570 c_0p += a_01 * b_1p;
571 c_0p += a_02 * b_2p;
572 c_0p += a_03 * b_3p;
573 c_1p = a_10 * b_0p;
574 c_1p += a_11 * b_1p;
575 c_1p += a_12 * b_2p;
576 c_1p += a_13 * b_3p;
577 c_2p = a_20 * b_0p;
578 c_2p += a_21 * b_1p;
579 c_2p += a_22 * b_2p;
580 c_2p += a_23 * b_3p;
581 c_3p = a_30 * b_0p;
582 c_3p += a_31 * b_1p;
583 c_3p += a_32 * b_2p;
584 c_3p += a_33 * b_3p;
585
586 C[ic0 + p * _cs1] += alpha * c_0p;
587 C[ic1 + p * _cs1] += alpha * c_1p;
588 C[ic2 + p * _cs1] += alpha * c_2p;
589 C[ic3 + p * _cs1] += alpha * c_3p;
590 }
591
592 return 0;
593 }
594
595 template <>
596 template <typename ScalarType, typename ValueType>
597 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<4, 3>::serial_invoke(const ScalarType alpha,
598 const ValueType *KOKKOS_RESTRICT A,
599 const ValueType *KOKKOS_RESTRICT B, const int n,
600 /**/ ValueType *KOKKOS_RESTRICT C) {
601 if (n <= 0) return 0;
602
603 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
604 a_10 = A[1 * _as0 + 0 * _as1], a_11 = A[1 * _as0 + 1 * _as1], a_12 = A[1 * _as0 + 2 * _as1],
605 a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1], a_22 = A[2 * _as0 + 2 * _as1],
606 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1];
607
608 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, c_2p,
609 /**/ c_3p;
610
611 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0, ic2 = 2 * _cs0,
612 ic3 = 3 * _cs0;
613
614 for (int p = 0; p < n; ++p) {
615 b_0p = B[ib0 + p * _bs1];
616 b_1p = B[ib1 + p * _bs1];
617 b_2p = B[ib2 + p * _bs1];
618
619 c_0p = a_00 * b_0p;
620 c_0p += a_01 * b_1p;
621 c_0p += a_02 * b_2p;
622 c_1p = a_10 * b_0p;
623 c_1p += a_11 * b_1p;
624 c_1p += a_12 * b_2p;
625 c_2p = a_20 * b_0p;
626 c_2p += a_21 * b_1p;
627 c_2p += a_22 * b_2p;
628 c_3p = a_30 * b_0p;
629 c_3p += a_31 * b_1p;
630 c_3p += a_32 * b_2p;
631
632 C[ic0 + p * _cs1] += alpha * c_0p;
633 C[ic1 + p * _cs1] += alpha * c_1p;
634 C[ic2 + p * _cs1] += alpha * c_2p;
635 C[ic3 + p * _cs1] += alpha * c_3p;
636 }
637
638 return 0;
639 }
640
641 template <>
642 template <typename ScalarType, typename ValueType>
643 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<4, 2>::serial_invoke(const ScalarType alpha,
644 const ValueType *KOKKOS_RESTRICT A,
645 const ValueType *KOKKOS_RESTRICT B, const int n,
646 /**/ ValueType *KOKKOS_RESTRICT C) {
647 if (n <= 0) return 0;
648
649 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_10 = A[1 * _as0 + 0 * _as1],
650 a_11 = A[1 * _as0 + 1 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1],
651 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1];
652
653 ValueType b_0p, c_0p, b_1p, c_1p,
654 /**/ c_2p,
655 /**/ c_3p;
656
657 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0, ic2 = 2 * _cs0, ic3 = 3 * _cs0;
658
659 for (int p = 0; p < n; ++p) {
660 b_0p = B[ib0 + p * _bs1];
661 b_1p = B[ib1 + p * _bs1];
662
663 c_0p = a_00 * b_0p;
664 c_0p += a_01 * b_1p;
665 c_1p = a_10 * b_0p;
666 c_1p += a_11 * b_1p;
667 c_2p = a_20 * b_0p;
668 c_2p += a_21 * b_1p;
669 c_3p = a_30 * b_0p;
670 c_3p += a_31 * b_1p;
671
672 C[ic0 + p * _cs1] += alpha * c_0p;
673 C[ic1 + p * _cs1] += alpha * c_1p;
674 C[ic2 + p * _cs1] += alpha * c_2p;
675 C[ic3 + p * _cs1] += alpha * c_3p;
676 }
677
678 return 0;
679 }
680
681 template <>
682 template <typename ScalarType, typename ValueType>
683 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<4, 1>::serial_invoke(const ScalarType alpha,
684 const ValueType *KOKKOS_RESTRICT A,
685 const ValueType *KOKKOS_RESTRICT B, const int n,
686 /**/ ValueType *KOKKOS_RESTRICT C) {
687 if (n <= 0) return 0;
688
689 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_10 = A[1 * _as0 + 0 * _as1], a_20 = A[2 * _as0 + 0 * _as1],
690 a_30 = A[3 * _as0 + 0 * _as1];
691
692 ValueType b_0p, c_0p,
693 /**/ c_1p,
694 /**/ c_2p,
695 /**/ c_3p;
696
697 const int ib0 = 0 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0, ic2 = 2 * _cs0, ic3 = 3 * _cs0;
698
699 for (int p = 0; p < n; ++p) {
700 b_0p = B[ib0 + p * _bs1];
701
702 c_0p = a_00 * b_0p;
703 c_1p = a_10 * b_0p;
704 c_2p = a_20 * b_0p;
705 c_3p = a_30 * b_0p;
706
707 C[ic0 + p * _cs1] += alpha * c_0p;
708 C[ic1 + p * _cs1] += alpha * c_1p;
709 C[ic2 + p * _cs1] += alpha * c_2p;
710 C[ic3 + p * _cs1] += alpha * c_3p;
711 }
712
713 return 0;
714 }
715
716 template <>
717 template <typename ScalarType, typename ValueType>
718 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<3, 4>::serial_invoke(const ScalarType alpha,
719 const ValueType *KOKKOS_RESTRICT A,
720 const ValueType *KOKKOS_RESTRICT B, const int n,
721 /**/ ValueType *KOKKOS_RESTRICT C) {
722 if (n <= 0) return 0;
723
724 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
725 a_03 = A[0 * _as0 + 3 * _as1], a_10 = A[1 * _as0 + 0 * _as1], a_11 = A[1 * _as0 + 1 * _as1],
726 a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1], a_20 = A[2 * _as0 + 0 * _as1],
727 a_21 = A[2 * _as0 + 1 * _as1], a_22 = A[2 * _as0 + 2 * _as1], a_23 = A[2 * _as0 + 3 * _as1];
728
729 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, c_2p, b_3p;
730
731 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0,
732 ic2 = 2 * _cs0;
733
734 for (int p = 0; p < n; ++p) {
735 b_0p = B[ib0 + p * _bs1];
736 b_1p = B[ib1 + p * _bs1];
737 b_2p = B[ib2 + p * _bs1];
738 b_3p = B[ib3 + p * _bs1];
739
740 c_0p = a_00 * b_0p;
741 c_0p += a_01 * b_1p;
742 c_0p += a_02 * b_2p;
743 c_0p += a_03 * b_3p;
744 c_1p = a_10 * b_0p;
745 c_1p += a_11 * b_1p;
746 c_1p += a_12 * b_2p;
747 c_1p += a_13 * b_3p;
748 c_2p = a_20 * b_0p;
749 c_2p += a_21 * b_1p;
750 c_2p += a_22 * b_2p;
751 c_2p += a_23 * b_3p;
752
753 C[ic0 + p * _cs1] += alpha * c_0p;
754 C[ic1 + p * _cs1] += alpha * c_1p;
755 C[ic2 + p * _cs1] += alpha * c_2p;
756 }
757
758 return 0;
759 }
760
761 template <>
762 template <typename ScalarType, typename ValueType>
763 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<2, 4>::serial_invoke(const ScalarType alpha,
764 const ValueType *KOKKOS_RESTRICT A,
765 const ValueType *KOKKOS_RESTRICT B, const int n,
766 /**/ ValueType *KOKKOS_RESTRICT C) {
767 if (n <= 0) return 0;
768
769 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
770 a_03 = A[0 * _as0 + 3 * _as1], a_10 = A[1 * _as0 + 0 * _as1], a_11 = A[1 * _as0 + 1 * _as1],
771 a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1];
772
773 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, b_3p;
774
775 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0;
776
777 for (int p = 0; p < n; ++p) {
778 b_0p = B[ib0 + p * _bs1];
779 b_1p = B[ib1 + p * _bs1];
780 b_2p = B[ib2 + p * _bs1];
781 b_3p = B[ib3 + p * _bs1];
782
783 c_0p = a_00 * b_0p;
784 c_0p += a_01 * b_1p;
785 c_0p += a_02 * b_2p;
786 c_0p += a_03 * b_3p;
787 c_1p = a_10 * b_0p;
788 c_1p += a_11 * b_1p;
789 c_1p += a_12 * b_2p;
790 c_1p += a_13 * b_3p;
791
792 C[ic0 + p * _cs1] += alpha * c_0p;
793 C[ic1 + p * _cs1] += alpha * c_1p;
794 }
795
796 return 0;
797 }
798
799 template <>
800 template <typename ScalarType, typename ValueType>
801 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<1, 4>::serial_invoke(const ScalarType alpha,
802 const ValueType *KOKKOS_RESTRICT A,
803 const ValueType *KOKKOS_RESTRICT B, const int n,
804 /**/ ValueType *KOKKOS_RESTRICT C) {
805 if (n <= 0) return 0;
806
807 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
808 a_03 = A[0 * _as0 + 3 * _as1];
809
810 ValueType b_0p, c_0p, b_1p, b_2p, b_3p;
811
812 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ib3 = 3 * _bs0, ic0 = 0 * _cs0;
813
814 for (int p = 0; p < n; ++p) {
815 b_0p = B[ib0 + p * _bs1];
816 b_1p = B[ib1 + p * _bs1];
817 b_2p = B[ib2 + p * _bs1];
818 b_3p = B[ib3 + p * _bs1];
819
820 c_0p = a_00 * b_0p;
821 c_0p += a_01 * b_1p;
822 c_0p += a_02 * b_2p;
823 c_0p += a_03 * b_3p;
824
825 C[ic0 + p * _cs1] += alpha * c_0p;
826 }
827
828 return 0;
829 }
830
831 template <>
832 template <typename ScalarType, typename ValueType>
833 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<4, 4>::serial_invoke(const ScalarType alpha,
834 const ValueType *KOKKOS_RESTRICT A,
835 const ValueType *KOKKOS_RESTRICT B, const int m,
836 const int n, const int k,
837 /**/ ValueType *KOKKOS_RESTRICT C) {
838 if (m <= 0 || n <= 0 || k <= 0) return 0;
839
840 switch (m * 10 + k) {
841 case 44: {
842 InnerGemmFixA<4, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
843 inner.serial_invoke(alpha, A, B, n, C);
844 break;
845 }
846 case 43: {
847 InnerGemmFixA<4, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
848 inner.serial_invoke(alpha, A, B, n, C);
849 break;
850 }
851 case 42: {
852 InnerGemmFixA<4, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
853 inner.serial_invoke(alpha, A, B, n, C);
854 break;
855 }
856 case 41: {
857 InnerGemmFixA<4, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
858 inner.serial_invoke(alpha, A, B, n, C);
859 break;
860 }
861 case 34: {
862 InnerGemmFixA<3, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
863 inner.serial_invoke(alpha, A, B, n, C);
864 break;
865 }
866 case 24: {
867 InnerGemmFixA<2, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
868 inner.serial_invoke(alpha, A, B, n, C);
869 break;
870 }
871 case 14: {
872 InnerGemmFixA<1, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
873 inner.serial_invoke(alpha, A, B, n, C);
874 break;
875 }
876 default: {
877 if (m < 4 && n < 4) {
878 InnerGemmFixA<2, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
879 for (int i = 0; i < m; i += 2)
880 for (int p = 0; p < k; p += 2)
881 inner.serial_invoke(alpha, A + i * _as0 + p * _as1, B + p * _bs0, (i + 2 > m ? 1 : 2), n,
882 (p + 2 > k ? 1 : 2), C + i * _cs0);
883 } else {
884 Kokkos::abort("InnerGemmFixA<4,4>::serial_invoke, assert failure (m<4 && n<4)");
885 }
886 break;
887 }
888 }
889
890 return 0;
891 }
892
893 ///
894 /// Inner kernel (3x3)
895 /// ==================
896
897 template <>
898 template <typename ScalarType, typename ValueType>
899 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<3, 3>::serial_invoke(const ScalarType alpha,
900 const ValueType *KOKKOS_RESTRICT A,
901 const ValueType *KOKKOS_RESTRICT B, const int n,
902 /**/ ValueType *KOKKOS_RESTRICT C) {
903 if (n <= 0) return 0;
904
905 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
906 a_10 = A[1 * _as0 + 0 * _as1], a_11 = A[1 * _as0 + 1 * _as1], a_12 = A[1 * _as0 + 2 * _as1],
907 a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1], a_22 = A[2 * _as0 + 2 * _as1];
908
909 ValueType b_0p, c_0p, b_1p, c_1p, b_2p, c_2p;
910
911 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0, ic2 = 2 * _cs0;
912
913 for (int p = 0; p < n; ++p) {
914 b_0p = B[ib0 + p * _bs1];
915 b_1p = B[ib1 + p * _bs1];
916 b_2p = B[ib2 + p * _bs1];
917
918 c_0p = a_00 * b_0p;
919 c_0p += a_01 * b_1p;
920 c_0p += a_02 * b_2p;
921 c_1p = a_10 * b_0p;
922 c_1p += a_11 * b_1p;
923 c_1p += a_12 * b_2p;
924 c_2p = a_20 * b_0p;
925 c_2p += a_21 * b_1p;
926 c_2p += a_22 * b_2p;
927
928 C[ic0 + p * _cs1] += alpha * c_0p;
929 C[ic1 + p * _cs1] += alpha * c_1p;
930 C[ic2 + p * _cs1] += alpha * c_2p;
931 }
932
933 return 0;
934 }
935
936 template <>
937 template <typename ScalarType, typename ValueType>
938 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<3, 2>::serial_invoke(const ScalarType alpha,
939 const ValueType *KOKKOS_RESTRICT A,
940 const ValueType *KOKKOS_RESTRICT B, const int n,
941 /**/ ValueType *KOKKOS_RESTRICT C) {
942 if (n <= 0) return 0;
943
944 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_10 = A[1 * _as0 + 0 * _as1],
945 a_11 = A[1 * _as0 + 1 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1];
946
947 ValueType b_0p, c_0p, b_1p, c_1p,
948 /**/ c_2p;
949
950 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0, ic2 = 2 * _cs0;
951
952 for (int p = 0; p < n; ++p) {
953 b_0p = B[ib0 + p * _bs1];
954 b_1p = B[ib1 + p * _bs1];
955
956 c_0p = a_00 * b_0p;
957 c_0p += a_01 * b_1p;
958 c_1p = a_10 * b_0p;
959 c_1p += a_11 * b_1p;
960 c_2p = a_20 * b_0p;
961 c_2p += a_21 * b_1p;
962
963 C[ic0 + p * _cs1] += alpha * c_0p;
964 C[ic1 + p * _cs1] += alpha * c_1p;
965 C[ic2 + p * _cs1] += alpha * c_2p;
966 }
967
968 return 0;
969 }
970
971 template <>
972 template <typename ScalarType, typename ValueType>
973 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<3, 1>::serial_invoke(const ScalarType alpha,
974 const ValueType *KOKKOS_RESTRICT A,
975 const ValueType *KOKKOS_RESTRICT B, const int n,
976 /**/ ValueType *KOKKOS_RESTRICT C) {
977 if (n <= 0) return 0;
978
979 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_10 = A[1 * _as0 + 0 * _as1], a_20 = A[2 * _as0 + 0 * _as1];
980
981 ValueType b_0p, c_0p,
982 /**/ c_1p,
983 /**/ c_2p;
984
985 const int ib0 = 0 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0, ic2 = 2 * _cs0;
986
987 for (int p = 0; p < n; ++p) {
988 b_0p = B[ib0 + p * _bs1];
989
990 c_0p = a_00 * b_0p;
991 c_1p = a_10 * b_0p;
992 c_2p = a_20 * b_0p;
993
994 C[ic0 + p * _cs1] += alpha * c_0p;
995 C[ic1 + p * _cs1] += alpha * c_1p;
996 C[ic2 + p * _cs1] += alpha * c_2p;
997 }
998
999 return 0;
1000 }
1001
1002 template <>
1003 template <typename ScalarType, typename ValueType>
1004 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<2, 3>::serial_invoke(const ScalarType alpha,
1005 const ValueType *KOKKOS_RESTRICT A,
1006 const ValueType *KOKKOS_RESTRICT B, const int n,
1007 /**/ ValueType *KOKKOS_RESTRICT C) {
1008 if (n <= 0) return 0;
1009
1010 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
1011 a_10 = A[1 * _as0 + 0 * _as1], a_11 = A[1 * _as0 + 1 * _as1], a_12 = A[1 * _as0 + 2 * _as1];
1012
1013 ValueType b_0p, c_0p, b_1p, c_1p, b_2p;
1014
1015 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0;
1016
1017 for (int p = 0; p < n; ++p) {
1018 b_0p = B[ib0 + p * _bs1];
1019 b_1p = B[ib1 + p * _bs1];
1020 b_2p = B[ib2 + p * _bs1];
1021
1022 c_0p = a_00 * b_0p;
1023 c_0p += a_01 * b_1p;
1024 c_0p += a_02 * b_2p;
1025 c_1p = a_10 * b_0p;
1026 c_1p += a_11 * b_1p;
1027 c_1p += a_12 * b_2p;
1028
1029 C[ic0 + p * _cs1] += alpha * c_0p;
1030 C[ic1 + p * _cs1] += alpha * c_1p;
1031 }
1032
1033 return 0;
1034 }
1035 template <>
1036 template <typename ScalarType, typename ValueType>
1037 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<1, 3>::serial_invoke(const ScalarType alpha,
1038 const ValueType *KOKKOS_RESTRICT A,
1039 const ValueType *KOKKOS_RESTRICT B, const int n,
1040 /**/ ValueType *KOKKOS_RESTRICT C) {
1041 if (n <= 0) return 0;
1042
1043 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1];
1044
1045 ValueType b_0p, c_0p, b_1p, b_2p;
1046
1047 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ib2 = 2 * _bs0, ic0 = 0 * _cs0;
1048
1049 for (int p = 0; p < n; ++p) {
1050 b_0p = B[ib0 + p * _bs1];
1051 b_1p = B[ib1 + p * _bs1];
1052 b_2p = B[ib2 + p * _bs1];
1053
1054 c_0p = a_00 * b_0p;
1055 c_0p += a_01 * b_1p;
1056 c_0p += a_02 * b_2p;
1057
1058 C[ic0 + p * _cs1] += alpha * c_0p;
1059 }
1060
1061 return 0;
1062 }
1063
1064 template <>
1065 template <typename ScalarType, typename ValueType>
1066 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<3, 3>::serial_invoke(const ScalarType alpha,
1067 const ValueType *KOKKOS_RESTRICT A,
1068 const ValueType *KOKKOS_RESTRICT B, const int m,
1069 const int n, const int k,
1070 /**/ ValueType *KOKKOS_RESTRICT C) {
1071 if (m <= 0 || n <= 0 || k <= 0) return 0;
1072
1073 switch (m * 10 + k) {
1074 case 33: {
1075 InnerGemmFixA<3, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1076 inner.serial_invoke(alpha, A, B, n, C);
1077 break;
1078 }
1079 case 32: {
1080 InnerGemmFixA<3, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1081 inner.serial_invoke(alpha, A, B, n, C);
1082 break;
1083 }
1084 case 31: {
1085 InnerGemmFixA<3, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1086 inner.serial_invoke(alpha, A, B, n, C);
1087 break;
1088 }
1089 case 23: {
1090 InnerGemmFixA<2, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1091 inner.serial_invoke(alpha, A, B, n, C);
1092 break;
1093 }
1094 case 13: {
1095 InnerGemmFixA<1, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1096 inner.serial_invoke(alpha, A, B, n, C);
1097 break;
1098 }
1099 default: {
1100 if (m < 3 && n < 3) {
1101 InnerGemmFixA<2, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1102 for (int i = 0; i < m; i += 2)
1103 for (int p = 0; p < k; p += 2)
1104 inner.serial_invoke(alpha, A + i * _as0 + p * _as1, B + p * _bs0, (i + 2 > m ? 1 : 2), n,
1105 (p + 2 > k ? 1 : 2), C + i * _cs0);
1106 } else {
1107 Kokkos::abort("InnerGemmFixA<3,3>::serial_invoke, assert failure (m<3 && n<3)");
1108 }
1109 break;
1110 }
1111 }
1112
1113 return 0;
1114 }
1115
1116 ///
1117 /// Inner kernel (2x2)
1118 /// ==================
1119
1120 template <>
1121 template <typename ScalarType, typename ValueType>
1122 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<2, 2>::serial_invoke(const ScalarType alpha,
1123 const ValueType *KOKKOS_RESTRICT A,
1124 const ValueType *KOKKOS_RESTRICT B, const int n,
1125 /**/ ValueType *KOKKOS_RESTRICT C) {
1126 if (n <= 0) return 0;
1127
1128 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1], a_10 = A[1 * _as0 + 0 * _as1],
1129 a_11 = A[1 * _as0 + 1 * _as1];
1130
1131 ValueType b_0p, c_0p, b_1p, c_1p;
1132
1133 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0;
1134
1135 for (int p = 0; p < n; ++p) {
1136 b_0p = B[ib0 + p * _bs1];
1137 b_1p = B[ib1 + p * _bs1];
1138
1139 c_0p = a_00 * b_0p;
1140 c_0p += a_01 * b_1p;
1141 c_1p = a_10 * b_0p;
1142 c_1p += a_11 * b_1p;
1143
1144 C[ic0 + p * _cs1] += alpha * c_0p;
1145 C[ic1 + p * _cs1] += alpha * c_1p;
1146 }
1147
1148 return 0;
1149 }
1150
1151 template <>
1152 template <typename ScalarType, typename ValueType>
1153 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<2, 1>::serial_invoke(const ScalarType alpha,
1154 const ValueType *KOKKOS_RESTRICT A,
1155 const ValueType *KOKKOS_RESTRICT B, const int n,
1156 /**/ ValueType *KOKKOS_RESTRICT C) {
1157 if (n <= 0) return 0;
1158
1159 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_10 = A[1 * _as0 + 0 * _as1];
1160
1161 ValueType b_0p, c_0p,
1162 /**/ c_1p;
1163
1164 const int ib0 = 0 * _bs0, ic0 = 0 * _cs0, ic1 = 1 * _cs0;
1165
1166 for (int p = 0; p < n; ++p) {
1167 b_0p = B[ib0 + p * _bs1];
1168
1169 c_0p = a_00 * b_0p;
1170 c_1p = a_10 * b_0p;
1171
1172 C[ic0 + p * _cs1] += alpha * c_0p;
1173 C[ic1 + p * _cs1] += alpha * c_1p;
1174 }
1175
1176 return 0;
1177 }
1178
1179 template <>
1180 template <typename ScalarType, typename ValueType>
1181 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<1, 2>::serial_invoke(const ScalarType alpha,
1182 const ValueType *KOKKOS_RESTRICT A,
1183 const ValueType *KOKKOS_RESTRICT B, const int n,
1184 /**/ ValueType *KOKKOS_RESTRICT C) {
1185 if (n <= 0) return 0;
1186
1187 const ValueType a_00 = A[0 * _as0 + 0 * _as1], a_01 = A[0 * _as0 + 1 * _as1];
1188
1189 ValueType b_0p, c_0p, b_1p;
1190
1191 const int ib0 = 0 * _bs0, ib1 = 1 * _bs0, ic0 = 0 * _cs0;
1192
1193 for (int p = 0; p < n; ++p) {
1194 b_0p = B[ib0 + p * _bs1];
1195 b_1p = B[ib1 + p * _bs1];
1196
1197 c_0p = a_00 * b_0p;
1198 c_0p += a_01 * b_1p;
1199
1200 C[ic0 + p * _cs1] += alpha * c_0p;
1201 }
1202
1203 return 0;
1204 }
1205
1206 template <>
1207 template <typename ScalarType, typename ValueType>
1208 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<2, 2>::serial_invoke(const ScalarType alpha,
1209 const ValueType *KOKKOS_RESTRICT A,
1210 const ValueType *KOKKOS_RESTRICT B, const int m,
1211 const int n, const int k,
1212 /**/ ValueType *KOKKOS_RESTRICT C) {
1213 if (m <= 0 || n <= 0 || k <= 0) return 0;
1214
1215 switch (m * 10 + k) {
1216 case 22: {
1217 InnerGemmFixA<2, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1218 inner.serial_invoke(alpha, A, B, n, C);
1219 break;
1220 }
1221 case 21: {
1222 InnerGemmFixA<2, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1223 inner.serial_invoke(alpha, A, B, n, C);
1224 break;
1225 }
1226 case 12: {
1227 InnerGemmFixA<1, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1228 inner.serial_invoke(alpha, A, B, n, C);
1229 break;
1230 }
1231 case 11: {
1232 InnerGemmFixA<1, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1233 inner.serial_invoke(alpha, A, B, n, C);
1234 break;
1235 }
1236 default: {
1237 Kokkos::abort("InnerGemmFixA<2,2>::serial_invoke, assert failure (m<2 && n<2)");
1238 break;
1239 }
1240 }
1241
1242 return 0;
1243 }
1244
1245 ///
1246 /// Inner kernel (1x1)
1247 /// ==================
1248
1249 template <>
1250 template <typename ScalarType, typename ValueType>
1251 KOKKOS_INLINE_FUNCTION int InnerGemmFixA<1, 1>::serial_invoke(const ScalarType alpha,
1252 const ValueType *KOKKOS_RESTRICT A,
1253 const ValueType *KOKKOS_RESTRICT B, const int n,
1254 /**/ ValueType *KOKKOS_RESTRICT C) {
1255 if (n <= 0) return 0;
1256
1257 const ValueType a_00 = A[0 * _as0 + 0 * _as1];
1258
1259 ValueType b_0p, c_0p;
1260
1261 const int ib0 = 0 * _bs0, ic0 = 0 * _cs0;
1262
1263 for (int p = 0; p < n; ++p) {
1264 b_0p = B[ib0 + p * _bs1];
1265
1266 c_0p = a_00 * b_0p;
1267
1268 C[ic0 + p * _cs1] += alpha * c_0p;
1269 }
1270
1271 return 0;
1272 }
1273
1274 } // namespace KokkosBatched
1275
1276 #endif