Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_InnerGemmFixB_Serial_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_INNER_GEMM_FIX_B_SERIAL_IMPL_HPP
17 #define KOKKOSBATCHED_INNER_GEMM_FIX_B_SERIAL_IMPL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosBatched_InnerGemmFixB_Decl.hpp"
23
24 namespace KokkosBatched {
25
26 ///
27 /// Inner kernel (5x5)
28 /// ==================
29
30 template <>
31 template <typename ScalarType, typename ValueType>
32 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<5, 5>::serial_invoke(const ScalarType alpha,
33 const ValueType *KOKKOS_RESTRICT A,
34 const ValueType *KOKKOS_RESTRICT B, const int m,
35 /**/ ValueType *KOKKOS_RESTRICT C) {
36 if (m <= 0) return 0;
37
38 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
39 b_03 = B[0 * _bs0 + 3 * _bs1], b_04 = B[0 * _bs0 + 4 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1],
40 b_11 = B[1 * _bs0 + 1 * _bs1], b_12 = B[1 * _bs0 + 2 * _bs1], b_13 = B[1 * _bs0 + 3 * _bs1],
41 b_14 = B[1 * _bs0 + 4 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1], b_21 = B[2 * _bs0 + 1 * _bs1],
42 b_22 = B[2 * _bs0 + 2 * _bs1], b_23 = B[2 * _bs0 + 3 * _bs1], b_24 = B[2 * _bs0 + 4 * _bs1],
43 b_30 = B[3 * _bs0 + 0 * _bs1], b_31 = B[3 * _bs0 + 1 * _bs1], b_32 = B[3 * _bs0 + 2 * _bs1],
44 b_33 = B[3 * _bs0 + 3 * _bs1], b_34 = B[3 * _bs0 + 4 * _bs1], b_40 = B[4 * _bs0 + 0 * _bs1],
45 b_41 = B[4 * _bs0 + 1 * _bs1], b_42 = B[4 * _bs0 + 2 * _bs1], b_43 = B[4 * _bs0 + 3 * _bs1],
46 b_44 = B[4 * _bs0 + 4 * _bs1];
47
48 ValueType a_p0, a_p1, a_p2, a_p3, a_p4, c_p0, c_p1, c_p2, c_p3, c_p4;
49
50 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, ja4 = 4 * _as1, jc0 = 0 * _cs1,
51 jc1 = 1 * _cs1, jc2 = 2 * _cs1, jc3 = 3 * _cs1, jc4 = 4 * _cs1;
52
53 for (int p = 0; p < m; ++p) {
54 a_p0 = A[p * _bs0 + ja0];
55 a_p1 = A[p * _bs0 + ja1];
56 a_p2 = A[p * _bs0 + ja2];
57 a_p3 = A[p * _bs0 + ja3];
58 a_p4 = A[p * _bs0 + ja4];
59
60 c_p0 = a_p0 * b_00;
61 c_p1 = a_p0 * b_01;
62 c_p2 = a_p0 * b_02;
63 c_p3 = a_p0 * b_03;
64 c_p4 = a_p0 * b_04;
65 c_p0 += a_p1 * b_10;
66 c_p1 += a_p1 * b_11;
67 c_p2 += a_p1 * b_12;
68 c_p3 += a_p1 * b_13;
69 c_p4 += a_p1 * b_14;
70 c_p0 += a_p2 * b_20;
71 c_p1 += a_p2 * b_21;
72 c_p2 += a_p2 * b_22;
73 c_p3 += a_p2 * b_23;
74 c_p4 += a_p2 * b_24;
75 c_p0 += a_p3 * b_30;
76 c_p1 += a_p3 * b_31;
77 c_p2 += a_p3 * b_32;
78 c_p3 += a_p3 * b_33;
79 c_p4 += a_p3 * b_34;
80 c_p0 += a_p4 * b_40;
81 c_p1 += a_p4 * b_41;
82 c_p2 += a_p4 * b_42;
83 c_p3 += a_p4 * b_43;
84 c_p4 += a_p4 * b_44;
85
86 C[p * _cs0 + jc0] += alpha * c_p0;
87 C[p * _cs0 + jc1] += alpha * c_p1;
88 C[p * _cs0 + jc2] += alpha * c_p2;
89 C[p * _cs0 + jc3] += alpha * c_p3;
90 C[p * _cs0 + jc4] += alpha * c_p4;
91 }
92
93 return 0;
94 }
95
96 template <>
97 template <typename ScalarType, typename ValueType>
98 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<5, 4>::serial_invoke(const ScalarType alpha,
99 const ValueType *KOKKOS_RESTRICT A,
100 const ValueType *KOKKOS_RESTRICT B, const int m,
101 /**/ ValueType *KOKKOS_RESTRICT C) {
102 if (m <= 0) return 0;
103
104 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
105 b_03 = B[0 * _bs0 + 3 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1], b_11 = B[1 * _bs0 + 1 * _bs1],
106 b_12 = B[1 * _bs0 + 2 * _bs1], b_13 = B[1 * _bs0 + 3 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1],
107 b_21 = B[2 * _bs0 + 1 * _bs1], b_22 = B[2 * _bs0 + 2 * _bs1], b_23 = B[2 * _bs0 + 3 * _bs1],
108 b_30 = B[3 * _bs0 + 0 * _bs1], b_31 = B[3 * _bs0 + 1 * _bs1], b_32 = B[3 * _bs0 + 2 * _bs1],
109 b_33 = B[3 * _bs0 + 3 * _bs1], b_40 = B[4 * _bs0 + 0 * _bs1], b_41 = B[4 * _bs0 + 1 * _bs1],
110 b_42 = B[4 * _bs0 + 2 * _bs1], b_43 = B[4 * _bs0 + 3 * _bs1];
111
112 ValueType a_p0, a_p1, a_p2, a_p3, a_p4, c_p0, c_p1, c_p2, c_p3;
113
114 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, ja4 = 4 * _as1, jc0 = 0 * _cs1,
115 jc1 = 1 * _cs1, jc2 = 2 * _cs1, jc3 = 3 * _cs1;
116
117 for (int p = 0; p < m; ++p) {
118 a_p0 = A[p * _bs0 + ja0];
119 a_p1 = A[p * _bs0 + ja1];
120 a_p2 = A[p * _bs0 + ja2];
121 a_p3 = A[p * _bs0 + ja3];
122 a_p4 = A[p * _bs0 + ja4];
123
124 c_p0 = a_p0 * b_00;
125 c_p1 = a_p0 * b_01;
126 c_p2 = a_p0 * b_02;
127 c_p3 = a_p0 * b_03;
128 c_p0 += a_p1 * b_10;
129 c_p1 += a_p1 * b_11;
130 c_p2 += a_p1 * b_12;
131 c_p3 += a_p1 * b_13;
132 c_p0 += a_p2 * b_20;
133 c_p1 += a_p2 * b_21;
134 c_p2 += a_p2 * b_22;
135 c_p3 += a_p2 * b_23;
136 c_p0 += a_p3 * b_30;
137 c_p1 += a_p3 * b_31;
138 c_p2 += a_p3 * b_32;
139 c_p3 += a_p3 * b_33;
140 c_p0 += a_p4 * b_40;
141 c_p1 += a_p4 * b_41;
142 c_p2 += a_p4 * b_42;
143 c_p3 += a_p4 * b_43;
144
145 C[p * _cs0 + jc0] += alpha * c_p0;
146 C[p * _cs0 + jc1] += alpha * c_p1;
147 C[p * _cs0 + jc2] += alpha * c_p2;
148 C[p * _cs0 + jc3] += alpha * c_p3;
149 }
150
151 return 0;
152 }
153
154 template <>
155 template <typename ScalarType, typename ValueType>
156 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<5, 3>::serial_invoke(const ScalarType alpha,
157 const ValueType *KOKKOS_RESTRICT A,
158 const ValueType *KOKKOS_RESTRICT B, const int m,
159 /**/ ValueType *KOKKOS_RESTRICT C) {
160 if (m <= 0) return 0;
161
162 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
163 b_10 = B[1 * _bs0 + 0 * _bs1], b_11 = B[1 * _bs0 + 1 * _bs1], b_12 = B[1 * _bs0 + 2 * _bs1],
164 b_20 = B[2 * _bs0 + 0 * _bs1], b_21 = B[2 * _bs0 + 1 * _bs1], b_22 = B[2 * _bs0 + 2 * _bs1],
165 b_30 = B[3 * _bs0 + 0 * _bs1], b_31 = B[3 * _bs0 + 1 * _bs1], b_32 = B[3 * _bs0 + 2 * _bs1],
166 b_40 = B[4 * _bs0 + 0 * _bs1], b_41 = B[4 * _bs0 + 1 * _bs1], b_42 = B[4 * _bs0 + 2 * _bs1];
167
168 ValueType a_p0, a_p1, a_p2, a_p3, a_p4, c_p0, c_p1, c_p2;
169
170 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, ja4 = 4 * _as1, jc0 = 0 * _cs1,
171 jc1 = 1 * _cs1, jc2 = 2 * _cs1;
172
173 for (int p = 0; p < m; ++p) {
174 a_p0 = A[p * _bs0 + ja0];
175 a_p1 = A[p * _bs0 + ja1];
176 a_p2 = A[p * _bs0 + ja2];
177 a_p3 = A[p * _bs0 + ja3];
178 a_p4 = A[p * _bs0 + ja4];
179
180 c_p0 = a_p0 * b_00;
181 c_p1 = a_p0 * b_01;
182 c_p2 = a_p0 * b_02;
183 c_p0 += a_p1 * b_10;
184 c_p1 += a_p1 * b_11;
185 c_p2 += a_p1 * b_12;
186 c_p0 += a_p2 * b_20;
187 c_p1 += a_p2 * b_21;
188 c_p2 += a_p2 * b_22;
189 c_p0 += a_p3 * b_30;
190 c_p1 += a_p3 * b_31;
191 c_p2 += a_p3 * b_32;
192 c_p0 += a_p4 * b_40;
193 c_p1 += a_p4 * b_41;
194 c_p2 += a_p4 * b_42;
195
196 C[p * _cs0 + jc0] += alpha * c_p0;
197 C[p * _cs0 + jc1] += alpha * c_p1;
198 C[p * _cs0 + jc2] += alpha * c_p2;
199 }
200
201 return 0;
202 }
203 template <>
204 template <typename ScalarType, typename ValueType>
205 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<5, 2>::serial_invoke(const ScalarType alpha,
206 const ValueType *KOKKOS_RESTRICT A,
207 const ValueType *KOKKOS_RESTRICT B, const int m,
208 /**/ ValueType *KOKKOS_RESTRICT C) {
209 if (m <= 0) return 0;
210
211 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1],
212 b_11 = B[1 * _bs0 + 1 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1], b_21 = B[2 * _bs0 + 1 * _bs1],
213 b_30 = B[3 * _bs0 + 0 * _bs1], b_31 = B[3 * _bs0 + 1 * _bs1], b_40 = B[4 * _bs0 + 0 * _bs1],
214 b_41 = B[4 * _bs0 + 1 * _bs1];
215
216 ValueType a_p0, a_p1, a_p2, a_p3, a_p4, c_p0, c_p1;
217
218 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, ja4 = 4 * _as1, jc0 = 0 * _cs1,
219 jc1 = 1 * _cs1;
220
221 for (int p = 0; p < m; ++p) {
222 a_p0 = A[p * _bs0 + ja0];
223 a_p1 = A[p * _bs0 + ja1];
224 a_p2 = A[p * _bs0 + ja2];
225 a_p3 = A[p * _bs0 + ja3];
226 a_p4 = A[p * _bs0 + ja4];
227
228 c_p0 = a_p0 * b_00;
229 c_p1 = a_p0 * b_01;
230 c_p0 += a_p1 * b_10;
231 c_p1 += a_p1 * b_11;
232 c_p0 += a_p2 * b_20;
233 c_p1 += a_p2 * b_21;
234 c_p0 += a_p3 * b_30;
235 c_p1 += a_p3 * b_31;
236 c_p0 += a_p4 * b_40;
237 c_p1 += a_p4 * b_41;
238
239 C[p * _cs0 + jc0] += alpha * c_p0;
240 C[p * _cs0 + jc1] += alpha * c_p1;
241 }
242
243 return 0;
244 }
245 template <>
246 template <typename ScalarType, typename ValueType>
247 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<5, 1>::serial_invoke(const ScalarType alpha,
248 const ValueType *KOKKOS_RESTRICT A,
249 const ValueType *KOKKOS_RESTRICT B, const int m,
250 /**/ ValueType *KOKKOS_RESTRICT C) {
251 if (m <= 0) return 0;
252
253 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1],
254 b_30 = B[3 * _bs0 + 0 * _bs1], b_40 = B[4 * _bs0 + 0 * _bs1];
255
256 ValueType a_p0, a_p1, a_p2, a_p3, a_p4, c_p0;
257
258 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, ja4 = 4 * _as1, jc0 = 0 * _cs1;
259
260 for (int p = 0; p < m; ++p) {
261 a_p0 = A[p * _bs0 + ja0];
262 a_p1 = A[p * _bs0 + ja1];
263 a_p2 = A[p * _bs0 + ja2];
264 a_p3 = A[p * _bs0 + ja3];
265 a_p4 = A[p * _bs0 + ja4];
266
267 c_p0 = a_p0 * b_00;
268 c_p0 += a_p1 * b_10;
269 c_p0 += a_p2 * b_20;
270 c_p0 += a_p3 * b_30;
271 c_p0 += a_p4 * b_40;
272
273 C[p * _cs0 + jc0] += alpha * c_p0;
274 }
275
276 return 0;
277 }
278
279 template <>
280 template <typename ScalarType, typename ValueType>
281 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<4, 5>::serial_invoke(const ScalarType alpha,
282 const ValueType *KOKKOS_RESTRICT A,
283 const ValueType *KOKKOS_RESTRICT B, const int m,
284 /**/ ValueType *KOKKOS_RESTRICT C) {
285 if (m <= 0) return 0;
286
287 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
288 b_03 = B[0 * _bs0 + 3 * _bs1], b_04 = B[0 * _bs0 + 4 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1],
289 b_11 = B[1 * _bs0 + 1 * _bs1], b_12 = B[1 * _bs0 + 2 * _bs1], b_13 = B[1 * _bs0 + 3 * _bs1],
290 b_14 = B[1 * _bs0 + 4 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1], b_21 = B[2 * _bs0 + 1 * _bs1],
291 b_22 = B[2 * _bs0 + 2 * _bs1], b_23 = B[2 * _bs0 + 3 * _bs1], b_24 = B[2 * _bs0 + 4 * _bs1],
292 b_30 = B[3 * _bs0 + 0 * _bs1], b_31 = B[3 * _bs0 + 1 * _bs1], b_32 = B[3 * _bs0 + 2 * _bs1],
293 b_33 = B[3 * _bs0 + 3 * _bs1], b_34 = B[3 * _bs0 + 4 * _bs1];
294
295 ValueType a_p0, a_p1, a_p2, a_p3, c_p0, c_p1, c_p2, c_p3, c_p4;
296
297 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1,
298 jc2 = 2 * _cs1, jc3 = 3 * _cs1, jc4 = 4 * _cs1;
299
300 for (int p = 0; p < m; ++p) {
301 a_p0 = A[p * _bs0 + ja0];
302 a_p1 = A[p * _bs0 + ja1];
303 a_p2 = A[p * _bs0 + ja2];
304 a_p3 = A[p * _bs0 + ja3];
305
306 c_p0 = a_p0 * b_00;
307 c_p1 = a_p0 * b_01;
308 c_p2 = a_p0 * b_02;
309 c_p3 = a_p0 * b_03;
310 c_p4 = a_p0 * b_04;
311 c_p0 += a_p1 * b_10;
312 c_p1 += a_p1 * b_11;
313 c_p2 += a_p1 * b_12;
314 c_p3 += a_p1 * b_13;
315 c_p4 += a_p1 * b_14;
316 c_p0 += a_p2 * b_20;
317 c_p1 += a_p2 * b_21;
318 c_p2 += a_p2 * b_22;
319 c_p3 += a_p2 * b_23;
320 c_p4 += a_p2 * b_24;
321 c_p0 += a_p3 * b_30;
322 c_p1 += a_p3 * b_31;
323 c_p2 += a_p3 * b_32;
324 c_p3 += a_p3 * b_33;
325 c_p4 += a_p3 * b_34;
326
327 C[p * _cs0 + jc0] += alpha * c_p0;
328 C[p * _cs0 + jc1] += alpha * c_p1;
329 C[p * _cs0 + jc2] += alpha * c_p2;
330 C[p * _cs0 + jc3] += alpha * c_p3;
331 C[p * _cs0 + jc4] += alpha * c_p4;
332 }
333
334 return 0;
335 }
336
337 template <>
338 template <typename ScalarType, typename ValueType>
339 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<3, 5>::serial_invoke(const ScalarType alpha,
340 const ValueType *KOKKOS_RESTRICT A,
341 const ValueType *KOKKOS_RESTRICT B, const int m,
342 /**/ ValueType *KOKKOS_RESTRICT C) {
343 if (m <= 0) return 0;
344
345 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
346 b_03 = B[0 * _bs0 + 3 * _bs1], b_04 = B[0 * _bs0 + 4 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1],
347 b_11 = B[1 * _bs0 + 1 * _bs1], b_12 = B[1 * _bs0 + 2 * _bs1], b_13 = B[1 * _bs0 + 3 * _bs1],
348 b_14 = B[1 * _bs0 + 4 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1], b_21 = B[2 * _bs0 + 1 * _bs1],
349 b_22 = B[2 * _bs0 + 2 * _bs1], b_23 = B[2 * _bs0 + 3 * _bs1], b_24 = B[2 * _bs0 + 4 * _bs1];
350
351 ValueType a_p0, a_p1, a_p2, c_p0, c_p1, c_p2, c_p3, c_p4;
352
353 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1, jc2 = 2 * _cs1,
354 jc3 = 3 * _cs1, jc4 = 4 * _cs1;
355
356 for (int p = 0; p < m; ++p) {
357 a_p0 = A[p * _bs0 + ja0];
358 a_p1 = A[p * _bs0 + ja1];
359 a_p2 = A[p * _bs0 + ja2];
360
361 c_p0 = a_p0 * b_00;
362 c_p1 = a_p0 * b_01;
363 c_p2 = a_p0 * b_02;
364 c_p3 = a_p0 * b_03;
365 c_p4 = a_p0 * b_04;
366 c_p0 += a_p1 * b_10;
367 c_p1 += a_p1 * b_11;
368 c_p2 += a_p1 * b_12;
369 c_p3 += a_p1 * b_13;
370 c_p4 += a_p1 * b_14;
371 c_p0 += a_p2 * b_20;
372 c_p1 += a_p2 * b_21;
373 c_p2 += a_p2 * b_22;
374 c_p3 += a_p2 * b_23;
375 c_p4 += a_p2 * b_24;
376
377 C[p * _cs0 + jc0] += alpha * c_p0;
378 C[p * _cs0 + jc1] += alpha * c_p1;
379 C[p * _cs0 + jc2] += alpha * c_p2;
380 C[p * _cs0 + jc3] += alpha * c_p3;
381 C[p * _cs0 + jc4] += alpha * c_p4;
382 }
383
384 return 0;
385 }
386 template <>
387 template <typename ScalarType, typename ValueType>
388 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<2, 5>::serial_invoke(const ScalarType alpha,
389 const ValueType *KOKKOS_RESTRICT A,
390 const ValueType *KOKKOS_RESTRICT B, const int m,
391 /**/ ValueType *KOKKOS_RESTRICT C) {
392 if (m <= 0) return 0;
393
394 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
395 b_03 = B[0 * _bs0 + 3 * _bs1], b_04 = B[0 * _bs0 + 4 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1],
396 b_11 = B[1 * _bs0 + 1 * _bs1], b_12 = B[1 * _bs0 + 2 * _bs1], b_13 = B[1 * _bs0 + 3 * _bs1],
397 b_14 = B[1 * _bs0 + 4 * _bs1];
398
399 ValueType a_p0, a_p1, c_p0, c_p1, c_p2, c_p3, c_p4;
400
401 const int ja0 = 0 * _as1, ja1 = 1 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1, jc2 = 2 * _cs1, jc3 = 3 * _cs1,
402 jc4 = 4 * _cs1;
403
404 for (int p = 0; p < m; ++p) {
405 a_p0 = A[p * _bs0 + ja0];
406 a_p1 = A[p * _bs0 + ja1];
407
408 c_p0 = a_p0 * b_00;
409 c_p1 = a_p0 * b_01;
410 c_p2 = a_p0 * b_02;
411 c_p3 = a_p0 * b_03;
412 c_p4 = a_p0 * b_04;
413 c_p0 += a_p1 * b_10;
414 c_p1 += a_p1 * b_11;
415 c_p2 += a_p1 * b_12;
416 c_p3 += a_p1 * b_13;
417 c_p4 += a_p1 * b_14;
418
419 C[p * _cs0 + jc0] += alpha * c_p0;
420 C[p * _cs0 + jc1] += alpha * c_p1;
421 C[p * _cs0 + jc2] += alpha * c_p2;
422 C[p * _cs0 + jc3] += alpha * c_p3;
423 C[p * _cs0 + jc4] += alpha * c_p4;
424 }
425
426 return 0;
427 }
428 template <>
429 template <typename ScalarType, typename ValueType>
430 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<1, 5>::serial_invoke(const ScalarType alpha,
431 const ValueType *KOKKOS_RESTRICT A,
432 const ValueType *KOKKOS_RESTRICT B, const int m,
433 /**/ ValueType *KOKKOS_RESTRICT C) {
434 if (m <= 0) return 0;
435
436 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
437 b_03 = B[0 * _bs0 + 3 * _bs1], b_04 = B[0 * _bs0 + 4 * _bs1];
438
439 ValueType a_p0, c_p0, c_p1, c_p2, c_p3, c_p4;
440
441 const int ja0 = 0 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1, jc2 = 2 * _cs1, jc3 = 3 * _cs1, jc4 = 4 * _cs1;
442
443 for (int p = 0; p < m; ++p) {
444 a_p0 = A[p * _bs0 + ja0];
445
446 c_p0 = a_p0 * b_00;
447 c_p1 = a_p0 * b_01;
448 c_p2 = a_p0 * b_02;
449 c_p3 = a_p0 * b_03;
450 c_p4 = a_p0 * b_04;
451
452 C[p * _cs0 + jc0] += alpha * c_p0;
453 C[p * _cs0 + jc1] += alpha * c_p1;
454 C[p * _cs0 + jc2] += alpha * c_p2;
455 C[p * _cs0 + jc3] += alpha * c_p3;
456 C[p * _cs0 + jc4] += alpha * c_p4;
457 }
458
459 return 0;
460 }
461
462 template <>
463 template <typename ScalarType, typename ValueType>
464 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<5, 5>::serial_invoke(const ScalarType alpha,
465 const ValueType *KOKKOS_RESTRICT A,
466 const ValueType *KOKKOS_RESTRICT B, const int m,
467 const int n, const int k,
468 /**/ ValueType *KOKKOS_RESTRICT C) {
469 if (m <= 0 || n <= 0 || k <= 0) return 0;
470
471 switch (k * 10 + n) {
472 case 54: {
473 InnerGemmFixB<5, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
474 inner.serial_invoke(alpha, A, B, m, C);
475 break;
476 }
477 case 53: {
478 InnerGemmFixB<5, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
479 inner.serial_invoke(alpha, A, B, m, C);
480 break;
481 }
482 case 52: {
483 InnerGemmFixB<5, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
484 inner.serial_invoke(alpha, A, B, m, C);
485 break;
486 }
487 case 51: {
488 InnerGemmFixB<5, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
489 inner.serial_invoke(alpha, A, B, m, C);
490 break;
491 }
492 case 45: {
493 InnerGemmFixB<4, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
494 inner.serial_invoke(alpha, A, B, m, C);
495 break;
496 }
497 case 35: {
498 InnerGemmFixB<3, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
499 inner.serial_invoke(alpha, A, B, m, C);
500 break;
501 }
502 case 25: {
503 InnerGemmFixB<2, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
504 inner.serial_invoke(alpha, A, B, m, C);
505 break;
506 }
507 case 15: {
508 InnerGemmFixB<1, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
509 inner.serial_invoke(alpha, A, B, m, C);
510 break;
511 }
512 }
513
514 return 0;
515 }
516
517 ///
518 /// Inner kernel (4x4)
519 /// ==================
520
521 template <>
522 template <typename ScalarType, typename ValueType>
523 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<4, 4>::serial_invoke(const ScalarType alpha,
524 const ValueType *KOKKOS_RESTRICT A,
525 const ValueType *KOKKOS_RESTRICT B, const int m,
526 /**/ ValueType *KOKKOS_RESTRICT C) {
527 if (m <= 0) return 0;
528
529 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
530 b_03 = B[0 * _bs0 + 3 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1], b_11 = B[1 * _bs0 + 1 * _bs1],
531 b_12 = B[1 * _bs0 + 2 * _bs1], b_13 = B[1 * _bs0 + 3 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1],
532 b_21 = B[2 * _bs0 + 1 * _bs1], b_22 = B[2 * _bs0 + 2 * _bs1], b_23 = B[2 * _bs0 + 3 * _bs1],
533 b_30 = B[3 * _bs0 + 0 * _bs1], b_31 = B[3 * _bs0 + 1 * _bs1], b_32 = B[3 * _bs0 + 2 * _bs1],
534 b_33 = B[3 * _bs0 + 3 * _bs1];
535
536 ValueType a_p0, a_p1, a_p2, a_p3, c_p0, c_p1, c_p2, c_p3;
537
538 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1,
539 jc2 = 2 * _cs1, jc3 = 3 * _cs1;
540
541 for (int p = 0; p < m; ++p) {
542 a_p0 = A[p * _bs0 + ja0];
543 a_p1 = A[p * _bs0 + ja1];
544 a_p2 = A[p * _bs0 + ja2];
545 a_p3 = A[p * _bs0 + ja3];
546
547 c_p0 = a_p0 * b_00;
548 c_p1 = a_p0 * b_01;
549 c_p2 = a_p0 * b_02;
550 c_p3 = a_p0 * b_03;
551 c_p0 += a_p1 * b_10;
552 c_p1 += a_p1 * b_11;
553 c_p2 += a_p1 * b_12;
554 c_p3 += a_p1 * b_13;
555 c_p0 += a_p2 * b_20;
556 c_p1 += a_p2 * b_21;
557 c_p2 += a_p2 * b_22;
558 c_p3 += a_p2 * b_23;
559 c_p0 += a_p3 * b_30;
560 c_p1 += a_p3 * b_31;
561 c_p2 += a_p3 * b_32;
562 c_p3 += a_p3 * b_33;
563
564 C[p * _cs0 + jc0] += alpha * c_p0;
565 C[p * _cs0 + jc1] += alpha * c_p1;
566 C[p * _cs0 + jc2] += alpha * c_p2;
567 C[p * _cs0 + jc3] += alpha * c_p3;
568 }
569
570 return 0;
571 }
572
573 template <>
574 template <typename ScalarType, typename ValueType>
575 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<4, 3>::serial_invoke(const ScalarType alpha,
576 const ValueType *KOKKOS_RESTRICT A,
577 const ValueType *KOKKOS_RESTRICT B, const int m,
578 /**/ ValueType *KOKKOS_RESTRICT C) {
579 if (m <= 0) return 0;
580
581 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
582 b_10 = B[1 * _bs0 + 0 * _bs1], b_11 = B[1 * _bs0 + 1 * _bs1], b_12 = B[1 * _bs0 + 2 * _bs1],
583 b_20 = B[2 * _bs0 + 0 * _bs1], b_21 = B[2 * _bs0 + 1 * _bs1], b_22 = B[2 * _bs0 + 2 * _bs1],
584 b_30 = B[3 * _bs0 + 0 * _bs1], b_31 = B[3 * _bs0 + 1 * _bs1], b_32 = B[3 * _bs0 + 2 * _bs1];
585
586 ValueType a_p0, a_p1, a_p2, a_p3, c_p0, c_p1, c_p2;
587
588 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1,
589 jc2 = 2 * _cs1;
590
591 for (int p = 0; p < m; ++p) {
592 a_p0 = A[p * _bs0 + ja0];
593 a_p1 = A[p * _bs0 + ja1];
594 a_p2 = A[p * _bs0 + ja2];
595 a_p3 = A[p * _bs0 + ja3];
596
597 c_p0 = a_p0 * b_00;
598 c_p1 = a_p0 * b_01;
599 c_p2 = a_p0 * b_02;
600 c_p0 += a_p1 * b_10;
601 c_p1 += a_p1 * b_11;
602 c_p2 += a_p1 * b_12;
603 c_p0 += a_p2 * b_20;
604 c_p1 += a_p2 * b_21;
605 c_p2 += a_p2 * b_22;
606 c_p0 += a_p3 * b_30;
607 c_p1 += a_p3 * b_31;
608 c_p2 += a_p3 * b_32;
609
610 C[p * _cs0 + jc0] += alpha * c_p0;
611 C[p * _cs0 + jc1] += alpha * c_p1;
612 C[p * _cs0 + jc2] += alpha * c_p2;
613 }
614
615 return 0;
616 }
617
618 template <>
619 template <typename ScalarType, typename ValueType>
620 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<4, 2>::serial_invoke(const ScalarType alpha,
621 const ValueType *KOKKOS_RESTRICT A,
622 const ValueType *KOKKOS_RESTRICT B, const int m,
623 /**/ ValueType *KOKKOS_RESTRICT C) {
624 if (m <= 0) return 0;
625
626 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1],
627 b_11 = B[1 * _bs0 + 1 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1], b_21 = B[2 * _bs0 + 1 * _bs1],
628 b_30 = B[3 * _bs0 + 0 * _bs1], b_31 = B[3 * _bs0 + 1 * _bs1];
629
630 ValueType a_p0, a_p1, a_p2, a_p3, c_p0, c_p1;
631
632 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1;
633
634 for (int p = 0; p < m; ++p) {
635 a_p0 = A[p * _bs0 + ja0];
636 a_p1 = A[p * _bs0 + ja1];
637 a_p2 = A[p * _bs0 + ja2];
638 a_p3 = A[p * _bs0 + ja3];
639
640 c_p0 = a_p0 * b_00;
641 c_p1 = a_p0 * b_01;
642 c_p0 += a_p1 * b_10;
643 c_p1 += a_p1 * b_11;
644 c_p0 += a_p2 * b_20;
645 c_p1 += a_p2 * b_21;
646 c_p0 += a_p3 * b_30;
647 c_p1 += a_p3 * b_31;
648
649 C[p * _cs0 + jc0] += alpha * c_p0;
650 C[p * _cs0 + jc1] += alpha * c_p1;
651 }
652
653 return 0;
654 }
655
656 template <>
657 template <typename ScalarType, typename ValueType>
658 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<4, 1>::serial_invoke(const ScalarType alpha,
659 const ValueType *KOKKOS_RESTRICT A,
660 const ValueType *KOKKOS_RESTRICT B, const int m,
661 /**/ ValueType *KOKKOS_RESTRICT C) {
662 if (m <= 0) return 0;
663
664 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1],
665 b_30 = B[3 * _bs0 + 0 * _bs1];
666
667 ValueType a_p0, a_p1, a_p2, a_p3, c_p0;
668
669 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, ja3 = 3 * _as1, jc0 = 0 * _cs1;
670
671 for (int p = 0; p < m; ++p) {
672 a_p0 = A[p * _bs0 + ja0];
673 a_p1 = A[p * _bs0 + ja1];
674 a_p2 = A[p * _bs0 + ja2];
675 a_p3 = A[p * _bs0 + ja3];
676
677 c_p0 = a_p0 * b_00;
678 c_p0 += a_p1 * b_10;
679 c_p0 += a_p2 * b_20;
680 c_p0 += a_p3 * b_30;
681
682 C[p * _cs0 + jc0] += alpha * c_p0;
683 }
684
685 return 0;
686 }
687
688 template <>
689 template <typename ScalarType, typename ValueType>
690 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<3, 4>::serial_invoke(const ScalarType alpha,
691 const ValueType *KOKKOS_RESTRICT A,
692 const ValueType *KOKKOS_RESTRICT B, const int m,
693 /**/ ValueType *KOKKOS_RESTRICT C) {
694 if (m <= 0) return 0;
695
696 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
697 b_03 = B[0 * _bs0 + 3 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1], b_11 = B[1 * _bs0 + 1 * _bs1],
698 b_12 = B[1 * _bs0 + 2 * _bs1], b_13 = B[1 * _bs0 + 3 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1],
699 b_21 = B[2 * _bs0 + 1 * _bs1], b_22 = B[2 * _bs0 + 2 * _bs1], b_23 = B[2 * _bs0 + 3 * _bs1];
700
701 ValueType a_p0, a_p1, a_p2, c_p0, c_p1, c_p2, c_p3;
702
703 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1, jc2 = 2 * _cs1,
704 jc3 = 3 * _cs1;
705
706 for (int p = 0; p < m; ++p) {
707 a_p0 = A[p * _bs0 + ja0];
708 a_p1 = A[p * _bs0 + ja1];
709 a_p2 = A[p * _bs0 + ja2];
710
711 c_p0 = a_p0 * b_00;
712 c_p1 = a_p0 * b_01;
713 c_p2 = a_p0 * b_02;
714 c_p3 = a_p0 * b_03;
715 c_p0 += a_p1 * b_10;
716 c_p1 += a_p1 * b_11;
717 c_p2 += a_p1 * b_12;
718 c_p3 += a_p1 * b_13;
719 c_p0 += a_p2 * b_20;
720 c_p1 += a_p2 * b_21;
721 c_p2 += a_p2 * b_22;
722 c_p3 += a_p2 * b_23;
723
724 C[p * _cs0 + jc0] += alpha * c_p0;
725 C[p * _cs0 + jc1] += alpha * c_p1;
726 C[p * _cs0 + jc2] += alpha * c_p2;
727 C[p * _cs0 + jc3] += alpha * c_p3;
728 }
729
730 return 0;
731 }
732
733 template <>
734 template <typename ScalarType, typename ValueType>
735 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<2, 4>::serial_invoke(const ScalarType alpha,
736 const ValueType *KOKKOS_RESTRICT A,
737 const ValueType *KOKKOS_RESTRICT B, const int m,
738 /**/ ValueType *KOKKOS_RESTRICT C) {
739 if (m <= 0) return 0;
740
741 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
742 b_03 = B[0 * _bs0 + 3 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1], b_11 = B[1 * _bs0 + 1 * _bs1],
743 b_12 = B[1 * _bs0 + 2 * _bs1], b_13 = B[1 * _bs0 + 3 * _bs1];
744
745 ValueType a_p0, a_p1, c_p0, c_p1, c_p2, c_p3;
746
747 const int ja0 = 0 * _as1, ja1 = 1 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1, jc2 = 2 * _cs1, jc3 = 3 * _cs1;
748
749 for (int p = 0; p < m; ++p) {
750 a_p0 = A[p * _bs0 + ja0];
751 a_p1 = A[p * _bs0 + ja1];
752
753 c_p0 = a_p0 * b_00;
754 c_p1 = a_p0 * b_01;
755 c_p2 = a_p0 * b_02;
756 c_p3 = a_p0 * b_03;
757 c_p0 += a_p1 * b_10;
758 c_p1 += a_p1 * b_11;
759 c_p2 += a_p1 * b_12;
760 c_p3 += a_p1 * b_13;
761
762 C[p * _cs0 + jc0] += alpha * c_p0;
763 C[p * _cs0 + jc1] += alpha * c_p1;
764 C[p * _cs0 + jc2] += alpha * c_p2;
765 C[p * _cs0 + jc3] += alpha * c_p3;
766 }
767
768 return 0;
769 }
770
771 template <>
772 template <typename ScalarType, typename ValueType>
773 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<1, 4>::serial_invoke(const ScalarType alpha,
774 const ValueType *KOKKOS_RESTRICT A,
775 const ValueType *KOKKOS_RESTRICT B, const int m,
776 /**/ ValueType *KOKKOS_RESTRICT C) {
777 if (m <= 0) return 0;
778
779 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
780 b_03 = B[0 * _bs0 + 3 * _bs1];
781
782 ValueType a_p0, c_p0, c_p1, c_p2, c_p3;
783
784 const int ja0 = 0 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1, jc2 = 2 * _cs1, jc3 = 3 * _cs1;
785
786 for (int p = 0; p < m; ++p) {
787 a_p0 = A[p * _bs0 + ja0];
788
789 c_p0 = a_p0 * b_00;
790 c_p1 = a_p0 * b_01;
791 c_p2 = a_p0 * b_02;
792 c_p3 = a_p0 * b_03;
793
794 C[p * _cs0 + jc0] += alpha * c_p0;
795 C[p * _cs0 + jc1] += alpha * c_p1;
796 C[p * _cs0 + jc2] += alpha * c_p2;
797 C[p * _cs0 + jc3] += alpha * c_p3;
798 }
799
800 return 0;
801 }
802
803 template <>
804 template <typename ScalarType, typename ValueType>
805 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<4, 4>::serial_invoke(const ScalarType alpha,
806 const ValueType *KOKKOS_RESTRICT A,
807 const ValueType *KOKKOS_RESTRICT B, const int m,
808 const int n, const int k,
809 /**/ ValueType *KOKKOS_RESTRICT C) {
810 if (m <= 0 || n <= 0 || k <= 0) return 0;
811
812 switch (k * 10 + n) {
813 case 43: {
814 InnerGemmFixB<4, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
815 inner.serial_invoke(alpha, A, B, m, C);
816 break;
817 }
818 case 42: {
819 InnerGemmFixB<4, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
820 inner.serial_invoke(alpha, A, B, m, C);
821 break;
822 }
823 case 41: {
824 InnerGemmFixB<4, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
825 inner.serial_invoke(alpha, A, B, m, C);
826 break;
827 }
828 case 34: {
829 InnerGemmFixB<3, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
830 inner.serial_invoke(alpha, A, B, m, C);
831 break;
832 }
833 case 24: {
834 InnerGemmFixB<2, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
835 inner.serial_invoke(alpha, A, B, m, C);
836 break;
837 }
838 case 14: {
839 InnerGemmFixB<1, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
840 inner.serial_invoke(alpha, A, B, m, C);
841 break;
842 }
843 }
844
845 return 0;
846 }
847
848 ///
849 /// Inner kernel (3x3)
850 /// ==================
851
852 template <>
853 template <typename ScalarType, typename ValueType>
854 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<3, 3>::serial_invoke(const ScalarType alpha,
855 const ValueType *KOKKOS_RESTRICT A,
856 const ValueType *KOKKOS_RESTRICT B, const int m,
857 /**/ ValueType *KOKKOS_RESTRICT C) {
858 if (m <= 0) return 0;
859
860 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
861 b_10 = B[1 * _bs0 + 0 * _bs1], b_11 = B[1 * _bs0 + 1 * _bs1], b_12 = B[1 * _bs0 + 2 * _bs1],
862 b_20 = B[2 * _bs0 + 0 * _bs1], b_21 = B[2 * _bs0 + 1 * _bs1], b_22 = B[2 * _bs0 + 2 * _bs1];
863
864 ValueType a_p0, a_p1, a_p2, c_p0, c_p1, c_p2;
865
866 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1, jc2 = 2 * _cs1;
867
868 for (int p = 0; p < m; ++p) {
869 a_p0 = A[p * _bs0 + ja0];
870 a_p1 = A[p * _bs0 + ja1];
871 a_p2 = A[p * _bs0 + ja2];
872
873 c_p0 = a_p0 * b_00;
874 c_p1 = a_p0 * b_01;
875 c_p2 = a_p0 * b_02;
876 c_p0 += a_p1 * b_10;
877 c_p1 += a_p1 * b_11;
878 c_p2 += a_p1 * b_12;
879 c_p0 += a_p2 * b_20;
880 c_p1 += a_p2 * b_21;
881 c_p2 += a_p2 * b_22;
882
883 C[p * _cs0 + jc0] += alpha * c_p0;
884 C[p * _cs0 + jc1] += alpha * c_p1;
885 C[p * _cs0 + jc2] += alpha * c_p2;
886 }
887
888 return 0;
889 }
890
891 template <>
892 template <typename ScalarType, typename ValueType>
893 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<3, 2>::serial_invoke(const ScalarType alpha,
894 const ValueType *KOKKOS_RESTRICT A,
895 const ValueType *KOKKOS_RESTRICT B, const int m,
896 /**/ ValueType *KOKKOS_RESTRICT C) {
897 if (m <= 0) return 0;
898
899 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1],
900 b_11 = B[1 * _bs0 + 1 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1], b_21 = B[2 * _bs0 + 1 * _bs1];
901
902 ValueType a_p0, a_p1, a_p2, c_p0, c_p1;
903
904 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1;
905
906 for (int p = 0; p < m; ++p) {
907 a_p0 = A[p * _bs0 + ja0];
908 a_p1 = A[p * _bs0 + ja1];
909 a_p2 = A[p * _bs0 + ja2];
910
911 c_p0 = a_p0 * b_00;
912 c_p1 = a_p0 * b_01;
913 c_p0 += a_p1 * b_10;
914 c_p1 += a_p1 * b_11;
915 c_p0 += a_p2 * b_20;
916 c_p1 += a_p2 * b_21;
917
918 C[p * _cs0 + jc0] += alpha * c_p0;
919 C[p * _cs0 + jc1] += alpha * c_p1;
920 }
921
922 return 0;
923 }
924
925 template <>
926 template <typename ScalarType, typename ValueType>
927 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<3, 1>::serial_invoke(const ScalarType alpha,
928 const ValueType *KOKKOS_RESTRICT A,
929 const ValueType *KOKKOS_RESTRICT B, const int m,
930 /**/ ValueType *KOKKOS_RESTRICT C) {
931 if (m <= 0) return 0;
932
933 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1], b_20 = B[2 * _bs0 + 0 * _bs1];
934
935 ValueType a_p0, a_p1, a_p2, c_p0;
936
937 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, jc0 = 0 * _cs1;
938
939 for (int p = 0; p < m; ++p) {
940 a_p0 = A[p * _bs0 + ja0];
941 a_p1 = A[p * _bs0 + ja1];
942 a_p2 = A[p * _bs0 + ja2];
943
944 c_p0 = a_p0 * b_00;
945 c_p0 += a_p1 * b_10;
946 c_p0 += a_p2 * b_20;
947
948 C[p * _cs0 + jc0] += alpha * c_p0;
949 }
950
951 return 0;
952 }
953
954 template <>
955 template <typename ScalarType, typename ValueType>
956 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<2, 3>::serial_invoke(const ScalarType alpha,
957 const ValueType *KOKKOS_RESTRICT A,
958 const ValueType *KOKKOS_RESTRICT B, const int m,
959 /**/ ValueType *KOKKOS_RESTRICT C) {
960 if (m <= 0) return 0;
961
962 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_02 = B[0 * _bs0 + 2 * _bs1],
963 b_10 = B[1 * _bs0 + 0 * _bs1], b_11 = B[1 * _bs0 + 1 * _bs1], b_12 = B[1 * _bs0 + 2 * _bs1];
964
965 ValueType a_p0, a_p1, a_p2, c_p0, c_p1;
966
967 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1;
968
969 for (int p = 0; p < m; ++p) {
970 a_p0 = A[p * _bs0 + ja0];
971 a_p1 = A[p * _bs0 + ja1];
972
973 c_p0 = a_p0 * b_00;
974 c_p1 = a_p0 * b_01;
975 c_p2 = a_p0 * b_02;
976 c_p0 += a_p1 * b_10;
977 c_p1 += a_p1 * b_11;
978 c_p2 += a_p1 * b_12;
979
980 C[p * _cs0 + jc0] += alpha * c_p0;
981 C[p * _cs0 + jc1] += alpha * c_p1;
982 C[p * _cs0 + jc2] += alpha * c_p2;
983 }
984
985 return 0;
986 }
987 template <>
988 template <typename ScalarType, typename ValueType>
989 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<1, 3>::serial_invoke(const ScalarType alpha,
990 const ValueType *KOKKOS_RESTRICT A,
991 const ValueType *KOKKOS_RESTRICT B, const int m,
992 /**/ ValueType *KOKKOS_RESTRICT C) {
993 if (m <= 0) return 0;
994
995 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1],
996 b_11 = B[1 * _bs0 + 1 * _bs1];
997
998 ValueType a_p0, a_p1, a_p2, c_p0;
999
1000 const int ja0 = 0 * _as1, ja1 = 1 * _as1, ja2 = 2 * _as1, jc0 = 0 * _cs1;
1001
1002 for (int p = 0; p < m; ++p) {
1003 a_p0 = A[p * _bs0 + ja0];
1004
1005 c_p0 = a_p0 * b_00;
1006 c_p1 = a_p0 * b_01;
1007 c_p2 = a_p0 * b_02;
1008
1009 C[p * _cs0 + jc0] += alpha * c_p0;
1010 C[p * _cs0 + jc1] += alpha * c_p1;
1011 C[p * _cs0 + jc2] += alpha * c_p2;
1012 }
1013
1014 return 0;
1015 }
1016
1017 template <>
1018 template <typename ScalarType, typename ValueType>
1019 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<3, 3>::serial_invoke(const ScalarType alpha,
1020 const ValueType *KOKKOS_RESTRICT A,
1021 const ValueType *KOKKOS_RESTRICT B, const int m,
1022 const int n, const int k,
1023 /**/ ValueType *KOKKOS_RESTRICT C) {
1024 if (m <= 0 || n <= 0 || k <= 0) return 0;
1025
1026 switch (k * 10 + n) {
1027 case 32: {
1028 InnerGemmFixB<3, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1029 inner.serial_invoke(alpha, A, B, m, C);
1030 break;
1031 }
1032 case 31: {
1033 InnerGemmFixB<3, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1034 inner.serial_invoke(alpha, A, B, m, C);
1035 break;
1036 }
1037 case 23: {
1038 InnerGemmFixB<2, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1039 inner.serial_invoke(alpha, A, B, m, C);
1040 break;
1041 }
1042 case 13: {
1043 InnerGemmFixB<1, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1044 inner.serial_invoke(alpha, A, B, m, C);
1045 break;
1046 }
1047 }
1048
1049 return 0;
1050 }
1051
1052 ///
1053 /// Inner kernel (2x2)
1054 /// ==================
1055
1056 template <>
1057 template <typename ScalarType, typename ValueType>
1058 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<2, 2>::serial_invoke(const ScalarType alpha,
1059 const ValueType *KOKKOS_RESTRICT A,
1060 const ValueType *KOKKOS_RESTRICT B, const int m,
1061 /**/ ValueType *KOKKOS_RESTRICT C) {
1062 if (m <= 0) return 0;
1063
1064 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1],
1065 b_11 = B[1 * _bs0 + 1 * _bs1];
1066
1067 ValueType a_p0, a_p1, c_p0, c_p1;
1068
1069 const int ja0 = 0 * _as1, ja1 = 1 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1;
1070
1071 for (int p = 0; p < m; ++p) {
1072 a_p0 = A[p * _bs0 + ja0];
1073 a_p1 = A[p * _bs0 + ja1];
1074
1075 c_p0 = a_p0 * b_00;
1076 c_p1 = a_p0 * b_01;
1077 c_p0 += a_p1 * b_10;
1078 c_p1 += a_p1 * b_11;
1079
1080 C[p * _cs0 + jc0] += alpha * c_p0;
1081 C[p * _cs0 + jc1] += alpha * c_p1;
1082 }
1083
1084 return 0;
1085 }
1086
1087 template <>
1088 template <typename ScalarType, typename ValueType>
1089 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<2, 1>::serial_invoke(const ScalarType alpha,
1090 const ValueType *KOKKOS_RESTRICT A,
1091 const ValueType *KOKKOS_RESTRICT B, const int m,
1092 /**/ ValueType *KOKKOS_RESTRICT C) {
1093 if (m <= 0) return 0;
1094
1095 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_10 = B[1 * _bs0 + 0 * _bs1];
1096
1097 ValueType a_p0, a_p1, c_p0;
1098
1099 const int ja0 = 0 * _as1, ja1 = 1 * _as1, jc0 = 0 * _cs1;
1100
1101 for (int p = 0; p < m; ++p) {
1102 a_p0 = A[p * _bs0 + ja0];
1103 a_p1 = A[p * _bs0 + ja1];
1104
1105 c_p0 = a_p0 * b_00;
1106 c_p0 += a_p1 * b_10;
1107
1108 C[p * _cs0 + jc0] += alpha * c_p0;
1109 }
1110
1111 return 0;
1112 }
1113
1114 template <>
1115 template <typename ScalarType, typename ValueType>
1116 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<1, 2>::serial_invoke(const ScalarType alpha,
1117 const ValueType *KOKKOS_RESTRICT A,
1118 const ValueType *KOKKOS_RESTRICT B, const int m,
1119 /**/ ValueType *KOKKOS_RESTRICT C) {
1120 if (m <= 0) return 0;
1121
1122 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1], b_01 = B[0 * _bs0 + 1 * _bs1];
1123
1124 ValueType a_p0, c_p0, c_p1;
1125
1126 const int ja0 = 0 * _as1, jc0 = 0 * _cs1, jc1 = 1 * _cs1;
1127
1128 for (int p = 0; p < m; ++p) {
1129 a_p0 = A[p * _bs0 + ja0];
1130
1131 c_p0 = a_p0 * b_00;
1132 c_p1 = a_p0 * b_01;
1133
1134 C[p * _cs0 + jc0] += alpha * c_p0;
1135 C[p * _cs0 + jc1] += alpha * c_p1;
1136 }
1137
1138 return 0;
1139 }
1140
1141 template <>
1142 template <typename ScalarType, typename ValueType>
1143 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<2, 2>::serial_invoke(const ScalarType alpha,
1144 const ValueType *KOKKOS_RESTRICT A,
1145 const ValueType *KOKKOS_RESTRICT B, const int m,
1146 const int n, const int k,
1147 /**/ ValueType *KOKKOS_RESTRICT C) {
1148 if (m <= 0 || n <= 0 || k <= 0) return 0;
1149
1150 switch (k * 10 + n) {
1151 case 21: {
1152 InnerGemmFixB<2, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1153 inner.serial_invoke(alpha, A, B, m, C);
1154 break;
1155 }
1156 case 12: {
1157 InnerGemmFixB<1, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1158 inner.serial_invoke(alpha, A, B, m, C);
1159 break;
1160 }
1161 }
1162
1163 return 0;
1164 }
1165
1166 ///
1167 /// Inner kernel (1x1)
1168 /// ==================
1169
1170 template <>
1171 template <typename ScalarType, typename ValueType>
1172 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<1, 1>::serial_invoke(const ScalarType alpha,
1173 const ValueType *KOKKOS_RESTRICT A,
1174 const ValueType *KOKKOS_RESTRICT B, const int m,
1175 /**/ ValueType *KOKKOS_RESTRICT C) {
1176 if (m <= 0) return 0;
1177
1178 const ValueType b_00 = B[0 * _bs0 + 0 * _bs1];
1179
1180 ValueType a_p0, c_p0;
1181
1182 const int ja0 = 0 * _as1, jc0 = 0 * _cs1;
1183
1184 for (int p = 0; p < m; ++p) {
1185 a_p0 = A[p * _bs0 + ja0];
1186
1187 c_p0 = a_p0 * b_00;
1188
1189 C[p * _cs0 + jc0] += alpha * c_p0;
1190 }
1191
1192 return 0;
1193 }
1194
1195 ///
1196 /// Inner kernel (remainders)
1197 /// =========================
1198
1199 template <>
1200 template <typename ScalarType, typename ValueType>
1201 KOKKOS_INLINE_FUNCTION int InnerGemmFixB<0, 0>::serial_invoke(const ScalarType alpha,
1202 const ValueType *KOKKOS_RESTRICT A,
1203 const ValueType *KOKKOS_RESTRICT B, const int m,
1204 const int n, const int k,
1205 /**/ ValueType *KOKKOS_RESTRICT C) {
1206 if (m <= 0 || n <= 0 || k <= 0) return 0;
1207
1208 if (k == n) {
1209 switch (k) {
1210 case 5: {
1211 InnerGemmFixB<5, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1212 inner.serial_invoke(alpha, A, B, m, C);
1213 break;
1214 }
1215 case 4: {
1216 InnerGemmFixB<4, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1217 inner.serial_invoke(alpha, A, B, m, C);
1218 break;
1219 }
1220 case 3: {
1221 InnerGemmFixB<3, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1222 inner.serial_invoke(alpha, A, B, m, C);
1223 break;
1224 }
1225 case 2: {
1226 InnerGemmFixB<2, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1227 inner.serial_invoke(alpha, A, B, m, C);
1228 break;
1229 }
1230 case 1: {
1231 InnerGemmFixB<1, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1232 inner.serial_invoke(alpha, A, B, m, C);
1233 break;
1234 }
1235 }
1236 } else {
1237 for (int i = 0; i < m; ++i) {
1238 const ValueType *KOKKOS_RESTRICT iA = A + i * _as0;
1239 /**/ ValueType *KOKKOS_RESTRICT iC = C + i * _cs0;
1240 for (int j = 0; j < n; ++j) {
1241 const ValueType *KOKKOS_RESTRICT jB = B + j * _bs1;
1242 /**/ ValueType tC = 0;
1243 for (int p = 0; p < k; ++p) tC += iA[p * _as1] * jB[p * _bs0];
1244 pC[i * _cs0] += alpha * tC;
1245 }
1246 }
1247 }
1248 return 0;
1249 }
1250 } // namespace KokkosBatched
1251
1252 #endif