Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_InnerGemmFixC_Serial_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_INNER_GEMM_FIX_C_SERIAL_IMPL_HPP
17 #define KOKKOSBATCHED_INNER_GEMM_FIX_C_SERIAL_IMPL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosBatched_InnerGemmFixC_Decl.hpp"
23
24 namespace KokkosBatched {
25
26 ///
27 /// Inner kernel (5x5)
28 /// ==================
29
30 template <>
31 template <typename ScalarType, typename ValueType>
32 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<5, 5>::serial_invoke(const ScalarType alpha,
33 const ValueType *KOKKOS_RESTRICT A,
34 const ValueType *KOKKOS_RESTRICT B, const int k,
35 /**/ ValueType *KOKKOS_RESTRICT C) {
36 if (k <= 0) return 0;
37
38 ValueType a_0p, b_p0, c_00 = 0, c_01 = 0, c_02 = 0, c_03 = 0, c_04 = 0, a_1p, b_p1, c_10 = 0, c_11 = 0, c_12 = 0,
39 c_13 = 0, c_14 = 0, a_2p, b_p2, c_20 = 0, c_21 = 0, c_22 = 0, c_23 = 0, c_24 = 0, a_3p, b_p3,
40 c_30 = 0, c_31 = 0, c_32 = 0, c_33 = 0, c_34 = 0, a_4p, b_p4, c_40 = 0, c_41 = 0, c_42 = 0,
41 c_43 = 0, c_44 = 0;
42
43 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, i4 = 4 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1,
44 j2 = 2 * _bs1, j3 = 3 * _bs1, j4 = 4 * _bs1;
45
46 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
47 #pragma unroll
48 #endif
49 for (int p = 0; p < k; ++p) {
50 a_0p = A[i0 + p * _as1];
51 b_p0 = B[p * _bs0 + j0];
52 a_1p = A[i1 + p * _as1];
53 b_p1 = B[p * _bs0 + j1];
54 a_2p = A[i2 + p * _as1];
55 b_p2 = B[p * _bs0 + j2];
56 a_3p = A[i3 + p * _as1];
57 b_p3 = B[p * _bs0 + j3];
58 a_4p = A[i4 + p * _as1];
59 b_p4 = B[p * _bs0 + j4];
60
61 c_00 += a_0p * b_p0;
62 c_01 += a_0p * b_p1;
63 c_02 += a_0p * b_p2;
64 c_03 += a_0p * b_p3;
65 c_04 += a_0p * b_p4;
66 c_10 += a_1p * b_p0;
67 c_11 += a_1p * b_p1;
68 c_12 += a_1p * b_p2;
69 c_13 += a_1p * b_p3;
70 c_14 += a_1p * b_p4;
71 c_20 += a_2p * b_p0;
72 c_21 += a_2p * b_p1;
73 c_22 += a_2p * b_p2;
74 c_23 += a_2p * b_p3;
75 c_24 += a_2p * b_p4;
76 c_30 += a_3p * b_p0;
77 c_31 += a_3p * b_p1;
78 c_32 += a_3p * b_p2;
79 c_33 += a_3p * b_p3;
80 c_34 += a_3p * b_p4;
81 c_40 += a_4p * b_p0;
82 c_41 += a_4p * b_p1;
83 c_42 += a_4p * b_p2;
84 c_43 += a_4p * b_p3;
85 c_44 += a_4p * b_p4;
86 }
87
88 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
89 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
90 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
91 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
92 C[0 * _cs0 + 4 * _cs1] += alpha * c_04;
93 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
94 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
95 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
96 C[1 * _cs0 + 3 * _cs1] += alpha * c_13;
97 C[1 * _cs0 + 4 * _cs1] += alpha * c_14;
98 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
99 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
100 C[2 * _cs0 + 2 * _cs1] += alpha * c_22;
101 C[2 * _cs0 + 3 * _cs1] += alpha * c_23;
102 C[2 * _cs0 + 4 * _cs1] += alpha * c_24;
103 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
104 C[3 * _cs0 + 1 * _cs1] += alpha * c_31;
105 C[3 * _cs0 + 2 * _cs1] += alpha * c_32;
106 C[3 * _cs0 + 3 * _cs1] += alpha * c_33;
107 C[3 * _cs0 + 4 * _cs1] += alpha * c_34;
108 C[4 * _cs0 + 0 * _cs1] += alpha * c_40;
109 C[4 * _cs0 + 1 * _cs1] += alpha * c_41;
110 C[4 * _cs0 + 2 * _cs1] += alpha * c_42;
111 C[4 * _cs0 + 3 * _cs1] += alpha * c_43;
112 C[4 * _cs0 + 4 * _cs1] += alpha * c_44;
113
114 return 0;
115 }
116
117 template <>
118 template <typename ScalarType, typename ValueType>
119 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<5, 4>::serial_invoke(const ScalarType alpha,
120 const ValueType *KOKKOS_RESTRICT A,
121 const ValueType *KOKKOS_RESTRICT B, const int k,
122 /**/ ValueType *KOKKOS_RESTRICT C) {
123 if (k <= 0) return 0;
124
125 ValueType a_0p, b_p0, c_00 = 0, c_01 = 0, c_02 = 0, c_03 = 0, a_1p, b_p1, c_10 = 0, c_11 = 0, c_12 = 0, c_13 = 0,
126 a_2p, b_p2, c_20 = 0, c_21 = 0, c_22 = 0, c_23 = 0, a_3p, b_p3, c_30 = 0, c_31 = 0, c_32 = 0,
127 c_33 = 0, a_4p, c_40 = 0, c_41 = 0, c_42 = 0, c_43 = 0;
128
129 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, i4 = 4 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1,
130 j2 = 2 * _bs1, j3 = 3 * _bs1;
131
132 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
133 #pragma unroll
134 #endif
135 for (int p = 0; p < k; ++p) {
136 a_0p = A[i0 + p * _as1];
137 b_p0 = B[p * _bs0 + j0];
138 a_1p = A[i1 + p * _as1];
139 b_p1 = B[p * _bs0 + j1];
140 a_2p = A[i2 + p * _as1];
141 b_p2 = B[p * _bs0 + j2];
142 a_3p = A[i3 + p * _as1];
143 b_p3 = B[p * _bs0 + j3];
144 a_4p = A[i4 + p * _as1];
145
146 c_00 += a_0p * b_p0;
147 c_01 += a_0p * b_p1;
148 c_02 += a_0p * b_p2;
149 c_03 += a_0p * b_p3;
150 c_10 += a_1p * b_p0;
151 c_11 += a_1p * b_p1;
152 c_12 += a_1p * b_p2;
153 c_13 += a_1p * b_p3;
154 c_20 += a_2p * b_p0;
155 c_21 += a_2p * b_p1;
156 c_22 += a_2p * b_p2;
157 c_23 += a_2p * b_p3;
158 c_30 += a_3p * b_p0;
159 c_31 += a_3p * b_p1;
160 c_32 += a_3p * b_p2;
161 c_33 += a_3p * b_p3;
162 c_40 += a_4p * b_p0;
163 c_41 += a_4p * b_p1;
164 c_42 += a_4p * b_p2;
165 c_43 += a_4p * b_p3;
166 }
167
168 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
169 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
170 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
171 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
172 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
173 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
174 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
175 C[1 * _cs0 + 3 * _cs1] += alpha * c_13;
176 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
177 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
178 C[2 * _cs0 + 2 * _cs1] += alpha * c_22;
179 C[2 * _cs0 + 3 * _cs1] += alpha * c_23;
180 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
181 C[3 * _cs0 + 1 * _cs1] += alpha * c_31;
182 C[3 * _cs0 + 2 * _cs1] += alpha * c_32;
183 C[3 * _cs0 + 3 * _cs1] += alpha * c_33;
184 C[4 * _cs0 + 0 * _cs1] += alpha * c_40;
185 C[4 * _cs0 + 1 * _cs1] += alpha * c_41;
186 C[4 * _cs0 + 2 * _cs1] += alpha * c_42;
187 C[4 * _cs0 + 3 * _cs1] += alpha * c_43;
188
189 return 0;
190 }
191
192 template <>
193 template <typename ScalarType, typename ValueType>
194 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<5, 3>::serial_invoke(const ScalarType alpha,
195 const ValueType *KOKKOS_RESTRICT A,
196 const ValueType *KOKKOS_RESTRICT B, const int k,
197 /**/ ValueType *KOKKOS_RESTRICT C) {
198 if (k <= 0) return 0;
199
200 ValueType a_0p, b_p0, c_00 = 0, c_01 = 0, c_02 = 0, a_1p, b_p1, c_10 = 0, c_11 = 0, c_12 = 0, a_2p, b_p2, c_20 = 0,
201 c_21 = 0, c_22 = 0, a_3p, c_30 = 0, c_31 = 0, c_32 = 0, a_4p, c_40 = 0, c_41 = 0, c_42 = 0;
202
203 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, i4 = 4 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1,
204 j2 = 2 * _bs1;
205
206 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
207 #pragma unroll
208 #endif
209 for (int p = 0; p < k; ++p) {
210 a_0p = A[i0 + p * _as1];
211 b_p0 = B[p * _bs0 + j0];
212 a_1p = A[i1 + p * _as1];
213 b_p1 = B[p * _bs0 + j1];
214 a_2p = A[i2 + p * _as1];
215 b_p2 = B[p * _bs0 + j2];
216 a_3p = A[i3 + p * _as1];
217 a_4p = A[i4 + p * _as1];
218
219 c_00 += a_0p * b_p0;
220 c_01 += a_0p * b_p1;
221 c_02 += a_0p * b_p2;
222 c_10 += a_1p * b_p0;
223 c_11 += a_1p * b_p1;
224 c_12 += a_1p * b_p2;
225 c_20 += a_2p * b_p0;
226 c_21 += a_2p * b_p1;
227 c_22 += a_2p * b_p2;
228 c_30 += a_3p * b_p0;
229 c_31 += a_3p * b_p1;
230 c_32 += a_3p * b_p2;
231 c_40 += a_4p * b_p0;
232 c_41 += a_4p * b_p1;
233 c_42 += a_4p * b_p2;
234 }
235
236 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
237 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
238 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
239 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
240 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
241 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
242 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
243 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
244 C[2 * _cs0 + 2 * _cs1] += alpha * c_22;
245 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
246 C[3 * _cs0 + 1 * _cs1] += alpha * c_31;
247 C[3 * _cs0 + 2 * _cs1] += alpha * c_32;
248 C[4 * _cs0 + 0 * _cs1] += alpha * c_40;
249 C[4 * _cs0 + 1 * _cs1] += alpha * c_41;
250 C[4 * _cs0 + 2 * _cs1] += alpha * c_42;
251
252 return 0;
253 }
254
255 template <>
256 template <typename ScalarType, typename ValueType>
257 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<5, 2>::serial_invoke(const ScalarType alpha,
258 const ValueType *KOKKOS_RESTRICT A,
259 const ValueType *KOKKOS_RESTRICT B, const int k,
260 /**/ ValueType *KOKKOS_RESTRICT C) {
261 if (k <= 0) return 0;
262
263 ValueType a_0p, b_p0, c_00 = 0, c_01 = 0, a_1p, b_p1, c_10 = 0, c_11 = 0, a_2p, c_20 = 0, c_21 = 0, a_3p, c_30 = 0,
264 c_31 = 0, a_4p, c_40 = 0, c_41 = 0;
265
266 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, i4 = 4 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1;
267
268 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
269 #pragma unroll
270 #endif
271 for (int p = 0; p < k; ++p) {
272 a_0p = A[i0 + p * _as1];
273 b_p0 = B[p * _bs0 + j0];
274 a_1p = A[i1 + p * _as1];
275 b_p1 = B[p * _bs0 + j1];
276 a_2p = A[i2 + p * _as1];
277 a_3p = A[i3 + p * _as1];
278 a_4p = A[i4 + p * _as1];
279
280 c_00 += a_0p * b_p0;
281 c_01 += a_0p * b_p1;
282 c_10 += a_1p * b_p0;
283 c_11 += a_1p * b_p1;
284 c_20 += a_2p * b_p0;
285 c_21 += a_2p * b_p1;
286 c_30 += a_3p * b_p0;
287 c_31 += a_3p * b_p1;
288 c_40 += a_4p * b_p0;
289 c_41 += a_4p * b_p1;
290 }
291
292 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
293 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
294 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
295 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
296 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
297 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
298 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
299 C[3 * _cs0 + 1 * _cs1] += alpha * c_31;
300 C[4 * _cs0 + 0 * _cs1] += alpha * c_40;
301 C[4 * _cs0 + 1 * _cs1] += alpha * c_41;
302
303 return 0;
304 }
305
306 template <>
307 template <typename ScalarType, typename ValueType>
308 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<5, 1>::serial_invoke(const ScalarType alpha,
309 const ValueType *KOKKOS_RESTRICT A,
310 const ValueType *KOKKOS_RESTRICT B, const int k,
311 /**/ ValueType *KOKKOS_RESTRICT C) {
312 if (k <= 0) return 0;
313
314 ValueType a_0p, b_p0, c_00 = 0, a_1p, c_10 = 0, a_2p, c_20 = 0, a_3p, c_30 = 0, a_4p, c_40 = 0;
315
316 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, i4 = 4 * _as0, j0 = 0 * _bs1;
317
318 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
319 #pragma unroll
320 #endif
321 for (int p = 0; p < k; ++p) {
322 a_0p = A[i0 + p * _as1];
323 b_p0 = B[p * _bs0 + j0];
324 a_1p = A[i1 + p * _as1];
325 a_2p = A[i2 + p * _as1];
326 a_3p = A[i3 + p * _as1];
327 a_4p = A[i4 + p * _as1];
328
329 c_00 += a_0p * b_p0;
330 c_10 += a_1p * b_p0;
331 c_20 += a_2p * b_p0;
332 c_30 += a_3p * b_p0;
333 c_40 += a_4p * b_p0;
334 }
335
336 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
337 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
338 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
339 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
340 C[4 * _cs0 + 0 * _cs1] += alpha * c_40;
341
342 return 0;
343 }
344
345 template <>
346 template <typename ScalarType, typename ValueType>
347 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<4, 5>::serial_invoke(const ScalarType alpha,
348 const ValueType *KOKKOS_RESTRICT A,
349 const ValueType *KOKKOS_RESTRICT B, const int k,
350 /**/ ValueType *KOKKOS_RESTRICT C) {
351 if (k <= 0) return 0;
352
353 ValueType a_0p, b_p0, c_00 = 0, c_01 = 0, c_02 = 0, c_03 = 0, c_04 = 0, a_1p, b_p1, c_10 = 0, c_11 = 0, c_12 = 0,
354 c_13 = 0, c_14 = 0, a_2p, b_p2, c_20 = 0, c_21 = 0, c_22 = 0, c_23 = 0, c_24 = 0, a_3p, b_p3,
355 c_30 = 0, c_31 = 0, c_32 = 0, c_33 = 0, c_34 = 0,
356 /**/ b_p4;
357
358 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1,
359 j3 = 3 * _bs1, j4 = 4 * _bs1;
360
361 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
362 #pragma unroll
363 #endif
364 for (int p = 0; p < k; ++p) {
365 a_0p = A[i0 + p * _as1];
366 b_p0 = B[p * _bs0 + j0];
367 a_1p = A[i1 + p * _as1];
368 b_p1 = B[p * _bs0 + j1];
369 a_2p = A[i2 + p * _as1];
370 b_p2 = B[p * _bs0 + j2];
371 a_3p = A[i3 + p * _as1];
372 b_p3 = B[p * _bs0 + j3];
373 /**/ b_p4 = B[p * _bs0 + j4];
374
375 c_00 += a_0p * b_p0;
376 c_01 += a_0p * b_p1;
377 c_02 += a_0p * b_p2;
378 c_03 += a_0p * b_p3;
379 c_04 += a_0p * b_p4;
380 c_10 += a_1p * b_p0;
381 c_11 += a_1p * b_p1;
382 c_12 += a_1p * b_p2;
383 c_13 += a_1p * b_p3;
384 c_14 += a_1p * b_p4;
385 c_20 += a_2p * b_p0;
386 c_21 += a_2p * b_p1;
387 c_22 += a_2p * b_p2;
388 c_23 += a_2p * b_p3;
389 c_24 += a_2p * b_p4;
390 c_30 += a_3p * b_p0;
391 c_31 += a_3p * b_p1;
392 c_32 += a_3p * b_p2;
393 c_33 += a_3p * b_p3;
394 c_34 += a_3p * b_p4;
395 }
396
397 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
398 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
399 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
400 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
401 C[0 * _cs0 + 4 * _cs1] += alpha * c_04;
402 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
403 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
404 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
405 C[1 * _cs0 + 3 * _cs1] += alpha * c_13;
406 C[1 * _cs0 + 4 * _cs1] += alpha * c_14;
407 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
408 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
409 C[2 * _cs0 + 2 * _cs1] += alpha * c_22;
410 C[2 * _cs0 + 3 * _cs1] += alpha * c_23;
411 C[2 * _cs0 + 4 * _cs1] += alpha * c_24;
412 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
413 C[3 * _cs0 + 1 * _cs1] += alpha * c_31;
414 C[3 * _cs0 + 2 * _cs1] += alpha * c_32;
415 C[3 * _cs0 + 3 * _cs1] += alpha * c_33;
416 C[3 * _cs0 + 4 * _cs1] += alpha * c_34;
417
418 return 0;
419 }
420
421 template <>
422 template <typename ScalarType, typename ValueType>
423 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<3, 5>::serial_invoke(const ScalarType alpha,
424 const ValueType *KOKKOS_RESTRICT A,
425 const ValueType *KOKKOS_RESTRICT B, const int k,
426 /**/ ValueType *KOKKOS_RESTRICT C) {
427 if (k <= 0) return 0;
428
429 ValueType a_0p, b_p0, c_00 = 0, c_01 = 0, c_02 = 0, c_03 = 0, c_04 = 0, a_1p, b_p1, c_10 = 0, c_11 = 0, c_12 = 0,
430 c_13 = 0, c_14 = 0, a_2p, b_p2, c_20 = 0, c_21 = 0, c_22 = 0, c_23 = 0, c_24 = 0,
431 /**/ b_p3,
432 /**/ b_p4;
433
434 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1, j3 = 3 * _bs1,
435 j4 = 4 * _bs1;
436
437 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
438 #pragma unroll
439 #endif
440 for (int p = 0; p < k; ++p) {
441 a_0p = A[i0 + p * _as1];
442 b_p0 = B[p * _bs0 + j0];
443 a_1p = A[i1 + p * _as1];
444 b_p1 = B[p * _bs0 + j1];
445 a_2p = A[i2 + p * _as1];
446 b_p2 = B[p * _bs0 + j2];
447 /**/ b_p3 = B[p * _bs0 + j3];
448 /**/ b_p4 = B[p * _bs0 + j4];
449
450 c_00 += a_0p * b_p0;
451 c_01 += a_0p * b_p1;
452 c_02 += a_0p * b_p2;
453 c_03 += a_0p * b_p3;
454 c_04 += a_0p * b_p4;
455 c_10 += a_1p * b_p0;
456 c_11 += a_1p * b_p1;
457 c_12 += a_1p * b_p2;
458 c_13 += a_1p * b_p3;
459 c_14 += a_1p * b_p4;
460 c_20 += a_2p * b_p0;
461 c_21 += a_2p * b_p1;
462 c_22 += a_2p * b_p2;
463 c_23 += a_2p * b_p3;
464 c_24 += a_2p * b_p4;
465 }
466
467 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
468 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
469 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
470 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
471 C[0 * _cs0 + 4 * _cs1] += alpha * c_04;
472 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
473 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
474 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
475 C[1 * _cs0 + 3 * _cs1] += alpha * c_13;
476 C[1 * _cs0 + 4 * _cs1] += alpha * c_14;
477 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
478 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
479 C[2 * _cs0 + 2 * _cs1] += alpha * c_22;
480 C[2 * _cs0 + 3 * _cs1] += alpha * c_23;
481 C[2 * _cs0 + 4 * _cs1] += alpha * c_24;
482
483 return 0;
484 }
485
486 template <>
487 template <typename ScalarType, typename ValueType>
488 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<2, 5>::serial_invoke(const ScalarType alpha,
489 const ValueType *KOKKOS_RESTRICT A,
490 const ValueType *KOKKOS_RESTRICT B, const int k,
491 /**/ ValueType *KOKKOS_RESTRICT C) {
492 if (k <= 0) return 0;
493
494 ValueType a_0p, b_p0, c_00 = 0, c_01 = 0, c_02 = 0, c_03 = 0, c_04 = 0, a_1p, b_p1, c_10 = 0, c_11 = 0, c_12 = 0,
495 c_13 = 0, c_14 = 0,
496 /**/ b_p2,
497 /**/ b_p3,
498 /**/ b_p4;
499
500 const int i0 = 0 * _as0, i1 = 1 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1, j3 = 3 * _bs1, j4 = 4 * _bs1;
501
502 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
503 #pragma unroll
504 #endif
505 for (int p = 0; p < k; ++p) {
506 a_0p = A[i0 + p * _as1];
507 b_p0 = B[p * _bs0 + j0];
508 a_1p = A[i1 + p * _as1];
509 b_p1 = B[p * _bs0 + j1];
510 /**/ b_p2 = B[p * _bs0 + j2];
511 /**/ b_p3 = B[p * _bs0 + j3];
512 /**/ b_p4 = B[p * _bs0 + j4];
513
514 c_00 += a_0p * b_p0;
515 c_01 += a_0p * b_p1;
516 c_02 += a_0p * b_p2;
517 c_03 += a_0p * b_p3;
518 c_04 += a_0p * b_p4;
519 c_10 += a_1p * b_p0;
520 c_11 += a_1p * b_p1;
521 c_12 += a_1p * b_p2;
522 c_13 += a_1p * b_p3;
523 c_14 += a_1p * b_p4;
524 }
525
526 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
527 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
528 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
529 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
530 C[0 * _cs0 + 4 * _cs1] += alpha * c_04;
531 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
532 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
533 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
534 C[1 * _cs0 + 3 * _cs1] += alpha * c_13;
535 C[1 * _cs0 + 4 * _cs1] += alpha * c_14;
536
537 return 0;
538 }
539
540 template <>
541 template <typename ScalarType, typename ValueType>
542 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<1, 5>::serial_invoke(const ScalarType alpha,
543 const ValueType *KOKKOS_RESTRICT A,
544 const ValueType *KOKKOS_RESTRICT B, const int k,
545 /**/ ValueType *KOKKOS_RESTRICT C) {
546 if (k <= 0) return 0;
547
548 ValueType a_0p, b_p0, c_00 = 0, c_01 = 0, c_02 = 0, c_03 = 0, c_04 = 0,
549 /**/ b_p1,
550 /**/ b_p2,
551 /**/ b_p3,
552 /**/ b_p4;
553
554 const int i0 = 0 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1, j3 = 3 * _bs1, j4 = 4 * _bs1;
555
556 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
557 #pragma unroll
558 #endif
559 for (int p = 0; p < k; ++p) {
560 a_0p = A[i0 + p * _as1];
561 b_p0 = B[p * _bs0 + j0];
562 /**/ b_p1 = B[p * _bs0 + j1];
563 /**/ b_p2 = B[p * _bs0 + j2];
564 /**/ b_p3 = B[p * _bs0 + j3];
565 /**/ b_p4 = B[p * _bs0 + j4];
566
567 c_00 += a_0p * b_p0;
568 c_01 += a_0p * b_p1;
569 c_02 += a_0p * b_p2;
570 c_03 += a_0p * b_p3;
571 c_04 += a_0p * b_p4;
572 }
573
574 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
575 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
576 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
577 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
578 C[0 * _cs0 + 4 * _cs1] += alpha * c_04;
579
580 return 0;
581 }
582 ///
583 /// Inner kernel (4x4)
584 /// ==================
585
586 template <>
587 template <typename ScalarType, typename ValueType>
588 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<4, 4>::serial_invoke(const ScalarType alpha,
589 const ValueType *KOKKOS_RESTRICT A,
590 const ValueType *KOKKOS_RESTRICT B, const int k,
591 /**/ ValueType *KOKKOS_RESTRICT C) {
592 if (k <= 0) return 0;
593
594 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), c_02 = ValueType(0), c_03 = ValueType(0), a_1p, b_p1,
595 c_10 = ValueType(0), c_11 = ValueType(0), c_12 = ValueType(0), c_13 = ValueType(0), a_2p, b_p2,
596 c_20 = ValueType(0), c_21 = ValueType(0), c_22 = ValueType(0), c_23 = ValueType(0), a_3p, b_p3,
597 c_30 = ValueType(0), c_31 = ValueType(0), c_32 = ValueType(0), c_33 = ValueType(0);
598
599 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1,
600 j3 = 3 * _bs1;
601
602 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
603 #pragma unroll
604 #endif
605 for (int p = 0; p < k; ++p) {
606 a_0p = A[i0 + p * _as1];
607 b_p0 = B[p * _bs0 + j0];
608 a_1p = A[i1 + p * _as1];
609 b_p1 = B[p * _bs0 + j1];
610 a_2p = A[i2 + p * _as1];
611 b_p2 = B[p * _bs0 + j2];
612 a_3p = A[i3 + p * _as1];
613 b_p3 = B[p * _bs0 + j3];
614
615 c_00 += a_0p * b_p0;
616 c_01 += a_0p * b_p1;
617 c_02 += a_0p * b_p2;
618 c_03 += a_0p * b_p3;
619 c_10 += a_1p * b_p0;
620 c_11 += a_1p * b_p1;
621 c_12 += a_1p * b_p2;
622 c_13 += a_1p * b_p3;
623 c_20 += a_2p * b_p0;
624 c_21 += a_2p * b_p1;
625 c_22 += a_2p * b_p2;
626 c_23 += a_2p * b_p3;
627 c_30 += a_3p * b_p0;
628 c_31 += a_3p * b_p1;
629 c_32 += a_3p * b_p2;
630 c_33 += a_3p * b_p3;
631 }
632
633 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
634 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
635 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
636 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
637 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
638 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
639 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
640 C[1 * _cs0 + 3 * _cs1] += alpha * c_13;
641 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
642 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
643 C[2 * _cs0 + 2 * _cs1] += alpha * c_22;
644 C[2 * _cs0 + 3 * _cs1] += alpha * c_23;
645 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
646 C[3 * _cs0 + 1 * _cs1] += alpha * c_31;
647 C[3 * _cs0 + 2 * _cs1] += alpha * c_32;
648 C[3 * _cs0 + 3 * _cs1] += alpha * c_33;
649
650 return 0;
651 }
652
653 template <>
654 template <typename ScalarType, typename ValueType>
655 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<4, 3>::serial_invoke(const ScalarType alpha,
656 const ValueType *KOKKOS_RESTRICT A,
657 const ValueType *KOKKOS_RESTRICT B, const int k,
658 /**/ ValueType *KOKKOS_RESTRICT C) {
659 if (k <= 0) return 0;
660
661 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), c_02 = ValueType(0), a_1p, b_p1, c_10 = ValueType(0),
662 c_11 = ValueType(0), c_12 = ValueType(0), a_2p, b_p2, c_20 = ValueType(0), c_21 = ValueType(0),
663 c_22 = ValueType(0), a_3p, c_30 = ValueType(0), c_31 = ValueType(0), c_32 = ValueType(0);
664
665 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1;
666
667 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
668 #pragma unroll
669 #endif
670 for (int p = 0; p < k; ++p) {
671 a_0p = A[i0 + p * _as1];
672 b_p0 = B[p * _bs0 + j0];
673 a_1p = A[i1 + p * _as1];
674 b_p1 = B[p * _bs0 + j1];
675 a_2p = A[i2 + p * _as1];
676 b_p2 = B[p * _bs0 + j2];
677 a_3p = A[i3 + p * _as1];
678
679 c_00 += a_0p * b_p0;
680 c_01 += a_0p * b_p1;
681 c_02 += a_0p * b_p2;
682 c_10 += a_1p * b_p0;
683 c_11 += a_1p * b_p1;
684 c_12 += a_1p * b_p2;
685 c_20 += a_2p * b_p0;
686 c_21 += a_2p * b_p1;
687 c_22 += a_2p * b_p2;
688 c_30 += a_3p * b_p0;
689 c_31 += a_3p * b_p1;
690 c_32 += a_3p * b_p2;
691 }
692
693 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
694 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
695 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
696 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
697 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
698 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
699 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
700 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
701 C[2 * _cs0 + 2 * _cs1] += alpha * c_22;
702 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
703 C[3 * _cs0 + 1 * _cs1] += alpha * c_31;
704 C[3 * _cs0 + 2 * _cs1] += alpha * c_32;
705
706 return 0;
707 }
708
709 template <>
710 template <typename ScalarType, typename ValueType>
711 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<4, 2>::serial_invoke(const ScalarType alpha,
712 const ValueType *KOKKOS_RESTRICT A,
713 const ValueType *KOKKOS_RESTRICT B, const int k,
714 /**/ ValueType *KOKKOS_RESTRICT C) {
715 if (k <= 0) return 0;
716
717 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), a_1p, b_p1, c_10 = ValueType(0), c_11 = ValueType(0),
718 a_2p, c_20 = ValueType(0), c_21 = ValueType(0), a_3p, c_30 = ValueType(0), c_31 = ValueType(0);
719
720 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1;
721
722 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
723 #pragma unroll
724 #endif
725 for (int p = 0; p < k; ++p) {
726 a_0p = A[i0 + p * _as1];
727 b_p0 = B[p * _bs0 + j0];
728 a_1p = A[i1 + p * _as1];
729 b_p1 = B[p * _bs0 + j1];
730 a_2p = A[i2 + p * _as1];
731 a_3p = A[i3 + p * _as1];
732
733 c_00 += a_0p * b_p0;
734 c_01 += a_0p * b_p1;
735 c_10 += a_1p * b_p0;
736 c_11 += a_1p * b_p1;
737 c_20 += a_2p * b_p0;
738 c_21 += a_2p * b_p1;
739 c_30 += a_3p * b_p0;
740 c_31 += a_3p * b_p1;
741 }
742
743 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
744 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
745 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
746 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
747 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
748 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
749 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
750 C[3 * _cs0 + 1 * _cs1] += alpha * c_31;
751
752 return 0;
753 }
754
755 template <>
756 template <typename ScalarType, typename ValueType>
757 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<4, 1>::serial_invoke(const ScalarType alpha,
758 const ValueType *KOKKOS_RESTRICT A,
759 const ValueType *KOKKOS_RESTRICT B, const int k,
760 /**/ ValueType *KOKKOS_RESTRICT C) {
761 if (k <= 0) return 0;
762
763 ValueType a_0p, b_p0, c_00 = ValueType(0), a_1p, c_10 = ValueType(0), a_2p, c_20 = ValueType(0), a_3p,
764 c_30 = ValueType(0);
765
766 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, i3 = 3 * _as0, j0 = 0 * _bs1;
767
768 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
769 #pragma unroll
770 #endif
771 for (int p = 0; p < k; ++p) {
772 a_0p = A[i0 + p * _as1];
773 b_p0 = B[p * _bs0 + j0];
774 a_1p = A[i1 + p * _as1];
775 a_2p = A[i2 + p * _as1];
776 a_3p = A[i3 + p * _as1];
777
778 c_00 += a_0p * b_p0;
779 c_10 += a_1p * b_p0;
780 c_20 += a_2p * b_p0;
781 c_30 += a_3p * b_p0;
782 }
783
784 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
785 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
786 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
787 C[3 * _cs0 + 0 * _cs1] += alpha * c_30;
788
789 return 0;
790 }
791
792 template <>
793 template <typename ScalarType, typename ValueType>
794 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<3, 4>::serial_invoke(const ScalarType alpha,
795 const ValueType *KOKKOS_RESTRICT A,
796 const ValueType *KOKKOS_RESTRICT B, const int k,
797 /**/ ValueType *KOKKOS_RESTRICT C) {
798 if (k <= 0) return 0;
799
800 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), c_02 = ValueType(0), c_03 = ValueType(0), a_1p, b_p1,
801 c_10 = ValueType(0), c_11 = ValueType(0), c_12 = ValueType(0), c_13 = ValueType(0), a_2p, b_p2,
802 c_20 = ValueType(0), c_21 = ValueType(0), c_22 = ValueType(0), c_23 = ValueType(0),
803 /**/ b_p3;
804
805 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1, j3 = 3 * _bs1;
806
807 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
808 #pragma unroll
809 #endif
810 for (int p = 0; p < k; ++p) {
811 a_0p = A[i0 + p * _as1];
812 b_p0 = B[p * _bs0 + j0];
813 a_1p = A[i1 + p * _as1];
814 b_p1 = B[p * _bs0 + j1];
815 a_2p = A[i2 + p * _as1];
816 b_p2 = B[p * _bs0 + j2];
817 /**/ b_p3 = B[p * _bs0 + j3];
818
819 c_00 += a_0p * b_p0;
820 c_01 += a_0p * b_p1;
821 c_02 += a_0p * b_p2;
822 c_03 += a_0p * b_p3;
823 c_10 += a_1p * b_p0;
824 c_11 += a_1p * b_p1;
825 c_12 += a_1p * b_p2;
826 c_13 += a_1p * b_p3;
827 c_20 += a_2p * b_p0;
828 c_21 += a_2p * b_p1;
829 c_22 += a_2p * b_p2;
830 c_23 += a_2p * b_p3;
831 }
832
833 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
834 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
835 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
836 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
837 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
838 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
839 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
840 C[1 * _cs0 + 3 * _cs1] += alpha * c_13;
841 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
842 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
843 C[2 * _cs0 + 2 * _cs1] += alpha * c_22;
844 C[2 * _cs0 + 3 * _cs1] += alpha * c_23;
845
846 return 0;
847 }
848
849 template <>
850 template <typename ScalarType, typename ValueType>
851 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<2, 4>::serial_invoke(const ScalarType alpha,
852 const ValueType *KOKKOS_RESTRICT A,
853 const ValueType *KOKKOS_RESTRICT B, const int k,
854 /**/ ValueType *KOKKOS_RESTRICT C) {
855 if (k <= 0) return 0;
856
857 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), c_02 = ValueType(0), c_03 = ValueType(0), a_1p, b_p1,
858 c_10 = ValueType(0), c_11 = ValueType(0), c_12 = ValueType(0), c_13 = ValueType(0),
859 /**/ b_p2,
860 /**/ b_p3;
861
862 const int i0 = 0 * _as0, i1 = 1 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1, j3 = 3 * _bs1;
863
864 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
865 #pragma unroll
866 #endif
867 for (int p = 0; p < k; ++p) {
868 a_0p = A[i0 + p * _as1];
869 b_p0 = B[p * _bs0 + j0];
870 a_1p = A[i1 + p * _as1];
871 b_p1 = B[p * _bs0 + j1];
872 /**/ b_p2 = B[p * _bs0 + j2];
873 /**/ b_p3 = B[p * _bs0 + j3];
874
875 c_00 += a_0p * b_p0;
876 c_01 += a_0p * b_p1;
877 c_02 += a_0p * b_p2;
878 c_03 += a_0p * b_p3;
879 c_10 += a_1p * b_p0;
880 c_11 += a_1p * b_p1;
881 c_12 += a_1p * b_p2;
882 c_13 += a_1p * b_p3;
883 }
884
885 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
886 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
887 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
888 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
889 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
890 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
891 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
892 C[1 * _cs0 + 3 * _cs1] += alpha * c_13;
893
894 return 0;
895 }
896
897 template <>
898 template <typename ScalarType, typename ValueType>
899 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<1, 4>::serial_invoke(const ScalarType alpha,
900 const ValueType *KOKKOS_RESTRICT A,
901 const ValueType *KOKKOS_RESTRICT B, const int k,
902 /**/ ValueType *KOKKOS_RESTRICT C) {
903 if (k <= 0) return 0;
904
905 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), c_02 = ValueType(0), c_03 = ValueType(0),
906 /**/ b_p1,
907 /**/ b_p2,
908 /**/ b_p3;
909
910 const int i0 = 0 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1, j3 = 3 * _bs1;
911
912 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
913 #pragma unroll
914 #endif
915 for (int p = 0; p < k; ++p) {
916 a_0p = A[i0 + p * _as1];
917 b_p0 = B[p * _bs0 + j0];
918 /**/ b_p1 = B[p * _bs0 + j1];
919 /**/ b_p2 = B[p * _bs0 + j2];
920 /**/ b_p3 = B[p * _bs0 + j3];
921
922 c_00 += a_0p * b_p0;
923 c_01 += a_0p * b_p1;
924 c_02 += a_0p * b_p2;
925 c_03 += a_0p * b_p3;
926 }
927
928 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
929 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
930 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
931 C[0 * _cs0 + 3 * _cs1] += alpha * c_03;
932
933 return 0;
934 }
935
936 ///
937 /// Inner kernel (3x3)
938 /// ==================
939
940 template <>
941 template <typename ScalarType, typename ValueType>
942 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<3, 3>::serial_invoke(const ScalarType alpha,
943 const ValueType *KOKKOS_RESTRICT A,
944 const ValueType *KOKKOS_RESTRICT B, const int k,
945 /**/ ValueType *KOKKOS_RESTRICT C) {
946 if (k <= 0) return 0;
947
948 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), c_02 = ValueType(0), a_1p, b_p1, c_10 = ValueType(0),
949 c_11 = ValueType(0), c_12 = ValueType(0), a_2p, b_p2, c_20 = ValueType(0), c_21 = ValueType(0),
950 c_22 = ValueType(0);
951
952 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1;
953
954 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
955 #pragma unroll
956 #endif
957 for (int p = 0; p < k; ++p) {
958 a_0p = A[i0 + p * _as1];
959 b_p0 = B[p * _bs0 + j0];
960 a_1p = A[i1 + p * _as1];
961 b_p1 = B[p * _bs0 + j1];
962 a_2p = A[i2 + p * _as1];
963 b_p2 = B[p * _bs0 + j2];
964
965 c_00 += a_0p * b_p0;
966 c_01 += a_0p * b_p1;
967 c_02 += a_0p * b_p2;
968 c_10 += a_1p * b_p0;
969 c_11 += a_1p * b_p1;
970 c_12 += a_1p * b_p2;
971 c_20 += a_2p * b_p0;
972 c_21 += a_2p * b_p1;
973 c_22 += a_2p * b_p2;
974 }
975
976 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
977 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
978 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
979 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
980 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
981 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
982 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
983 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
984 C[2 * _cs0 + 2 * _cs1] += alpha * c_22;
985
986 return 0;
987 }
988
989 template <>
990 template <typename ScalarType, typename ValueType>
991 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<3, 2>::serial_invoke(const ScalarType alpha,
992 const ValueType *KOKKOS_RESTRICT A,
993 const ValueType *KOKKOS_RESTRICT B, const int k,
994 /**/ ValueType *KOKKOS_RESTRICT C) {
995 if (k <= 0) return 0;
996
997 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), a_1p, b_p1, c_10 = ValueType(0), c_11 = ValueType(0),
998 a_2p, c_20 = ValueType(0), c_21 = ValueType(0);
999
1000 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1;
1001
1002 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1003 #pragma unroll
1004 #endif
1005 for (int p = 0; p < k; ++p) {
1006 a_0p = A[i0 + p * _as1];
1007 b_p0 = B[p * _bs0 + j0];
1008 a_1p = A[i1 + p * _as1];
1009 b_p1 = B[p * _bs0 + j1];
1010 a_2p = A[i2 + p * _as1];
1011
1012 c_00 += a_0p * b_p0;
1013 c_01 += a_0p * b_p1;
1014 c_10 += a_1p * b_p0;
1015 c_11 += a_1p * b_p1;
1016 c_20 += a_2p * b_p0;
1017 c_21 += a_2p * b_p1;
1018 }
1019
1020 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
1021 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
1022 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
1023 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
1024 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
1025 C[2 * _cs0 + 1 * _cs1] += alpha * c_21;
1026
1027 return 0;
1028 }
1029
1030 template <>
1031 template <typename ScalarType, typename ValueType>
1032 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<3, 1>::serial_invoke(const ScalarType alpha,
1033 const ValueType *KOKKOS_RESTRICT A,
1034 const ValueType *KOKKOS_RESTRICT B, const int k,
1035 /**/ ValueType *KOKKOS_RESTRICT C) {
1036 if (k <= 0) return 0;
1037
1038 ValueType a_0p, b_p0, c_00 = ValueType(0), a_1p, c_10 = ValueType(0), a_2p, c_20 = ValueType(0);
1039
1040 const int i0 = 0 * _as0, i1 = 1 * _as0, i2 = 2 * _as0, j0 = 0 * _bs1;
1041
1042 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1043 #pragma unroll
1044 #endif
1045 for (int p = 0; p < k; ++p) {
1046 a_0p = A[i0 + p * _as1];
1047 b_p0 = B[p * _bs0 + j0];
1048 a_1p = A[i1 + p * _as1];
1049 a_2p = A[i2 + p * _as1];
1050
1051 c_00 += a_0p * b_p0;
1052 c_10 += a_1p * b_p0;
1053 c_20 += a_2p * b_p0;
1054 }
1055
1056 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
1057 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
1058 C[2 * _cs0 + 0 * _cs1] += alpha * c_20;
1059
1060 return 0;
1061 }
1062
1063 template <>
1064 template <typename ScalarType, typename ValueType>
1065 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<2, 3>::serial_invoke(const ScalarType alpha,
1066 const ValueType *KOKKOS_RESTRICT A,
1067 const ValueType *KOKKOS_RESTRICT B, const int k,
1068 /**/ ValueType *KOKKOS_RESTRICT C) {
1069 if (k <= 0) return 0;
1070
1071 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), c_02 = ValueType(0), a_1p, b_p1, c_10 = ValueType(0),
1072 c_11 = ValueType(0), c_12 = ValueType(0),
1073 /**/ b_p2;
1074
1075 const int i0 = 0 * _as0, i1 = 1 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1;
1076
1077 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1078 #pragma unroll
1079 #endif
1080 for (int p = 0; p < k; ++p) {
1081 a_0p = A[i0 + p * _as1];
1082 b_p0 = B[p * _bs0 + j0];
1083 a_1p = A[i1 + p * _as1];
1084 b_p1 = B[p * _bs0 + j1];
1085 /**/ b_p2 = B[p * _bs0 + j2];
1086
1087 c_00 += a_0p * b_p0;
1088 c_01 += a_0p * b_p1;
1089 c_02 += a_0p * b_p2;
1090 c_10 += a_1p * b_p0;
1091 c_11 += a_1p * b_p1;
1092 c_12 += a_1p * b_p2;
1093 }
1094
1095 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
1096 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
1097 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
1098 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
1099 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
1100 C[1 * _cs0 + 2 * _cs1] += alpha * c_12;
1101
1102 return 0;
1103 }
1104 template <>
1105 template <typename ScalarType, typename ValueType>
1106 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<1, 3>::serial_invoke(const ScalarType alpha,
1107 const ValueType *KOKKOS_RESTRICT A,
1108 const ValueType *KOKKOS_RESTRICT B, const int k,
1109 /**/ ValueType *KOKKOS_RESTRICT C) {
1110 if (k <= 0) return 0;
1111
1112 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), c_02 = ValueType(0),
1113 /**/ b_p1,
1114 /**/ b_p2;
1115
1116 const int i0 = 0 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1, j2 = 2 * _bs1;
1117
1118 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1119 #pragma unroll
1120 #endif
1121 for (int p = 0; p < k; ++p) {
1122 a_0p = A[i0 + p * _as1];
1123 b_p0 = B[p * _bs0 + j0];
1124 /**/ b_p1 = B[p * _bs0 + j1];
1125 /**/ b_p2 = B[p * _bs0 + j2];
1126
1127 c_00 += a_0p * b_p0;
1128 c_01 += a_0p * b_p1;
1129 c_02 += a_0p * b_p2;
1130 }
1131
1132 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
1133 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
1134 C[0 * _cs0 + 2 * _cs1] += alpha * c_02;
1135
1136 return 0;
1137 }
1138
1139 ///
1140 /// Inner kernel (2x2)
1141 /// ==================
1142
1143 template <>
1144 template <typename ScalarType, typename ValueType>
1145 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<2, 2>::serial_invoke(const ScalarType alpha,
1146 const ValueType *KOKKOS_RESTRICT A,
1147 const ValueType *KOKKOS_RESTRICT B, const int k,
1148 /**/ ValueType *KOKKOS_RESTRICT C) {
1149 if (k <= 0) return 0;
1150
1151 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0), a_1p, b_p1, c_10 = ValueType(0), c_11 = ValueType(0);
1152
1153 const int i0 = 0 * _as0, i1 = 1 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1;
1154
1155 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1156 #pragma unroll
1157 #endif
1158 for (int p = 0; p < k; ++p) {
1159 a_0p = A[i0 + p * _as1];
1160 b_p0 = B[p * _bs0 + j0];
1161 a_1p = A[i1 + p * _as1];
1162 b_p1 = B[p * _bs0 + j1];
1163
1164 c_00 += a_0p * b_p0;
1165 c_01 += a_0p * b_p1;
1166 c_10 += a_1p * b_p0;
1167 c_11 += a_1p * b_p1;
1168 }
1169
1170 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
1171 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
1172 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
1173 C[1 * _cs0 + 1 * _cs1] += alpha * c_11;
1174
1175 return 0;
1176 }
1177
1178 template <>
1179 template <typename ScalarType, typename ValueType>
1180 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<2, 1>::serial_invoke(const ScalarType alpha,
1181 const ValueType *KOKKOS_RESTRICT A,
1182 const ValueType *KOKKOS_RESTRICT B, const int k,
1183 /**/ ValueType *KOKKOS_RESTRICT C) {
1184 if (k <= 0) return 0;
1185
1186 ValueType a_0p, b_p0, c_00 = ValueType(0), a_1p, c_10 = ValueType(0);
1187
1188 const int i0 = 0 * _as0, i1 = 1 * _as0, j0 = 0 * _bs1;
1189
1190 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1191 #pragma unroll
1192 #endif
1193 for (int p = 0; p < k; ++p) {
1194 a_0p = A[i0 + p * _as1];
1195 b_p0 = B[p * _bs0 + j0];
1196 a_1p = A[i1 + p * _as1];
1197
1198 c_00 += a_0p * b_p0;
1199 c_10 += a_1p * b_p0;
1200 }
1201
1202 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
1203 C[1 * _cs0 + 0 * _cs1] += alpha * c_10;
1204
1205 return 0;
1206 }
1207
1208 template <>
1209 template <typename ScalarType, typename ValueType>
1210 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<1, 2>::serial_invoke(const ScalarType alpha,
1211 const ValueType *KOKKOS_RESTRICT A,
1212 const ValueType *KOKKOS_RESTRICT B, const int k,
1213 /**/ ValueType *KOKKOS_RESTRICT C) {
1214 if (k <= 0) return 0;
1215
1216 ValueType a_0p, b_p0, c_00 = ValueType(0), c_01 = ValueType(0),
1217 /**/ b_p1;
1218 const int i0 = 0 * _as0, j0 = 0 * _bs1, j1 = 1 * _bs1;
1219
1220 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1221 #pragma unroll
1222 #endif
1223 for (int p = 0; p < k; ++p) {
1224 a_0p = A[i0 + p * _as1];
1225 b_p0 = B[p * _bs0 + j0];
1226 /* */ b_p1 = B[p * _bs0 + j1];
1227 c_00 += a_0p * b_p0;
1228 c_01 += a_0p * b_p1;
1229 }
1230
1231 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
1232 C[0 * _cs0 + 1 * _cs1] += alpha * c_01;
1233
1234 return 0;
1235 }
1236
1237 ///
1238 /// Inner kernel (1x1)
1239 /// ==================
1240
1241 template <>
1242 template <typename ScalarType, typename ValueType>
1243 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<1, 1>::serial_invoke(const ScalarType alpha,
1244 const ValueType *KOKKOS_RESTRICT A,
1245 const ValueType *KOKKOS_RESTRICT B, const int k,
1246 /**/ ValueType *KOKKOS_RESTRICT C) {
1247 if (k <= 0) return 0;
1248
1249 ValueType a_0p, b_p0, c_00 = ValueType(0);
1250
1251 const int i0 = 0 * _as0, j0 = 0 * _bs1;
1252
1253 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1254 #pragma unroll
1255 #endif
1256 for (int p = 0; p < k; ++p) {
1257 a_0p = A[i0 + p * _as1];
1258 b_p0 = B[p * _bs0 + j0];
1259 c_00 += a_0p * b_p0;
1260 }
1261 C[0 * _cs0 + 0 * _cs1] += alpha * c_00;
1262
1263 return 0;
1264 }
1265
1266 template <>
1267 template <typename ScalarType, typename ValueType>
1268 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<0, 1>::serial_invoke(const ScalarType alpha,
1269 const ValueType *KOKKOS_RESTRICT A,
1270 const ValueType *KOKKOS_RESTRICT B, const int m,
1271 const int k,
1272 /**/ ValueType *KOKKOS_RESTRICT C) {
1273 if (m <= 0 || k <= 0) return 0;
1274
1275 switch (m) {
1276 case 5: {
1277 InnerGemmFixC<5, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1278 inner.serial_invoke(alpha, A, B, k, C);
1279 break;
1280 }
1281 case 4: {
1282 InnerGemmFixC<4, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1283 inner.serial_invoke(alpha, A, B, k, C);
1284 break;
1285 }
1286 case 3: {
1287 InnerGemmFixC<3, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1288 inner.serial_invoke(alpha, A, B, k, C);
1289 break;
1290 }
1291 case 2: {
1292 InnerGemmFixC<2, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1293 inner.serial_invoke(alpha, A, B, k, C);
1294 break;
1295 }
1296 case 1: {
1297 InnerGemmFixC<1, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1298 inner.serial_invoke(alpha, A, B, k, C);
1299 break;
1300 }
1301 default: {
1302 Kokkos::abort("InnerGemmFixC<0,1>::serial_invoke, assert failure (m<=5)");
1303 break;
1304 }
1305 }
1306 return 0;
1307 }
1308
1309 template <>
1310 template <typename ScalarType, typename ValueType>
1311 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<5, 5>::serial_invoke(const ScalarType alpha,
1312 const ValueType *KOKKOS_RESTRICT A,
1313 const ValueType *KOKKOS_RESTRICT B, const int m,
1314 const int n, const int k,
1315 /**/ ValueType *KOKKOS_RESTRICT C) {
1316 if (m <= 0 || n <= 0 || k <= 0) return 0;
1317 if (!(m <= 5 && n <= 5)) Kokkos::abort("InnerGemmFixC<5,5>::serial_invoke, assert failure (m<=5 && n<=5)");
1318
1319 switch (m * 10 + n) {
1320 case 55: {
1321 InnerGemmFixC<5, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1322 inner.serial_invoke(alpha, A, B, k, C);
1323 break;
1324 }
1325 case 54: {
1326 InnerGemmFixC<5, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1327 inner.serial_invoke(alpha, A, B, k, C);
1328 break;
1329 }
1330 case 53: {
1331 InnerGemmFixC<5, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1332 inner.serial_invoke(alpha, A, B, k, C);
1333 break;
1334 }
1335 case 52: {
1336 InnerGemmFixC<5, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1337 inner.serial_invoke(alpha, A, B, k, C);
1338 break;
1339 }
1340 case 51: {
1341 InnerGemmFixC<5, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1342 inner.serial_invoke(alpha, A, B, k, C);
1343 break;
1344 }
1345 case 45: {
1346 InnerGemmFixC<4, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1347 inner.serial_invoke(alpha, A, B, k, C);
1348 break;
1349 }
1350 case 35: {
1351 InnerGemmFixC<3, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1352 inner.serial_invoke(alpha, A, B, k, C);
1353 break;
1354 }
1355 case 25: {
1356 InnerGemmFixC<2, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1357 inner.serial_invoke(alpha, A, B, k, C);
1358 break;
1359 }
1360 case 15: {
1361 InnerGemmFixC<1, 5> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1362 inner.serial_invoke(alpha, A, B, k, C);
1363 break;
1364 }
1365 default: {
1366 InnerGemmFixC<4, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1367 inner.serial_invoke(alpha, A, B, m, n, k, C);
1368 break;
1369 }
1370 }
1371 return 0;
1372 }
1373
1374 template <>
1375 template <typename ScalarType, typename ValueType>
1376 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<4, 4>::serial_invoke(const ScalarType alpha,
1377 const ValueType *KOKKOS_RESTRICT A,
1378 const ValueType *KOKKOS_RESTRICT B, const int m,
1379 const int n, const int k,
1380 /**/ ValueType *KOKKOS_RESTRICT C) {
1381 if (m <= 0 || n <= 0 || k <= 0) return 0;
1382 if (!(m <= 4 && n <= 4)) Kokkos::abort("InnerGemmFixC<4,4>::serial_invoke, assert failure (m<=4 && n<=4)");
1383
1384 switch (m * 10 + n) {
1385 case 44: {
1386 InnerGemmFixC<4, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1387 inner.serial_invoke(alpha, A, B, k, C);
1388 break;
1389 }
1390 case 43: {
1391 InnerGemmFixC<4, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1392 inner.serial_invoke(alpha, A, B, k, C);
1393 break;
1394 }
1395 case 42: {
1396 InnerGemmFixC<4, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1397 inner.serial_invoke(alpha, A, B, k, C);
1398 break;
1399 }
1400 case 41: {
1401 InnerGemmFixC<4, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1402 inner.serial_invoke(alpha, A, B, k, C);
1403 break;
1404 }
1405 case 34: {
1406 InnerGemmFixC<3, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1407 inner.serial_invoke(alpha, A, B, k, C);
1408 break;
1409 }
1410 case 24: {
1411 InnerGemmFixC<2, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1412 inner.serial_invoke(alpha, A, B, k, C);
1413 break;
1414 }
1415 case 14: {
1416 InnerGemmFixC<1, 4> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1417 inner.serial_invoke(alpha, A, B, k, C);
1418 break;
1419 }
1420 default: {
1421 InnerGemmFixC<3, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1422 inner.serial_invoke(alpha, A, B, m, n, k, C);
1423 break;
1424 }
1425 }
1426 return 0;
1427 }
1428
1429 template <>
1430 template <typename ScalarType, typename ValueType>
1431 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<3, 3>::serial_invoke(const ScalarType alpha,
1432 const ValueType *KOKKOS_RESTRICT A,
1433 const ValueType *KOKKOS_RESTRICT B, const int m,
1434 const int n, const int k,
1435 /**/ ValueType *KOKKOS_RESTRICT C) {
1436 if (m <= 0 || n <= 0 || k <= 0) return 0;
1437 if (!(m <= 3 && n <= 3)) Kokkos::abort("InnerGemmFixC<3,3>::serial_invoke, assert failure (m<=3 && n<=3)");
1438
1439 switch (m * 10 + n) {
1440 case 33: {
1441 InnerGemmFixC<3, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1442 inner.serial_invoke(alpha, A, B, k, C);
1443 break;
1444 }
1445 case 32: {
1446 InnerGemmFixC<3, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1447 inner.serial_invoke(alpha, A, B, k, C);
1448 break;
1449 }
1450 case 31: {
1451 InnerGemmFixC<3, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1452 inner.serial_invoke(alpha, A, B, k, C);
1453 break;
1454 }
1455 case 23: {
1456 InnerGemmFixC<2, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1457 inner.serial_invoke(alpha, A, B, k, C);
1458 break;
1459 }
1460 case 13: {
1461 InnerGemmFixC<1, 3> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1462 inner.serial_invoke(alpha, A, B, k, C);
1463 break;
1464 }
1465 default: {
1466 InnerGemmFixC<2, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1467 inner.serial_invoke(alpha, A, B, m, n, k, C);
1468 break;
1469 }
1470 }
1471 return 0;
1472 }
1473
1474 template <>
1475 template <typename ScalarType, typename ValueType>
1476 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<2, 2>::serial_invoke(const ScalarType alpha,
1477 const ValueType *KOKKOS_RESTRICT A,
1478 const ValueType *KOKKOS_RESTRICT B, const int m,
1479 const int n, const int k,
1480 /**/ ValueType *KOKKOS_RESTRICT C) {
1481 if (m <= 0 || n <= 0 || k <= 0) return 0;
1482 if (!(m <= 2 && n <= 2)) Kokkos::abort("InnerGemmFixC<2,2>::serial_invoke, assert failure (m<=2 && n<=2)");
1483
1484 switch (m * 10 + n) {
1485 case 22: {
1486 InnerGemmFixC<2, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1487 inner.serial_invoke(alpha, A, B, k, C);
1488 break;
1489 }
1490 case 21: {
1491 InnerGemmFixC<2, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1492 inner.serial_invoke(alpha, A, B, k, C);
1493 break;
1494 }
1495 case 12: {
1496 InnerGemmFixC<1, 2> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1497 inner.serial_invoke(alpha, A, B, k, C);
1498 break;
1499 }
1500 case 11: {
1501 InnerGemmFixC<1, 1> inner(_as0, _as1, _bs0, _bs1, _cs0, _cs1);
1502 inner.serial_invoke(alpha, A, B, k, C);
1503 break;
1504 }
1505 }
1506 return 0;
1507 }
1508
1509 template <>
1510 template <typename ScalarType, typename ValueType>
1511 KOKKOS_INLINE_FUNCTION int InnerGemmFixC<1, 1>::serial_invoke(const ScalarType alpha,
1512 const ValueType *KOKKOS_RESTRICT A,
1513 const ValueType *KOKKOS_RESTRICT B, const int m,
1514 const int n, const int k,
1515 /**/ ValueType *KOKKOS_RESTRICT C) {
1516 if (m <= 0 || n <= 0 || k <= 0) return 0;
1517 if (!(m <= 1 && n <= 1)) Kokkos::abort("InnerGemmFixC<1,1>::serial_invoke, assert failure (m<=1 && n<=1)");
1518
1519 return serial_invoke(alpha, A, B, k, C);
1520 ;
1521 }
1522
1523 } // namespace KokkosBatched
1524
1525 #endif