Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_HostLevel_Gemm_Spec.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_SPEC_HPP
17 #define KOKKOSBATCHED_HOSTLEVEL_GEMM_SPEC_HPP
18
19 #include <Kokkos_Core.hpp>
20 #include <KokkosKernels_config.h>
21 #include <KokkosBatched_HostLevel_Gemm_Handle.hpp> // BatchedGemmHandle
22
23 #if !defined(KOKKOSKERNELS_ETI_ONLY) || KOKKOSKERNELS_IMPL_COMPILE_LIBRARY
24 #include "KokkosBatched_HostLevel_Gemm_Impl.hpp"
25 #include "KokkosKernels_ExecSpaceUtils.hpp"
26 #endif
27
28 namespace KokkosBatched {
29 namespace Impl {
30 // Specialization struct which defines whether a specialization exists
31 // This struct is currently never specialized.
32 template <class ArgTransA, class ArgTransB, class ArgBatchSzDim, class BatchedGemmHandleType, class ScalarType,
33 class AViewType, class BViewType, class CViewType>
34 struct batched_gemm_tpl_spec_avail {
35 enum : bool { value = false };
36 };
37
38 // Specialization struct which defines whether a specialization exists
39 template <class ArgTransA, class ArgTransB, class ArgBatchSzDim, class BatchedGemmHandleType, class ScalarType,
40 class AViewType, class BViewType, class CViewType>
41 struct batched_gemm_eti_spec_avail {
42 enum : bool { value = false };
43 };
44 } // namespace Impl
45 } // namespace KokkosBatched
46
47 // ETI specalization macros, consumed by generated *_eti_spec_avail.hpp files
48 #define KOKKOSBATCHED_GEMM_ETI_SPEC_AVAIL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
49 EXEC_SPACE, MEM_SPACE) \
50 template <> \
51 struct batched_gemm_eti_spec_avail<ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, BatchedGemmHandle, SCALAR, \
52 Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
53 Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
54 Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
55 Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
56 Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
57 Kokkos::MemoryTraits<Kokkos::Unmanaged>>> { \
58 enum : bool { value = true }; \
59 };
60
61 #if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT)
62 #define KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_AVAIL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
63 EXEC_SPACE, MEM_SPACE) \
64 KOKKOSBATCHED_GEMM_ETI_SPEC_AVAIL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, Kokkos::LayoutRight, \
65 EXEC_SPACE, MEM_SPACE)
66 #else
67 #define KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_AVAIL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
68 EXEC_SPACE, MEM_SPACE)
69 #endif
70
71 #if defined(KOKKOSKERNELS_INST_LAYOUTLEFT)
72 #define KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_AVAIL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
73 EXEC_SPACE, MEM_SPACE) \
74 KOKKOSBATCHED_GEMM_ETI_SPEC_AVAIL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, Kokkos::LayoutLeft, \
75 EXEC_SPACE, MEM_SPACE)
76 #else
77 #define KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_AVAIL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
78 EXEC_SPACE, MEM_SPACE)
79 #endif
80
81 ///////////////// BatchLayout::Left Permutations /////////////////
82 #define KOKKOSBATCHED_GEMM_NT_NT_BLL_ETI_SPEC_AVAIL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
83 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_AVAIL_INNER(Trans::NoTranspose, Trans::NoTranspose, BatchLayout::Left, SCALAR, \
84 LAYOUT, EXEC_SPACE, MEM_SPACE)
85
86 #define KOKKOSBATCHED_GEMM_NT_T_BLL_ETI_SPEC_AVAIL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
87 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_AVAIL_INNER(Trans::NoTranspose, Trans::Transpose, BatchLayout::Left, SCALAR, LAYOUT, \
88 EXEC_SPACE, MEM_SPACE)
89
90 #define KOKKOSBATCHED_GEMM_T_NT_BLL_ETI_SPEC_AVAIL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
91 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_AVAIL_INNER(Trans::Transpose, Trans::NoTranspose, BatchLayout::Left, SCALAR, LAYOUT, \
92 EXEC_SPACE, MEM_SPACE)
93
94 #define KOKKOSBATCHED_GEMM_T_T_BLL_ETI_SPEC_AVAIL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
95 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_AVAIL_INNER(Trans::Transpose, Trans::Transpose, BatchLayout::Left, SCALAR, LAYOUT, \
96 EXEC_SPACE, MEM_SPACE)
97
98 // Include the BLL ETI specalizations
99 #include <generated_specializations_hpp/KokkosBatched_Gemm_nt_nt_bll_eti_spec_avail.hpp>
100 #include <generated_specializations_hpp/KokkosBatched_Gemm_nt_t_bll_eti_spec_avail.hpp>
101 #include <generated_specializations_hpp/KokkosBatched_Gemm_t_nt_bll_eti_spec_avail.hpp>
102 #include <generated_specializations_hpp/KokkosBatched_Gemm_t_t_bll_eti_spec_avail.hpp>
103
104 ///////////////// BatchLayout::Right Permutations /////////////////
105 #define KOKKOSBATCHED_GEMM_NT_NT_BLR_ETI_SPEC_AVAIL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
106 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_AVAIL_INNER(Trans::NoTranspose, Trans::NoTranspose, BatchLayout::Right, SCALAR, \
107 LAYOUT, EXEC_SPACE, MEM_SPACE)
108
109 #define KOKKOSBATCHED_GEMM_NT_T_BLR_ETI_SPEC_AVAIL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
110 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_AVAIL_INNER(Trans::NoTranspose, Trans::Transpose, BatchLayout::Right, SCALAR, \
111 LAYOUT, EXEC_SPACE, MEM_SPACE)
112
113 #define KOKKOSBATCHED_GEMM_T_NT_BLR_ETI_SPEC_AVAIL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
114 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_AVAIL_INNER(Trans::Transpose, Trans::NoTranspose, BatchLayout::Right, SCALAR, \
115 LAYOUT, EXEC_SPACE, MEM_SPACE)
116
117 #define KOKKOSBATCHED_GEMM_T_T_BLR_ETI_SPEC_AVAIL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
118 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_AVAIL_INNER(Trans::Transpose, Trans::Transpose, BatchLayout::Right, SCALAR, LAYOUT, \
119 EXEC_SPACE, MEM_SPACE)
120
121 // Include the BLR ETI specalizations
122 #include <generated_specializations_hpp/KokkosBatched_Gemm_nt_nt_blr_eti_spec_avail.hpp>
123 #include <generated_specializations_hpp/KokkosBatched_Gemm_nt_t_blr_eti_spec_avail.hpp>
124 #include <generated_specializations_hpp/KokkosBatched_Gemm_t_nt_blr_eti_spec_avail.hpp>
125 #include <generated_specializations_hpp/KokkosBatched_Gemm_t_t_blr_eti_spec_avail.hpp>
126
127 namespace KokkosBatched {
128 namespace Impl {
129 template <class ArgTransA, class ArgTransB, class ArgBatchSzDim, class BatchedGemmHandleType, class ScalarType,
130 class AViewType, class BViewType, class CViewType,
131 bool tpl_spec_avail = batched_gemm_tpl_spec_avail<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType,
132 ScalarType, AViewType, BViewType, CViewType>::value,
133 bool eti_spec_avail = batched_gemm_eti_spec_avail<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType,
134 ScalarType, AViewType, BViewType, CViewType>::value>
135 struct BatchedGemmSpec {
136 static int run(BatchedGemmHandleType *const handle, const ScalarType alpha, const AViewType &A, const BViewType &B,
137 const ScalarType beta, const CViewType &C)
138 #if !defined(KOKKOSKERNELS_ETI_ONLY) || KOKKOSKERNELS_IMPL_COMPILE_LIBRARY
139 {
140 #ifdef KOKKOSKERNELS_ENABLE_CHECK_SPECIALIZATION
141 #if KOKKOSKERNELS_IMPL_COMPILE_LIBRARY
142 printf(
143 "KokkosBatched::BatchedGemm<> ETI specialization for < %s, %s, %s, "
144 "%s, %s, %s, %s, %s >\n",
145 typeid(ArgTransA).name(), typeid(ArgTransB).name(), typeid(ArgBatchSzDim).name(),
146 typeid(BatchedGemmHandleType).name(), typeid(ScalarType).name(), typeid(AViewType).name(),
147 typeid(BViewType).name(), typeid(CViewType).name());
148 #else
149 printf(
150 "KokkosBatched::BatchedGemm<> non-ETI specialization for < %s, %s, "
151 "%s, %s, %s, %s, %s, %s >\n",
152 typeid(ArgTransA).name(), typeid(ArgTransB).name(), typeid(ArgBatchSzDim).name(),
153 typeid(BatchedGemmHandleType).name(), typeid(ScalarType).name(), typeid(AViewType).name(),
154 typeid(BViewType).name(), typeid(CViewType).name());
155 #endif // KOKKOSKERNELS_IMPL_COMPILE_LIBRARY
156 #endif // KOKKOSKERNELS_ENABLE_CHECK_SPECIALIZATION
157 return KokkosBatched::Impl::BatchedGemmImpl<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType,
158 AViewType, BViewType, CViewType>(handle, alpha, A, B, beta, C);
159 }
160 #else
161 ;
162 #endif // !defined(KOKKOSKERNELS_ETI_ONLY) ||
163 // KOKKOSKERNELS_IMPL_COMPILE_LIBRARY
164 };
165 } // namespace Impl
166 } // namespace KokkosBatched
167
168 // ETI instantiation macros, consumed by *.cpp.in files
169 #define KOKKOSBATCHED_GEMM_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, EXEC_SPACE, \
170 MEM_SPACE) \
171 extern template struct BatchedGemmSpec<ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, BatchedGemmHandle, SCALAR, \
172 Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
173 Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
174 Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
175 Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
176 Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
177 Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
178 false, true>;
179
180 #if defined(KOKKOSKERNELS_DECL_LAYOUTRIGHT)
181 #define KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
182 EXEC_SPACE, MEM_SPACE) \
183 KOKKOSBATCHED_GEMM_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, Kokkos::LayoutRight, \
184 EXEC_SPACE, MEM_SPACE)
185 #else
186 #define KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
187 EXEC_SPACE, MEM_SPACE)
188 #endif
189
190 #if defined(KOKKOSKERNELS_DECL_LAYOUTLEFT)
191 #define KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
192 EXEC_SPACE, MEM_SPACE) \
193 KOKKOSBATCHED_GEMM_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, Kokkos::LayoutLeft, \
194 EXEC_SPACE, MEM_SPACE)
195 #else
196 #define KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
197 EXEC_SPACE, MEM_SPACE)
198 #endif
199
200 ///////////////// BatchLayout::Left Permutations /////////////////
201 #define KOKKOSBATCHED_GEMM_NT_NT_BLL_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
202 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(Trans::NoTranspose, Trans::NoTranspose, BatchLayout::Left, SCALAR, \
203 LAYOUT, EXEC_SPACE, MEM_SPACE)
204
205 #define KOKKOSBATCHED_GEMM_NT_T_BLL_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
206 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(Trans::NoTranspose, Trans::Transpose, BatchLayout::Left, SCALAR, LAYOUT, \
207 EXEC_SPACE, MEM_SPACE)
208
209 #define KOKKOSBATCHED_GEMM_T_NT_BLL_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
210 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(Trans::Transpose, Trans::NoTranspose, BatchLayout::Left, SCALAR, LAYOUT, \
211 EXEC_SPACE, MEM_SPACE)
212
213 #define KOKKOSBATCHED_GEMM_T_T_BLL_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
214 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(Trans::Transpose, Trans::Transpose, BatchLayout::Left, SCALAR, LAYOUT, \
215 EXEC_SPACE, MEM_SPACE)
216
217 ///////////////// BatchLayout::Right Permutations /////////////////
218 #define KOKKOSBATCHED_GEMM_NT_NT_BLR_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
219 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(Trans::NoTranspose, Trans::NoTranspose, BatchLayout::Right, SCALAR, \
220 LAYOUT, EXEC_SPACE, MEM_SPACE)
221
222 #define KOKKOSBATCHED_GEMM_NT_T_BLR_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
223 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(Trans::NoTranspose, Trans::Transpose, BatchLayout::Right, SCALAR, LAYOUT, \
224 EXEC_SPACE, MEM_SPACE)
225
226 #define KOKKOSBATCHED_GEMM_T_NT_BLR_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
227 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(Trans::Transpose, Trans::NoTranspose, BatchLayout::Right, SCALAR, LAYOUT, \
228 EXEC_SPACE, MEM_SPACE)
229
230 #define KOKKOSBATCHED_GEMM_T_T_BLR_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
231 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(Trans::Transpose, Trans::Transpose, BatchLayout::Right, SCALAR, LAYOUT, \
232 EXEC_SPACE, MEM_SPACE)
233
234 // ETI instantiation macros, consumed by *.cpp.in files
235 #define KOKKOSBATCHED_GEMM_ETI_SPEC_INST_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, EXEC_SPACE, \
236 MEM_SPACE) \
237 template struct BatchedGemmSpec<ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, BatchedGemmHandle, SCALAR, \
238 Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
239 Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
240 Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
241 Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
242 Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
243 Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
244 false, true>;
245
246 #if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT)
247 #define KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_INST_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
248 EXEC_SPACE, MEM_SPACE) \
249 KOKKOSBATCHED_GEMM_ETI_SPEC_INST_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, Kokkos::LayoutRight, \
250 EXEC_SPACE, MEM_SPACE)
251 #else
252 #define KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_INST_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
253 EXEC_SPACE, MEM_SPACE)
254 #endif
255
256 #if defined(KOKKOSKERNELS_INST_LAYOUTLEFT)
257 #define KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_INST_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
258 EXEC_SPACE, MEM_SPACE) \
259 KOKKOSBATCHED_GEMM_ETI_SPEC_INST_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, Kokkos::LayoutLeft, \
260 EXEC_SPACE, MEM_SPACE)
261 #else
262 #define KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_INST_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
263 EXEC_SPACE, MEM_SPACE)
264 #endif
265
266 ///////////////// BatchLayout::Left Permutations /////////////////
267 #define KOKKOSBATCHED_GEMM_NT_NT_BLL_ETI_SPEC_INST(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
268 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_INST_INNER(Trans::NoTranspose, Trans::NoTranspose, BatchLayout::Left, SCALAR, \
269 LAYOUT, EXEC_SPACE, MEM_SPACE)
270
271 #define KOKKOSBATCHED_GEMM_NT_T_BLL_ETI_SPEC_INST(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
272 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_INST_INNER(Trans::NoTranspose, Trans::Transpose, BatchLayout::Left, SCALAR, LAYOUT, \
273 EXEC_SPACE, MEM_SPACE)
274
275 #define KOKKOSBATCHED_GEMM_T_NT_BLL_ETI_SPEC_INST(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
276 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_INST_INNER(Trans::Transpose, Trans::NoTranspose, BatchLayout::Left, SCALAR, LAYOUT, \
277 EXEC_SPACE, MEM_SPACE)
278
279 #define KOKKOSBATCHED_GEMM_T_T_BLL_ETI_SPEC_INST(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
280 KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_INST_INNER(Trans::Transpose, Trans::Transpose, BatchLayout::Left, SCALAR, LAYOUT, \
281 EXEC_SPACE, MEM_SPACE)
282
283 ///////////////// BatchLayout::Right Permutations /////////////////
284 #define KOKKOSBATCHED_GEMM_NT_NT_BLR_ETI_SPEC_INST(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
285 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_INST_INNER(Trans::NoTranspose, Trans::NoTranspose, BatchLayout::Right, SCALAR, \
286 LAYOUT, EXEC_SPACE, MEM_SPACE)
287
288 #define KOKKOSBATCHED_GEMM_NT_T_BLR_ETI_SPEC_INST(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
289 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_INST_INNER(Trans::NoTranspose, Trans::Transpose, BatchLayout::Right, SCALAR, LAYOUT, \
290 EXEC_SPACE, MEM_SPACE)
291
292 #define KOKKOSBATCHED_GEMM_T_NT_BLR_ETI_SPEC_INST(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
293 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_INST_INNER(Trans::Transpose, Trans::NoTranspose, BatchLayout::Right, SCALAR, LAYOUT, \
294 EXEC_SPACE, MEM_SPACE)
295
296 #define KOKKOSBATCHED_GEMM_T_T_BLR_ETI_SPEC_INST(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
297 KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_INST_INNER(Trans::Transpose, Trans::Transpose, BatchLayout::Right, SCALAR, LAYOUT, \
298 EXEC_SPACE, MEM_SPACE)
299
300 #include <generated_specializations_hpp/KokkosBatched_Gemm_nt_nt_bll_eti_spec_avail.hpp>
301 #include <generated_specializations_hpp/KokkosBatched_Gemm_nt_t_bll_eti_spec_avail.hpp>
302 #include <generated_specializations_hpp/KokkosBatched_Gemm_t_nt_bll_eti_spec_avail.hpp>
303 #include <generated_specializations_hpp/KokkosBatched_Gemm_t_t_bll_eti_spec_avail.hpp>
304
305 #include <generated_specializations_hpp/KokkosBatched_Gemm_nt_nt_blr_eti_spec_avail.hpp>
306 #include <generated_specializations_hpp/KokkosBatched_Gemm_nt_t_blr_eti_spec_avail.hpp>
307 #include <generated_specializations_hpp/KokkosBatched_Gemm_t_nt_blr_eti_spec_avail.hpp>
308 #include <generated_specializations_hpp/KokkosBatched_Gemm_t_t_blr_eti_spec_avail.hpp>
309
310 #endif // KOKKOSBATCHED_HOSTLEVEL_GEMM_SPEC_HPP