Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Gemm_Serial_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_GEMM_SERIAL_IMPL_HPP
17 #define KOKKOSBATCHED_GEMM_SERIAL_IMPL_HPP
18
19 #include "KokkosBatched_Util.hpp"
20 #include "KokkosBatched_Gemm_Serial_Internal.hpp"
21
22 namespace KokkosBatched {
23 ///
24 /// Serial Impl
25 /// ===========
26
27 ///
28 /// Implemented:
29 /// NT/NT, T/NT, NT/T, T/T
30 ///
31 /// Not yet immplemented (ConjTranspose):
32 /// CT/NT, NT/CT, CT/CT
33 ///
34
35 ///
36 /// NT/NT
37 ///
38
39 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
40 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
41 template <>
42 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
43 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::NoTranspose, Trans::NoTranspose, Algo::Gemm::CompactMKL>::invoke(
44 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
45 typedef typename CViewType::value_type vector_type;
46 // typedef typename vector_type::value_type value_type;
47
48 const int m = C.extent(0), n = C.extent(1), k = A.extent(1);
49
50 static_assert(is_vector<vector_type>::value, "value type is not vector type");
51 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
52 "AVX, AVX2 and AVX512 is supported");
53 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
54
55 // no error check
56 int r_val = 0;
57 if (A.stride_0() == 1 && B.stride_0() == 1 && C.stride_0() == 1) {
58 mkl_dgemm_compact(MKL_COL_MAJOR, MKL_NOTRANS, MKL_NOTRANS, m, n, k, alpha, (const double *)A.data(), A.stride_1(),
59 (const double *)B.data(), B.stride_1(), beta, (double *)C.data(), C.stride_1(), format,
60 (MKL_INT)vector_type::vector_length);
61 } else if (A.stride_1() == 1 && B.stride_1() == 1 && C.stride_1() == 1) {
62 mkl_dgemm_compact(MKL_ROW_MAJOR, MKL_NOTRANS, MKL_NOTRANS, m, n, k, alpha, (const double *)A.data(), A.stride_0(),
63 (const double *)B.data(), B.stride_0(), beta, (double *)C.data(), C.stride_0(), format,
64 (MKL_INT)vector_type::vector_length);
65 } else {
66 r_val = -1;
67 }
68 return r_val;
69 }
70 #endif
71
72 template <>
73 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
74 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::NoTranspose, Trans::NoTranspose, Algo::Gemm::Unblocked>::invoke(
75 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
76 // C = beta C + alpha A B
77 // C (m x n), A(m x k), B(k x n)
78 return SerialGemmInternal<Algo::Gemm::Unblocked>::invoke(C.extent(0), C.extent(1), A.extent(1), alpha, A.data(),
79 A.stride_0(), A.stride_1(), B.data(), B.stride_0(),
80 B.stride_1(), beta, C.data(), C.stride_0(), C.stride_1());
81 }
82
83 template <>
84 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
85 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::NoTranspose, Trans::NoTranspose, Algo::Gemm::Blocked>::invoke(
86 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
87 // C = beta C + alpha A B
88 // C (m x n), A(m x k), B(k x n)
89 return SerialGemmInternal<Algo::Gemm::Blocked>::invoke(C.extent(0), C.extent(1), A.extent(1), alpha, A.data(),
90 A.stride_0(), A.stride_1(), B.data(), B.stride_0(),
91 B.stride_1(), beta, C.data(), C.stride_0(), C.stride_1());
92 }
93
94 ///
95 /// T/NT
96 ///
97
98 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
99 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
100 template <>
101 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
102 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::Transpose, Trans::NoTranspose, Algo::Gemm::CompactMKL>::invoke(
103 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
104 typedef typename CViewType::value_type vector_type;
105 // typedef typename vector_type::value_type value_type;
106
107 const int m = C.extent(0), n = C.extent(1), k = A.extent(0);
108
109 static_assert(is_vector<vector_type>::value, "value type is not vector type");
110 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
111 "AVX, AVX2 and AVX512 is supported");
112 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
113
114 // no error check
115 int r_val = 0;
116 if (A.stride_0() == 1 && B.stride_0() == 1 && C.stride_0() == 1) {
117 mkl_dgemm_compact(MKL_COL_MAJOR, MKL_TRANS, MKL_NOTRANS, m, n, k, alpha, (const double *)A.data(), A.stride_1(),
118 (const double *)B.data(), B.stride_1(), beta, (double *)C.data(), C.stride_1(), format,
119 (MKL_INT)vector_type::vector_length);
120 } else if (A.stride_1() == 1 && B.stride_1() == 1 && C.stride_1() == 1) {
121 mkl_dgemm_compact(MKL_ROW_MAJOR, MKL_TRANS, MKL_NOTRANS, m, n, k, alpha, (const double *)A.data(), A.stride_0(),
122 (const double *)B.data(), B.stride_0(), beta, (double *)C.data(), C.stride_0(), format,
123 (MKL_INT)vector_type::vector_length);
124 } else {
125 r_val = -1;
126 }
127 return r_val;
128 }
129 #endif
130
131 template <>
132 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
133 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::Transpose, Trans::NoTranspose, Algo::Gemm::Unblocked>::invoke(
134 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
135 // C = beta C + alpha A B
136 // C (m x n), A(m x k), B(k x n)
137 return SerialGemmInternal<Algo::Gemm::Unblocked>::invoke(C.extent(0), C.extent(1), A.extent(0), alpha, A.data(),
138 A.stride_1(), A.stride_0(), B.data(), B.stride_0(),
139 B.stride_1(), beta, C.data(), C.stride_0(), C.stride_1());
140 }
141
142 template <>
143 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
144 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::Transpose, Trans::NoTranspose, Algo::Gemm::Blocked>::invoke(
145 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
146 // C = beta C + alpha A B
147 // C (m x n), A(m x k), B(k x n)
148 return SerialGemmInternal<Algo::Gemm::Blocked>::invoke(C.extent(0), C.extent(1), A.extent(0), alpha, A.data(),
149 A.stride_1(), A.stride_0(), B.data(), B.stride_0(),
150 B.stride_1(), beta, C.data(), C.stride_0(), C.stride_1());
151 }
152
153 ///
154 /// NT/T
155 ///
156
157 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
158 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
159 template <>
160 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
161 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::NoTranspose, Trans::Transpose, Algo::Gemm::CompactMKL>::invoke(
162 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
163 typedef typename CViewType::value_type vector_type;
164 // typedef typename vector_type::value_type value_type;
165
166 const int m = C.extent(0), n = C.extent(1), k = A.extent(1);
167
168 static_assert(is_vector<vector_type>::value, "value type is not vector type");
169 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
170 "AVX, AVX2 and AVX512 is supported");
171 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
172
173 // no error check
174 int r_val = 0;
175 if (A.stride_0() == 1 && B.stride_0() == 1 && C.stride_0() == 1) {
176 mkl_dgemm_compact(MKL_COL_MAJOR, MKL_NOTRANS, MKL_TRANS, m, n, k, alpha, (const double *)A.data(), A.stride_1(),
177 (const double *)B.data(), B.stride_1(), beta, (double *)C.data(), C.stride_1(), format,
178 (MKL_INT)vector_type::vector_length);
179 } else if (A.stride_1() == 1 && B.stride_1() == 1 && C.stride_1() == 1) {
180 mkl_dgemm_compact(MKL_ROW_MAJOR, MKL_NOTRANS, MKL_TRANS, m, n, k, alpha, (const double *)A.data(), A.stride_0(),
181 (const double *)B.data(), B.stride_0(), beta, (double *)C.data(), C.stride_0(), format,
182 (MKL_INT)vector_type::vector_length);
183 } else {
184 r_val = -1;
185 }
186 return r_val;
187 }
188 #endif
189
190 template <>
191 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
192 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::NoTranspose, Trans::Transpose, Algo::Gemm::Unblocked>::invoke(
193 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
194 // C = beta C + alpha A B
195 // C (m x n), A(m x k), B(k x n)
196 return SerialGemmInternal<Algo::Gemm::Unblocked>::invoke(C.extent(0), C.extent(1), A.extent(1), alpha, A.data(),
197 A.stride_0(), A.stride_1(), B.data(), B.stride_1(),
198 B.stride_0(), beta, C.data(), C.stride_0(), C.stride_1());
199 }
200
201 template <>
202 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
203 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::NoTranspose, Trans::Transpose, Algo::Gemm::Blocked>::invoke(
204 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
205 // C = beta C + alpha A B
206 // C (m x n), A(m x k), B(k x n)
207 return SerialGemmInternal<Algo::Gemm::Blocked>::invoke(C.extent(0), C.extent(1), A.extent(1), alpha, A.data(),
208 A.stride_0(), A.stride_1(), B.data(), B.stride_1(),
209 B.stride_0(), beta, C.data(), C.stride_0(), C.stride_1());
210 }
211
212 ///
213 /// T/T
214 ///
215
216 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
217 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
218 template <>
219 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
220 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::Transpose, Trans::Transpose, Algo::Gemm::CompactMKL>::invoke(
221 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
222 typedef typename CViewType::value_type vector_type;
223 // typedef typename vector_type::value_type value_type;
224
225 const int m = C.extent(0), n = C.extent(1), k = A.extent(0);
226
227 static_assert(is_vector<vector_type>::value, "value type is not vector type");
228 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
229 "AVX, AVX2 and AVX512 is supported");
230 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
231
232 // no error check
233 int r_val = 0;
234 if (A.stride_0() == 1 && B.stride_0() == 1 && C.stride_0() == 1) {
235 mkl_dgemm_compact(MKL_COL_MAJOR, MKL_TRANS, MKL_TRANS, m, n, k, alpha, (const double *)A.data(), A.stride_1(),
236 (const double *)B.data(), B.stride_1(), beta, (double *)C.data(), C.stride_1(), format,
237 (MKL_INT)vector_type::vector_length);
238 } else if (A.stride_1() == 1 && B.stride_1() == 1 && C.stride_1() == 1) {
239 mkl_dgemm_compact(MKL_ROW_MAJOR, MKL_TRANS, MKL_TRANS, m, n, k, alpha, (const double *)A.data(), A.stride_0(),
240 (const double *)B.data(), B.stride_0(), beta, (double *)C.data(), C.stride_0(), format,
241 (MKL_INT)vector_type::vector_length);
242 } else {
243 r_val = -1;
244 }
245 return r_val;
246 }
247 #endif
248
249 template <>
250 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
251 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::Transpose, Trans::Transpose, Algo::Gemm::Unblocked>::invoke(
252 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
253 // C = beta C + alpha A B
254 // C (m x n), A(m x k), B(k x n)
255 return SerialGemmInternal<Algo::Gemm::Unblocked>::invoke(C.extent(0), C.extent(1), A.extent(0), alpha, A.data(),
256 A.stride_1(), A.stride_0(), B.data(), B.stride_1(),
257 B.stride_0(), beta, C.data(), C.stride_0(), C.stride_1());
258 }
259
260 template <>
261 template <typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
262 KOKKOS_INLINE_FUNCTION int SerialGemm<Trans::Transpose, Trans::Transpose, Algo::Gemm::Blocked>::invoke(
263 const ScalarType alpha, const AViewType &A, const BViewType &B, const ScalarType beta, const CViewType &C) {
264 // C = beta C + alpha A B
265 // C (m x n), A(m x k), B(k x n)
266 return SerialGemmInternal<Algo::Gemm::Blocked>::invoke(C.extent(0), C.extent(1), A.extent(0), alpha, A.data(),
267 A.stride_1(), A.stride_0(), B.data(), B.stride_1(),
268 B.stride_0(), beta, C.data(), C.stride_0(), C.stride_1());
269 }
270 } // namespace KokkosBatched
271
272 #endif