Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Dot_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_DOT_INTERNAL_HPP
17 #define KOKKOSBATCHED_DOT_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosBlas1_team_dot.hpp"
23
24 namespace KokkosBatched {
25
26 ///
27 /// Serial Internal Impl
28 /// ====================
29
30 struct SerialDotInternal {
31 // i \in [0,m)
32 // C = conj(A(:))*B(:)
33 template <typename ValueType, typename MagnitudeType>
34 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const int m, const ValueType *KOKKOS_RESTRICT A, const int as0,
35 const ValueType *KOKKOS_RESTRICT B, const int bs0,
36 /* */ MagnitudeType *KOKKOS_RESTRICT C) {
37 using ats = Kokkos::ArithTraits<ValueType>;
38 C[0] = ValueType(0);
39 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
40 #pragma unroll
41 #endif
42 for (int i = 0; i < m; ++i) {
43 const int idx_a = i * as0, idx_b = i * bs0;
44 C[0] += ats::conj(A[idx_a]) * B[idx_b];
45 }
46 return 0;
47 }
48
49 // j \in [0,n), i \in [0,m)
50 // C(j) = conj(A(:,j))*B(:,j)
51 template <typename ValueType, typename MagnitudeType>
52 KOKKOS_INLINE_FUNCTION static int invoke(const int m, const int n, const ValueType *KOKKOS_RESTRICT A, const int as0,
53 const int as1, const ValueType *KOKKOS_RESTRICT B, const int bs0,
54 const int bs1,
55 /* */ MagnitudeType *KOKKOS_RESTRICT C, const int cs) {
56 for (int j = 0; j < n; ++j) invoke(m, A + j * as1, as0, B + j * bs1, bs0, C + j * cs);
57 return 0;
58 }
59 };
60
61 ///
62 /// Team Internal Impl
63 /// ========================
64
65 // i \in [0,m)
66 // C = conj(A(:))*B(:)
67 struct TeamDotInternal {
68 template <typename MemberType, typename ValueType, typename MagnitudeType>
69 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const MemberType &member, const int m,
70 const ValueType *KOKKOS_RESTRICT A, const int as0,
71 const ValueType *KOKKOS_RESTRICT B, const int bs0,
72 /* */ MagnitudeType *KOKKOS_RESTRICT C) {
73 using ats = Kokkos::ArithTraits<ValueType>;
74 ValueType t(0);
75 Kokkos::parallel_reduce(
76 Kokkos::TeamThreadRange(member, m),
77 [&](const int &i, ValueType &update) {
78 const int idx_a = i * as0, idx_b = i * bs0;
79 update += ats::conj(A[idx_a]) * B[idx_b];
80 },
81 t);
82 Kokkos::single(Kokkos::PerThread(member), [&]() { C[0] = t; });
83 return 0;
84 }
85
86 // j \in [0,n), i \in [0,m)
87 // C(j) = conj(A(:,j))*B(:,j)
88 template <typename MemberType, typename ValueType, typename MagnitudeType>
89 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const MemberType &member, const int m, const int n,
90 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
91 const ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1,
92 /* */ MagnitudeType *KOKKOS_RESTRICT C, const int cs) {
93 using ats = Kokkos::ArithTraits<ValueType>;
94 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int &j) {
95 ValueType t(0);
96 const ValueType *KOKKOS_RESTRICT A_at_j = A + j * as1;
97 const ValueType *KOKKOS_RESTRICT B_at_j = B + j * bs1;
98 for (int i = 0; i < m; ++i) {
99 const int idx_a = i * as0, idx_b = i * bs0;
100 t += ats::conj(A_at_j[idx_a]) * B_at_j[idx_b];
101 }
102 Kokkos::single(Kokkos::PerThread(member), [&]() { C[j * cs] = t; });
103 });
104 return 0;
105 }
106 };
107
108 ///
109 /// TeamVector Internal Impl
110 /// ========================
111
112 // i \in [0,m)
113 // C = conj(A(:))*B(:)
114 struct TeamVectorDotInternal {
115 template <typename MemberType, typename ValueType, typename MagnitudeType>
116 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const MemberType &member, const int m,
117 const ValueType *KOKKOS_RESTRICT A, const int as0,
118 const ValueType *KOKKOS_RESTRICT B, const int bs0,
119 /* */ MagnitudeType *KOKKOS_RESTRICT C) {
120 using ats = Kokkos::ArithTraits<ValueType>;
121 ValueType t(0);
122 Kokkos::parallel_reduce(
123 Kokkos::TeamVectorRange(member, m),
124 [&](const int &i, ValueType &update) {
125 const int idx_a = i * as0, idx_b = i * bs0;
126 update += ats::conj(A[idx_a]) * B[idx_b];
127 },
128 t);
129 Kokkos::single(Kokkos::PerThread(member), [&]() { C[0] = t; });
130 return 0;
131 }
132
133 // j \in [0,n), i \in [0,m)
134 // C(j) = conj(A(:,j))*B(:,j)
135 template <typename MemberType, typename ValueType, typename MagnitudeType>
136 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const MemberType &member, const int m, const int n,
137 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
138 const ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1,
139 /* */ MagnitudeType *KOKKOS_RESTRICT C, const int cs) {
140 using ats = Kokkos::ArithTraits<ValueType>;
141 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int &j) {
142 ValueType t(0);
143 const ValueType *KOKKOS_RESTRICT A_at_j = A + j * as1;
144 const ValueType *KOKKOS_RESTRICT B_at_j = B + j * bs1;
145 Kokkos::parallel_reduce(
146 Kokkos::ThreadVectorRange(member, m),
147 [&](const int &i, ValueType &update) {
148 const int idx_a = i * as0, idx_b = i * bs0;
149 update += ats::conj(A_at_j[idx_a]) * B_at_j[idx_b];
150 },
151 t);
152 Kokkos::single(Kokkos::PerThread(member), [&]() { C[j * cs] = t; });
153 });
154 return 0;
155 }
156 };
157
158 ///
159 /// Serial Impl
160 /// ===========
161
162 template <>
163 struct SerialDot<Trans::Transpose> {
164 template <typename XViewType, typename YViewType, typename NormViewType>
165 KOKKOS_INLINE_FUNCTION static int invoke(const XViewType &X, const YViewType &Y, const NormViewType &dot) {
166 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
167 static_assert(Kokkos::is_view<XViewType>::value, "KokkosBatched::dot: XViewType is not a Kokkos::View.");
168 static_assert(Kokkos::is_view<YViewType>::value, "KokkosBatched::dot: YViewType is not a Kokkos::View.");
169 static_assert(Kokkos::is_view<NormViewType>::value, "KokkosBatched::dot: NormViewType is not a Kokkos::View.");
170 static_assert(XViewType::rank == 2, "KokkosBatched::dot: XViewType must have rank 2.");
171 static_assert(YViewType::rank == 2, "KokkosBatched::dot: YViewType must have rank 2.");
172 static_assert(NormViewType::rank == 1, "KokkosBatched::dot: NormViewType must have rank 1.");
173
174 // Check compatibility of dimensions at run time.
175 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
176 Kokkos::printf(
177 "KokkosBatched::dot: Dimensions of X and Y do not match: X: %d x %d, "
178 "Y: %d x %d\n",
179 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
180 return 1;
181 }
182 if (X.extent(1) != dot.extent(0)) {
183 Kokkos::printf(
184 "KokkosBatched::dot: Second dimension of X and alpha do not match: "
185 "X: "
186 "%d x %d, dot: %d\n",
187 (int)X.extent(0), (int)X.extent(1), (int)dot.extent(0));
188 return 1;
189 }
190 #endif
191 return SerialDotInternal::template invoke<typename XViewType::non_const_value_type,
192 typename NormViewType::non_const_value_type>(
193 X.extent(0), X.extent(1), X.data(), X.stride_0(), X.stride_1(), Y.data(), Y.stride_0(), Y.stride_1(),
194 dot.data(), dot.stride_0());
195 }
196 };
197
198 template <>
199 struct SerialDot<Trans::NoTranspose> {
200 template <typename XViewType, typename YViewType, typename NormViewType>
201 KOKKOS_INLINE_FUNCTION static int invoke(const XViewType &X, const YViewType &Y, const NormViewType &dot) {
202 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
203 static_assert(Kokkos::is_view<XViewType>::value, "KokkosBatched::dot: XViewType is not a Kokkos::View.");
204 static_assert(Kokkos::is_view<YViewType>::value, "KokkosBatched::dot: YViewType is not a Kokkos::View.");
205 static_assert(Kokkos::is_view<NormViewType>::value, "KokkosBatched::dot: NormViewType is not a Kokkos::View.");
206 static_assert(XViewType::rank == 2, "KokkosBatched::dot: XViewType must have rank 2.");
207 static_assert(YViewType::rank == 2, "KokkosBatched::dot: YViewType must have rank 2.");
208 static_assert(NormViewType::rank == 1, "KokkosBatched::dot: NormViewType must have rank 1.");
209
210 // Check compatibility of dimensions at run time.
211 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
212 Kokkos::printf(
213 "KokkosBatched::dot: Dimensions of X and Y do not match: X: %d x %d, "
214 "Y: %d x %d\n",
215 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
216 return 1;
217 }
218 if (X.extent(0) != dot.extent(0)) {
219 Kokkos::printf(
220 "KokkosBatched::dot: First dimension of X and alpha do not match: X: "
221 "%d x %d, dot: %d\n",
222 (int)X.extent(0), (int)X.extent(1), (int)dot.extent(0));
223 return 1;
224 }
225 #endif
226 return SerialDotInternal::template invoke<typename XViewType::non_const_value_type,
227 typename NormViewType::non_const_value_type>(
228 X.extent(1), X.extent(0), X.data(), X.stride_1(), X.stride_0(), Y.data(), Y.stride_1(), Y.stride_0(),
229 dot.data(), dot.stride_0());
230 }
231 };
232
233 ///
234 /// Team Impl
235 /// ===============
236
237 template <typename MemberType>
238 struct TeamDot<MemberType, Trans::Transpose> {
239 template <typename XViewType, typename YViewType, typename NormViewType>
240 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const XViewType &X, const YViewType &Y,
241 const NormViewType &dot) {
242 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
243 static_assert(Kokkos::is_view<XViewType>::value, "KokkosBatched::dot: XViewType is not a Kokkos::View.");
244 static_assert(Kokkos::is_view<YViewType>::value, "KokkosBatched::dot: YViewType is not a Kokkos::View.");
245 static_assert(Kokkos::is_view<NormViewType>::value, "KokkosBatched::dot: NormViewType is not a Kokkos::View.");
246 static_assert(XViewType::rank == 2, "KokkosBatched::dot: XViewType must have rank 2.");
247 static_assert(YViewType::rank == 2, "KokkosBatched::dot: YViewType must have rank 2.");
248 static_assert(NormViewType::rank == 1, "KokkosBatched::dot: NormViewType must have rank 1.");
249
250 // Check compatibility of dimensions at run time.
251 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
252 Kokkos::printf(
253 "KokkosBatched::dot: Dimensions of X and Y do not match: X: %d x %d, "
254 "Y: %d x %d\n",
255 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
256 return 1;
257 }
258 if (X.extent(1) != dot.extent(0)) {
259 Kokkos::printf(
260 "KokkosBatched::dot: Second dimension of X and alpha do not match: "
261 "X: "
262 "%d x %d, dot: %d\n",
263 (int)X.extent(0), (int)X.extent(1), (int)dot.extent(0));
264 return 1;
265 }
266 #endif
267
268 if (X.extent(1) == 1) {
269 dot(0) =
270 KokkosBlas::Experimental::dot(member, Kokkos::subview(X, Kokkos::ALL, 0), Kokkos::subview(Y, Kokkos::ALL, 0));
271 return 0;
272 }
273
274 return TeamDotInternal::template invoke<MemberType, typename XViewType::non_const_value_type,
275 typename NormViewType::non_const_value_type>(
276 member, X.extent(0), X.extent(1), X.data(), X.stride_0(), X.stride_1(), Y.data(), Y.stride_0(), Y.stride_1(),
277 dot.data(), dot.stride_0());
278 }
279 };
280
281 template <typename MemberType>
282 struct TeamDot<MemberType, Trans::NoTranspose> {
283 template <typename XViewType, typename YViewType, typename NormViewType>
284 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const XViewType &X, const YViewType &Y,
285 const NormViewType &dot) {
286 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
287 static_assert(Kokkos::is_view<XViewType>::value, "KokkosBatched::dot: XViewType is not a Kokkos::View.");
288 static_assert(Kokkos::is_view<YViewType>::value, "KokkosBatched::dot: YViewType is not a Kokkos::View.");
289 static_assert(Kokkos::is_view<NormViewType>::value, "KokkosBatched::dot: NormViewType is not a Kokkos::View.");
290 static_assert(XViewType::rank == 2, "KokkosBatched::dot: XViewType must have rank 2.");
291 static_assert(YViewType::rank == 2, "KokkosBatched::dot: YViewType must have rank 2.");
292 static_assert(NormViewType::rank == 1, "KokkosBatched::dot: NormViewType must have rank 1.");
293
294 // Check compatibility of dimensions at run time.
295 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
296 Kokkos::printf(
297 "KokkosBatched::dot: Dimensions of X and Y do not match: X: %d x %d, "
298 "Y: %d x %d\n",
299 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
300 return 1;
301 }
302 if (X.extent(0) != dot.extent(0)) {
303 Kokkos::printf(
304 "KokkosBatched::dot: First dimension of X and alpha do not match: X: "
305 "%d x %d, dot: %d\n",
306 (int)X.extent(0), (int)X.extent(1), (int)dot.extent(0));
307 return 1;
308 }
309 #endif
310
311 if (X.extent(0) == 1) {
312 dot(0) =
313 KokkosBlas::Experimental::dot(member, Kokkos::subview(X, 0, Kokkos::ALL), Kokkos::subview(Y, 0, Kokkos::ALL));
314 return 0;
315 }
316
317 return TeamDotInternal::template invoke<MemberType, typename XViewType::non_const_value_type,
318 typename NormViewType::non_const_value_type>(
319 member, X.extent(1), X.extent(0), X.data(), X.stride_1(), X.stride_0(), Y.data(), Y.stride_1(), Y.stride_0(),
320 dot.data(), dot.stride_0());
321 }
322 };
323
324 ///
325 /// TeamVector Impl
326 /// ===============
327
328 template <typename MemberType>
329 struct TeamVectorDot<MemberType, Trans::Transpose> {
330 template <typename XViewType, typename YViewType, typename NormViewType>
331 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const XViewType &X, const YViewType &Y,
332 const NormViewType &dot) {
333 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
334 static_assert(Kokkos::is_view<XViewType>::value, "KokkosBatched::dot: XViewType is not a Kokkos::View.");
335 static_assert(Kokkos::is_view<YViewType>::value, "KokkosBatched::dot: YViewType is not a Kokkos::View.");
336 static_assert(Kokkos::is_view<NormViewType>::value, "KokkosBatched::dot: NormViewType is not a Kokkos::View.");
337 static_assert(XViewType::rank == 2, "KokkosBatched::dot: XViewType must have rank 2.");
338 static_assert(YViewType::rank == 2, "KokkosBatched::dot: YViewType must have rank 2.");
339 static_assert(NormViewType::rank == 1, "KokkosBatched::dot: NormViewType must have rank 1.");
340
341 // Check compatibility of dimensions at run time.
342 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
343 Kokkos::printf(
344 "KokkosBatched::dot: Dimensions of X and Y do not match: X: %d x %d, "
345 "Y: %d x %d\n",
346 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
347 return 1;
348 }
349 if (X.extent(1) != dot.extent(0)) {
350 Kokkos::printf(
351 "KokkosBatched::dot: Second dimension of X and alpha do not match: "
352 "X: "
353 "%d x %d, dot: %d\n",
354 (int)X.extent(0), (int)X.extent(1), (int)dot.extent(0));
355 return 1;
356 }
357 #endif
358
359 if (X.extent(1) == 1) {
360 dot(0) =
361 KokkosBlas::Experimental::dot(member, Kokkos::subview(X, Kokkos::ALL, 0), Kokkos::subview(Y, Kokkos::ALL, 0));
362 return 0;
363 }
364
365 return TeamVectorDotInternal::template invoke<MemberType, typename XViewType::non_const_value_type,
366 typename NormViewType::non_const_value_type>(
367 member, X.extent(0), X.extent(1), X.data(), X.stride_0(), X.stride_1(), Y.data(), Y.stride_0(), Y.stride_1(),
368 dot.data(), dot.stride_0());
369 }
370 };
371
372 template <typename MemberType>
373 struct TeamVectorDot<MemberType, Trans::NoTranspose> {
374 template <typename XViewType, typename YViewType, typename NormViewType>
375 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const XViewType &X, const YViewType &Y,
376 const NormViewType &dot) {
377 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
378 static_assert(Kokkos::is_view<XViewType>::value, "KokkosBatched::dot: XViewType is not a Kokkos::View.");
379 static_assert(Kokkos::is_view<YViewType>::value, "KokkosBatched::dot: YViewType is not a Kokkos::View.");
380 static_assert(Kokkos::is_view<NormViewType>::value, "KokkosBatched::dot: NormViewType is not a Kokkos::View.");
381 static_assert(XViewType::rank == 2, "KokkosBatched::dot: XViewType must have rank 2.");
382 static_assert(YViewType::rank == 2, "KokkosBatched::dot: YViewType must have rank 2.");
383 static_assert(NormViewType::rank == 1, "KokkosBatched::dot: NormViewType must have rank 1.");
384
385 // Check compatibility of dimensions at run time.
386 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
387 Kokkos::printf(
388 "KokkosBatched::dot: Dimensions of X and Y do not match: X: %d x %d, "
389 "Y: %d x %d\n",
390 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
391 return 1;
392 }
393 if (X.extent(0) != dot.extent(0)) {
394 Kokkos::printf(
395 "KokkosBatched::dot: First dimension of X and alpha do not match: X: "
396 "%d x %d, dot: %d\n",
397 (int)X.extent(0), (int)X.extent(1), (int)dot.extent(0));
398 return 1;
399 }
400 #endif
401
402 if (X.extent(0) == 1) {
403 dot(0) =
404 KokkosBlas::Experimental::dot(member, Kokkos::subview(X, 0, Kokkos::ALL), Kokkos::subview(Y, 0, Kokkos::ALL));
405 return 0;
406 }
407
408 return TeamVectorDotInternal::template invoke<MemberType, typename XViewType::non_const_value_type,
409 typename NormViewType::non_const_value_type>(
410 member, X.extent(1), X.extent(0), X.data(), X.stride_1(), X.stride_0(), Y.data(), Y.stride_1(), Y.stride_0(),
411 dot.data(), dot.stride_0());
412 }
413 };
414
415 } // end namespace KokkosBatched
416
417 #endif