Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Copy_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_COPY_INTERNAL_HPP
17 #define KOKKOSBATCHED_COPY_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 namespace KokkosBatched {
24
25 ///
26 /// Serial Internal Impl
27 /// ====================
28
29 struct SerialCopyInternal {
30 template <typename ValueType>
31 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const int m, const ValueType *KOKKOS_RESTRICT A, const int as0,
32 /* */ ValueType *KOKKOS_RESTRICT B, const int bs0) {
33 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
34 #pragma unroll
35 #endif
36 for (int i = 0; i < m; ++i) B[i * bs0] = A[i * as0];
37
38 return 0;
39 }
40 template <typename ValueType>
41 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const int m, const int n, const ValueType *KOKKOS_RESTRICT A,
42 const int as0, const int as1,
43 /* */ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
44 if (as1 < as0)
45 for (int i = 0; i < m; ++i) invoke(n, A + i * as0, as1, B + i * bs0, bs1);
46 else
47 for (int j = 0; j < n; ++j) invoke(m, A + j * as1, as0, B + j * bs1, bs0);
48 return 0;
49 }
50 };
51
52 ///
53 /// Team Internal Impl
54 /// ==================
55 struct TeamCopyInternal {
56 template <typename MemberType, typename ValueType>
57 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const MemberType &member, const int m,
58 const ValueType *KOKKOS_RESTRICT A, const int as0,
59 /* */ ValueType *KOKKOS_RESTRICT B, const int bs0) {
60 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m), [&](const int &i) { B[i * bs0] = A[i * as0]; });
61 // member.team_barrier();
62 return 0;
63 }
64 template <typename MemberType, typename ValueType>
65 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const MemberType &member, const int m, const int n,
66 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
67 /* */ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
68 if (m >= n) {
69 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m),
70 [&](const int &i) { SerialCopyInternal::invoke(n, A + i * as0, as1, B + i * bs0, bs1); });
71 } else {
72 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n),
73 [&](const int &j) { SerialCopyInternal::invoke(m, A + j * as1, as0, B + j * bs1, bs0); });
74 }
75 // member.team_barrier();
76 return 0;
77 }
78 };
79
80 ///
81 /// TeamVector Internal Impl
82 /// ========================
83 struct TeamVectorCopyInternal {
84 template <typename MemberType, typename ValueType>
85 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const MemberType &member, const int m,
86 const ValueType *KOKKOS_RESTRICT A, const int as0,
87 /* */ ValueType *KOKKOS_RESTRICT B, const int bs0) {
88 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, m), [&](const int &i) { B[i * bs0] = A[i * as0]; });
89 // member.team_barrier();
90 return 0;
91 }
92 template <typename MemberType, typename ValueType>
93 KOKKOS_FORCEINLINE_FUNCTION static int invoke(const MemberType &member, const int m, const int n,
94 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
95 /* */ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
96 if (as0 > as1) {
97 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m), [&](const int &i) {
98 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, n),
99 [&](const int &j) { B[i * bs0 + j * bs1] = A[i * as0 + j * as1]; });
100 });
101 } else {
102 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, m), [&](const int &i) {
103 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n),
104 [&](const int &j) { B[i * bs0 + j * bs1] = A[i * as0 + j * as1]; });
105 });
106 }
107 // member.team_barrier();
108 return 0;
109 }
110 };
111
112 } // end namespace KokkosBatched
113
114 #endif