Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Xpay_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_XPAY_IMPL_HPP
17 #define KOKKOSBATCHED_XPAY_IMPL_HPP
18
19 /// \author Kim Liegeois ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 namespace KokkosBatched {
24
25 ///
26 /// Serial Internal Impl
27 /// ====================
28 struct SerialXpayInternal {
29 template <typename ScalarType, typename ValueType>
30 KOKKOS_INLINE_FUNCTION static int invoke(const int m, const ScalarType alpha, const ValueType* KOKKOS_RESTRICT X,
31 const int xs0,
32 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
33 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
34 #pragma unroll
35 #endif
36 for (int i = 0; i < m; ++i) {
37 Y[i * ys0] *= alpha;
38 Y[i * ys0] += X[i * xs0];
39 }
40
41 return 0;
42 }
43
44 template <typename ScalarType, typename ValueType>
45 KOKKOS_INLINE_FUNCTION static int invoke(const int m, const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
46 const ValueType* KOKKOS_RESTRICT X, const int xs0,
47 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
48 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
49 #pragma unroll
50 #endif
51 for (int i = 0; i < m; ++i) {
52 Y[i * ys0] *= alpha[i * alphas0];
53 Y[i * ys0] += X[i * xs0];
54 }
55
56 return 0;
57 }
58
59 template <typename ScalarType, typename ValueType>
60 KOKKOS_INLINE_FUNCTION static int invoke(const int m, const int n, const ScalarType* KOKKOS_RESTRICT alpha,
61 const int alphas0, const ValueType* KOKKOS_RESTRICT X, const int xs0,
62 const int xs1,
63 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0, const int ys1) {
64 if (xs0 > xs1)
65 for (int i = 0; i < m; ++i) invoke(n, alpha[i * alphas0], X + i * xs0, xs1, Y + i * ys0, ys1);
66 else
67 for (int j = 0; j < n; ++j) invoke(m, alpha, alphas0, X + j * xs1, xs0, Y + j * ys1, ys0);
68
69 return 0;
70 }
71 };
72
73 ///
74 /// Team Internal Impl
75 /// ====================
76 struct TeamXpayInternal {
77 template <typename MemberType, typename ScalarType, typename ValueType>
78 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m, const ScalarType alpha,
79 const ValueType* KOKKOS_RESTRICT X, const int xs0,
80 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
81 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m), [&](const int& i) {
82 Y[i * ys0] *= alpha;
83 Y[i * ys0] += X[i * xs0];
84 });
85 // member.team_barrier();
86 return 0;
87 }
88
89 template <typename MemberType, typename ScalarType, typename ValueType>
90 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m,
91 const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
92 const ValueType* KOKKOS_RESTRICT X, const int xs0,
93 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
94 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m), [&](const int& i) {
95 Y[i * ys0] *= alpha[i * alphas0];
96 Y[i * ys0] += X[i * xs0];
97 });
98 // member.team_barrier();
99 return 0;
100 }
101
102 template <typename MemberType, typename ScalarType, typename ValueType>
103 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m, const int n,
104 const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
105 const ValueType* KOKKOS_RESTRICT X, const int xs0, const int xs1,
106 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0, const int ys1) {
107 if (m > n) {
108 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m), [&](const int& i) {
109 SerialXpayInternal::invoke(n, alpha[i * alphas0], X + i * xs0, xs1, Y + i * ys0, ys1);
110 });
111 } else {
112 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int& j) {
113 SerialXpayInternal::invoke(m, alpha, alphas0, X + j * xs1, xs0, Y + j * ys1, ys0);
114 });
115 }
116 // member.team_barrier();
117 return 0;
118 }
119 };
120
121 ///
122 /// TeamVector Internal Impl
123 /// ========================
124 struct TeamVectorXpayInternal {
125 template <typename MemberType, typename ScalarType, typename ValueType>
126 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m, const ScalarType alpha,
127 const ValueType* KOKKOS_RESTRICT X, const int xs0,
128 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
129 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, m), [&](const int& i) {
130 Y[i * ys0] *= alpha;
131 Y[i * ys0] += X[i * xs0];
132 });
133 // member.team_barrier();
134 return 0;
135 }
136
137 template <typename MemberType, typename ScalarType, typename ValueType>
138 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m,
139 const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
140 const ValueType* KOKKOS_RESTRICT X, const int xs0,
141 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
142 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, m), [&](const int& i) {
143 Y[i * ys0] *= alpha[i * alphas0];
144 Y[i * ys0] += X[i * xs0];
145 });
146 // member.team_barrier();
147 return 0;
148 }
149
150 template <typename MemberType, typename ScalarType, typename ValueType, typename layout>
151 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m, const int n,
152 const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
153 const ValueType* KOKKOS_RESTRICT X, const int xs0, const int xs1,
154 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0, const int ys1) {
155 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, 0, m * n), [&](const int& iTemp) {
156 int i, j;
157 getIndices<int, layout>(iTemp, n, m, j, i);
158 Y[i * ys0 + j * ys1] *= alpha[i * alphas0];
159 Y[i * ys0 + j * ys1] += X[i * xs0 + j * xs1];
160 });
161 // member.team_barrier();
162 return 0;
163 }
164 };
165
166 ///
167 /// Serial Impl
168 /// ===========
169 template <typename ViewType, typename alphaViewType>
170 KOKKOS_INLINE_FUNCTION int SerialXpay::invoke(const alphaViewType& alpha, const ViewType& X, const ViewType& Y) {
171 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
172 static_assert(Kokkos::is_view<ViewType>::value, "KokkosBatched::xpay: ViewType is not a Kokkos::View.");
173 static_assert(Kokkos::is_view<alphaViewType>::value, "KokkosBatched::xpay: alphaViewType is not a Kokkos::View.");
174 static_assert(ViewType::rank == 2, "KokkosBatched::xpay: ViewType must have rank 2.");
175 static_assert(alphaViewType::rank == 1, "KokkosBatched::xpay: alphaViewType must have rank 1.");
176
177 // Check compatibility of dimensions at run time.
178 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
179 Kokkos::printf(
180 "KokkosBatched::xpay: Dimensions of X and Y do not match: X: %d x %d, "
181 "Y: %d x %d\n",
182 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
183 return 1;
184 }
185 if (X.extent(0) != alpha.extent(0)) {
186 Kokkos::printf(
187 "KokkosBatched::xpay: First dimension of X and alpha do not match: X: "
188 "%d x %d, alpha: %d\n",
189 (int)X.extent(0), (int)X.extent(1), (int)alpha.extent(0));
190 return 1;
191 }
192 #endif
193
194 return SerialXpayInternal::template invoke<typename alphaViewType::non_const_value_type,
195 typename ViewType::non_const_value_type>(
196 X.extent(0), X.extent(1), alpha.data(), alpha.stride_0(), X.data(), X.stride_0(), X.stride_1(), Y.data(),
197 Y.stride_0(), Y.stride_1());
198 }
199
200 ///
201 /// Team Impl
202 /// =========
203
204 template <typename MemberType>
205 template <typename ViewType, typename alphaViewType>
206 KOKKOS_INLINE_FUNCTION int TeamXpay<MemberType>::invoke(const MemberType& member, const alphaViewType& alpha,
207 const ViewType& X, const ViewType& Y) {
208 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
209 static_assert(Kokkos::is_view<ViewType>::value, "KokkosBatched::xpay: ViewType is not a Kokkos::View.");
210 static_assert(Kokkos::is_view<alphaViewType>::value, "KokkosBatched::xpay: alphaViewType is not a Kokkos::View.");
211 static_assert(ViewType::rank == 2, "KokkosBatched::xpay: ViewType must have rank 2.");
212 static_assert(alphaViewType::rank == 1, "KokkosBatched::xpay: alphaViewType must have rank 1.");
213
214 // Check compatibility of dimensions at run time.
215 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
216 Kokkos::printf(
217 "KokkosBatched::xpay: Dimensions of X and Y do not match: X: %d x %d, "
218 "Y: %d x %d\n",
219 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
220 return 1;
221 }
222 if (X.extent(0) != alpha.extent(0)) {
223 Kokkos::printf(
224 "KokkosBatched::xpay: First dimension of X and alpha do not match: X: "
225 "%d x %d, alpha: %d\n",
226 (int)X.extent(0), (int)X.extent(1), (int)alpha.extent(0));
227 return 1;
228 }
229 #endif
230
231 return TeamXpayInternal::template invoke<MemberType, typename alphaViewType::non_const_value_type,
232 typename ViewType::non_const_value_type>(
233 member, X.extent(0), X.extent(1), alpha.data(), alpha.stride_0(), X.data(), X.stride_0(), X.stride_1(), Y.data(),
234 Y.stride_0(), Y.stride_1());
235 }
236
237 ///
238 /// TeamVector Impl
239 /// ===============
240
241 template <typename MemberType>
242 template <typename ViewType, typename alphaViewType>
243 KOKKOS_INLINE_FUNCTION int TeamVectorXpay<MemberType>::invoke(const MemberType& member, const alphaViewType& alpha,
244 const ViewType& X, const ViewType& Y) {
245 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
246 static_assert(Kokkos::is_view<ViewType>::value, "KokkosBatched::xpay: ViewType is not a Kokkos::View.");
247 static_assert(Kokkos::is_view<alphaViewType>::value, "KokkosBatched::xpay: alphaViewType is not a Kokkos::View.");
248 static_assert(ViewType::rank == 2, "KokkosBatched::xpay: ViewType must have rank 2.");
249 static_assert(alphaViewType::rank == 1, "KokkosBatched::xpay: alphaViewType must have rank 1.");
250
251 // Check compatibility of dimensions at run time.
252 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
253 Kokkos::printf(
254 "KokkosBatched::xpay: Dimensions of X and Y do not match: X: %d x %d, "
255 "Y: %d x %d\n",
256 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
257 return 1;
258 }
259 if (X.extent(0) != alpha.extent(0)) {
260 Kokkos::printf(
261 "KokkosBatched::xpay: First dimension of X and alpha do not match: X: "
262 "%d x %d, alpha: %d\n",
263 (int)X.extent(0), (int)X.extent(1), (int)alpha.extent(0));
264 return 1;
265 }
266 #endif
267
268 return TeamVectorXpayInternal::invoke<MemberType, typename alphaViewType::non_const_value_type,
269 typename ViewType::non_const_value_type, typename ViewType::array_layout>(
270 member, X.extent(0), X.extent(1), alpha.data(), alpha.stride_0(), X.data(), X.stride_0(), X.stride_1(), Y.data(),
271 Y.stride_0(), Y.stride_1());
272 }
273
274 } // namespace KokkosBatched
275
276 #endif