Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Axpy_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_AXPY_IMPL_HPP
17 #define KOKKOSBATCHED_AXPY_IMPL_HPP
18
19 /// \author Kim Liegeois ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosBlas1_team_axpby.hpp"
23
24 namespace KokkosBatched {
25
26 ///
27 /// Serial Internal Impl
28 /// ====================
29 struct SerialAxpyInternal {
30 template <typename ScalarType, typename ValueType>
31 KOKKOS_INLINE_FUNCTION static int invoke(const int m, const ScalarType alpha, const ValueType* KOKKOS_RESTRICT X,
32 const int xs0,
33 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
34 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
35 #pragma unroll
36 #endif
37 for (int i = 0; i < m; ++i) Y[i * ys0] += alpha * X[i * xs0];
38
39 return 0;
40 }
41
42 template <typename ScalarType, typename ValueType>
43 KOKKOS_INLINE_FUNCTION static int invoke(const int m, const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
44 const ValueType* KOKKOS_RESTRICT X, const int xs0,
45 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
46 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
47 #pragma unroll
48 #endif
49 for (int i = 0; i < m; ++i) Y[i * ys0] += alpha[i * alphas0] * X[i * xs0];
50
51 return 0;
52 }
53
54 template <typename ScalarType, typename ValueType>
55 KOKKOS_INLINE_FUNCTION static int invoke(const int m, const int n, const ScalarType* KOKKOS_RESTRICT alpha,
56 const int alphas0, const ValueType* KOKKOS_RESTRICT X, const int xs0,
57 const int xs1,
58 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0, const int ys1) {
59 if (xs0 > xs1)
60 for (int i = 0; i < m; ++i) invoke(n, alpha[i * alphas0], X + i * xs0, xs1, Y + i * ys0, ys1);
61 else
62 for (int j = 0; j < n; ++j) invoke(m, alpha, alphas0, X + j * xs1, xs0, Y + j * ys1, ys0);
63
64 return 0;
65 }
66 };
67
68 ///
69 /// Team Internal Impl
70 /// ====================
71 struct TeamAxpyInternal {
72 template <typename MemberType, typename ScalarType, typename ValueType>
73 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m, const ScalarType alpha,
74 const ValueType* KOKKOS_RESTRICT X, const int xs0,
75 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
76 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m), [&](const int& i) { Y[i * ys0] += alpha * X[i * xs0]; });
77 // member.team_barrier();
78 return 0;
79 }
80
81 template <typename MemberType, typename ScalarType, typename ValueType>
82 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m,
83 const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
84 const ValueType* KOKKOS_RESTRICT X, const int xs0,
85 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
86 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m),
87 [&](const int& i) { Y[i * ys0] += alpha[i * alphas0] * X[i * xs0]; });
88 // member.team_barrier();
89 return 0;
90 }
91
92 template <typename MemberType, typename ScalarType, typename ValueType>
93 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m, const int n,
94 const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
95 const ValueType* KOKKOS_RESTRICT X, const int xs0, const int xs1,
96 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0, const int ys1) {
97 if (m > n) {
98 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m), [&](const int& i) {
99 SerialAxpyInternal::invoke(n, alpha[i * alphas0], X + i * xs0, xs1, Y + i * ys0, ys1);
100 });
101 } else {
102 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int& j) {
103 SerialAxpyInternal::invoke(m, alpha, alphas0, X + j * xs1, xs0, Y + j * ys1, ys0);
104 });
105 }
106 // member.team_barrier();
107 return 0;
108 }
109 };
110
111 ///
112 /// TeamVector Internal Impl
113 /// ========================
114 struct TeamVectorAxpyInternal {
115 template <typename MemberType, typename ScalarType, typename ValueType>
116 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m, const ScalarType alpha,
117 const ValueType* KOKKOS_RESTRICT X, const int xs0,
118 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
119 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, m), [&](const int& i) { Y[i * ys0] += alpha * X[i * xs0]; });
120 // member.team_barrier();
121 return 0;
122 }
123
124 template <typename MemberType, typename ScalarType, typename ValueType>
125 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m,
126 const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
127 const ValueType* KOKKOS_RESTRICT X, const int xs0,
128 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0) {
129 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, m),
130 [&](const int& i) { Y[i * ys0] += alpha[i * alphas0] * X[i * xs0]; });
131 // member.team_barrier();
132 return 0;
133 }
134
135 template <typename MemberType, typename ScalarType, typename ValueType, typename layout>
136 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, const int m, const int n,
137 const ScalarType* KOKKOS_RESTRICT alpha, const int alphas0,
138 const ValueType* KOKKOS_RESTRICT X, const int xs0, const int xs1,
139 /* */ ValueType* KOKKOS_RESTRICT Y, const int ys0, const int ys1) {
140 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, 0, m * n), [&](const int& iTemp) {
141 int i, j;
142 getIndices<int, layout>(iTemp, n, m, j, i);
143 Y[i * ys0 + j * ys1] += alpha[i * alphas0] * X[i * xs0 + j * xs1];
144 });
145 // member.team_barrier();
146 return 0;
147 }
148 };
149
150 ///
151 /// Serial Impl
152 /// ===========
153
154 template <typename XViewType, typename YViewType, typename alphaViewType>
155 KOKKOS_INLINE_FUNCTION int SerialAxpy::invoke(const alphaViewType& alpha, const XViewType& X, const YViewType& Y) {
156 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
157 static_assert(Kokkos::is_view<XViewType>::value, "KokkosBatched::axpy: XViewType is not a Kokkos::View.");
158 static_assert(Kokkos::is_view<YViewType>::value, "KokkosBatched::axpy: YViewType is not a Kokkos::View.");
159 static_assert(Kokkos::is_view<alphaViewType>::value, "KokkosBatched::axpy: alphaViewType is not a Kokkos::View.");
160 static_assert(XViewType::rank == 2, "KokkosBatched::axpy: XViewType must have rank 2.");
161 static_assert(YViewType::rank == 2, "KokkosBatched::axpy: YViewType must have rank 2.");
162 static_assert(alphaViewType::rank == 1, "KokkosBatched::axpy: alphaViewType must have rank 1.");
163
164 // Check compatibility of dimensions at run time.
165 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
166 Kokkos::printf(
167 "KokkosBatched::axpy: Dimensions of X and Y do not match: X: %d x %d, "
168 "Y: %d x %d\n",
169 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
170 return 1;
171 }
172 if (X.extent(0) != alpha.extent(0)) {
173 Kokkos::printf(
174 "KokkosBatched::axpy: First dimension of X and alpha do not match: X: "
175 "%d x %d, alpha: %d\n",
176 (int)X.extent(0), (int)X.extent(1), (int)alpha.extent(0));
177 return 1;
178 }
179 #endif
180
181 // No need to check if X.extent(0)==1 in the serial case as we don't
182 // parallelize the kernel anyway.
183
184 return SerialAxpyInternal::template invoke<typename alphaViewType::non_const_value_type,
185 typename XViewType::non_const_value_type>(
186 X.extent(0), X.extent(1), alpha.data(), alpha.stride_0(), X.data(), X.stride_0(), X.stride_1(), Y.data(),
187 Y.stride_0(), Y.stride_1());
188 }
189
190 ///
191 /// Team Impl
192 /// =========
193
194 template <typename MemberType>
195 template <typename XViewType, typename YViewType, typename alphaViewType>
196 KOKKOS_INLINE_FUNCTION int TeamAxpy<MemberType>::invoke(const MemberType& member, const alphaViewType& alpha,
197 const XViewType& X, const YViewType& Y) {
198 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
199 static_assert(Kokkos::is_view<XViewType>::value, "KokkosBatched::axpy: XViewType is not a Kokkos::View.");
200 static_assert(Kokkos::is_view<YViewType>::value, "KokkosBatched::axpy: YViewType is not a Kokkos::View.");
201 static_assert(Kokkos::is_view<alphaViewType>::value, "KokkosBatched::axpy: alphaViewType is not a Kokkos::View.");
202 static_assert(XViewType::rank == 2, "KokkosBatched::axpy: XViewType must have rank 2.");
203 static_assert(YViewType::rank == 2, "KokkosBatched::axpy: YViewType must have rank 2.");
204 static_assert(alphaViewType::rank == 1, "KokkosBatched::axpy: alphaViewType must have rank 1.");
205
206 // Check compatibility of dimensions at run time.
207 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
208 Kokkos::printf(
209 "KokkosBatched::axpy: Dimensions of X and Y do not match: X: %d x %d, "
210 "Y: %d x %d\n",
211 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
212 return 1;
213 }
214 if (X.extent(0) != alpha.extent(0)) {
215 Kokkos::printf(
216 "KokkosBatched::axpy: First dimension of X and alpha do not match: X: "
217 "%d x %d, alpha: %d\n",
218 (int)X.extent(0), (int)X.extent(1), (int)alpha.extent(0));
219 return 1;
220 }
221 #endif
222
223 if (X.extent(0) == 1) {
224 KokkosBlas::Experimental::axpy<MemberType>(member, alpha.data()[0], Kokkos::subview(X, 0, Kokkos::ALL),
225 Kokkos::subview(Y, 0, Kokkos::ALL));
226 return 0;
227 }
228
229 return TeamAxpyInternal::template invoke<MemberType, typename alphaViewType::non_const_value_type,
230 typename XViewType::non_const_value_type>(
231 member, X.extent(0), X.extent(1), alpha.data(), alpha.stride_0(), X.data(), X.stride_0(), X.stride_1(), Y.data(),
232 Y.stride_0(), Y.stride_1());
233 }
234
235 ///
236 /// TeamVector Impl
237 /// ===============
238
239 template <typename MemberType>
240 template <typename XViewType, typename YViewType, typename alphaViewType>
241 KOKKOS_INLINE_FUNCTION int TeamVectorAxpy<MemberType>::invoke(const MemberType& member, const alphaViewType& alpha,
242 const XViewType& X, const YViewType& Y) {
243 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
244 static_assert(Kokkos::is_view<XViewType>::value, "KokkosBatched::axpy: XViewType is not a Kokkos::View.");
245 static_assert(Kokkos::is_view<YViewType>::value, "KokkosBatched::axpy: YViewType is not a Kokkos::View.");
246 static_assert(Kokkos::is_view<alphaViewType>::value, "KokkosBatched::axpy: alphaViewType is not a Kokkos::View.");
247 static_assert(XViewType::rank == 2, "KokkosBatched::axpy: XViewType must have rank 2.");
248 static_assert(YViewType::rank == 2, "KokkosBatched::axpy: YViewType must have rank 2.");
249 static_assert(alphaViewType::rank == 1, "KokkosBatched::axpy: alphaViewType must have rank 1.");
250
251 // Check compatibility of dimensions at run time.
252 if (X.extent(0) != Y.extent(0) || X.extent(1) != Y.extent(1)) {
253 Kokkos::printf(
254 "KokkosBatched::axpy: Dimensions of X and Y do not match: X: %d x %d, "
255 "Y: %d x %d\n",
256 (int)X.extent(0), (int)X.extent(1), (int)Y.extent(0), (int)Y.extent(1));
257 return 1;
258 }
259 if (X.extent(0) != alpha.extent(0)) {
260 Kokkos::printf(
261 "KokkosBatched::axpy: First dimension of X and alpha do not match: X: "
262 "%d x %d, alpha: %d\n",
263 (int)X.extent(0), (int)X.extent(1), (int)alpha.extent(0));
264 return 1;
265 }
266 #endif
267
268 if (X.extent(0) == 1) {
269 KokkosBlas::Experimental::axpy<MemberType>(member, alpha.data()[0], Kokkos::subview(X, 0, Kokkos::ALL),
270 Kokkos::subview(Y, 0, Kokkos::ALL));
271 return 0;
272 }
273
274 return TeamVectorAxpyInternal::invoke<MemberType, typename alphaViewType::non_const_value_type,
275 typename XViewType::non_const_value_type, typename XViewType::array_layout>(
276 member, X.extent(0), X.extent(1), alpha.data(), alpha.stride_0(), X.data(), X.stride_0(), X.stride_1(), Y.data(),
277 Y.stride_0(), Y.stride_1());
278 }
279
280 } // namespace KokkosBatched
281
282 #endif