Debian Codegraph @

packages/kokkos-kernels/batched/dense/unit_test/Test_Batched_SerialTrmm.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #include "gtest/gtest.h"
17 #include "Kokkos_Core.hpp"
18 #include "Kokkos_Random.hpp"
19
20 #include "KokkosBatched_Trmm_Decl.hpp"
21 #include "KokkosBatched_Trmm_Serial_Impl.hpp"
22
23 #include "KokkosKernels_TestUtils.hpp"
24
25 #include <chrono>
26
27 using namespace KokkosBatched;
28
29 namespace Test {
30 namespace Trmm {
31
32 template <class ViewTypeA, class ExecutionSpace>
33 struct UnitDiagTRMM {
34 ViewTypeA A_;
35 using ScalarA = typename ViewTypeA::value_type;
36
37 UnitDiagTRMM(const ViewTypeA& A) : A_(A) {}
38
39 KOKKOS_INLINE_FUNCTION
40 void operator()(const int& i) const { A_(i, i) = ScalarA(1); }
41 };
42 template <class ViewTypeA, class ExecutionSpace>
43 struct NonUnitDiagTRMM {
44 ViewTypeA A_;
45 using ScalarA = typename ViewTypeA::value_type;
46
47 NonUnitDiagTRMM(const ViewTypeA& A) : A_(A) {}
48
49 KOKKOS_INLINE_FUNCTION
50 void operator()(const int& i) const { A_(i, i) = A_(i, i) + 10; }
51 };
52 template <class ViewTypeA, class ViewTypeB, class ViewTypeC, class ExecutionSpace>
53 struct VanillaGEMM {
54 bool A_t, B_t, A_c, B_c;
55 int N, K;
56 ViewTypeA A;
57 ViewTypeB B;
58 ViewTypeC C;
59
60 typedef typename ViewTypeA::value_type ScalarA;
61 typedef typename ViewTypeB::value_type ScalarB;
62 typedef typename ViewTypeC::value_type ScalarC;
63 typedef Kokkos::ArithTraits<ScalarC> APT;
64 typedef typename APT::mag_type mag_type;
65 ScalarA alpha;
66 ScalarC beta;
67
68 KOKKOS_INLINE_FUNCTION
69 void operator()(const typename Kokkos::TeamPolicy<ExecutionSpace>::member_type& team) const {
70 // GNU COMPILER BUG WORKAROUND
71 #if defined(KOKKOS_COMPILER_GNU) && !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
72 int i = team.league_rank();
73 #else
74 const int i = team.league_rank();
75 #endif
76 Kokkos::parallel_for(Kokkos::TeamThreadRange(team, N), [&](const int& j) {
77 ScalarC C_ij = 0.0;
78
79 // GNU 5.3, 5.4 and 6.1 (and maybe more) crash with another nested lambda
80 // here
81
82 #if defined(KOKKOS_COMPILER_GNU) && !defined(KOKKOS_COMPILER_NVCC)
83 for (int k = 0; k < K; k++) {
84 ScalarA A_ik = A_t ? (A_c ? APT::conj(A(k, i)) : A(k, i)) : A(i, k);
85 ScalarB B_kj = B_t ? (B_c ? APT::conj(B(j, k)) : B(j, k)) : B(k, j);
86 C_ij += A_ik * B_kj;
87 }
88 #else
89 Kokkos::parallel_reduce(Kokkos::ThreadVectorRange(team,K), [&] (const int& k, ScalarC& lsum) {
90 ScalarA A_ik = A_t?(A_c?APT::conj(A(k,i)):A(k,i)):A(i,k);
91 ScalarB B_kj = B_t?(B_c?APT::conj(B(j,k)):B(j,k)):B(k,j);
92 lsum += A_ik*B_kj;
93 },C_ij);
94 #endif
95
96 C(i, j) = beta * C(i, j) + alpha * C_ij;
97 });
98 }
99 };
100
101 template <typename S, typename U, typename T, typename D>
102 struct ParamTag {
103 typedef S side;
104 typedef U uplo;
105 typedef T trans;
106 typedef D diag;
107 };
108
109 template <typename DeviceType, typename ViewType, typename ScalarType, typename ParamTagType, typename AlgoTagType>
110 struct Functor_TestBatchedSerialTrmm {
111 using execution_space = typename DeviceType::execution_space;
112 ViewType _a, _b;
113
114 ScalarType _alpha;
115
116 KOKKOS_INLINE_FUNCTION
117 Functor_TestBatchedSerialTrmm(const ScalarType alpha, const ViewType& a, const ViewType& b)
118 : _a(a), _b(b), _alpha(alpha) {}
119
120 KOKKOS_INLINE_FUNCTION
121 void operator()(const ParamTagType&, const int k) const {
122 auto aa = Kokkos::subview(_a, k, Kokkos::ALL(), Kokkos::ALL());
123 auto bb = Kokkos::subview(_b, k, Kokkos::ALL(), Kokkos::ALL());
124
125 SerialTrmm<typename ParamTagType::side, typename ParamTagType::uplo, typename ParamTagType::trans,
126 typename ParamTagType::diag, AlgoTagType>::invoke(_alpha, aa, bb);
127 }
128
129 inline void run() {
130 typedef typename ViewType::value_type value_type;
131 std::string name_region("KokkosBatched::Test::SerialTrmm");
132 const std::string name_value_type = Test::value_type_name<value_type>();
133 std::string name = name_region + name_value_type;
134 Kokkos::Profiling::pushRegion(name.c_str());
135 Kokkos::RangePolicy<execution_space, ParamTagType> policy(0, _a.extent(0));
136 Kokkos::parallel_for(name.c_str(), policy, *this);
137 Kokkos::Profiling::popRegion();
138 }
139 };
140
141 template <typename DeviceType, typename ViewType, typename ScalarType, typename ParamTagType, typename AlgoTagType>
142 void impl_test_batched_trmm(const int N, const int nRows, const int nCols, const char* trans) {
143 typedef typename ViewType::value_type value_type;
144 typedef typename DeviceType::execution_space execution_space;
145 typedef Kokkos::ArithTraits<value_type> ats;
146
147 ScalarType alpha(1.0);
148 ScalarType beta(0.0);
149
150 const bool is_side_right = std::is_same<typename ParamTagType::side, Side::Right>::value;
151 const bool is_A_lower = std::is_same<typename ParamTagType::uplo, Uplo::Lower>::value;
152 const int K = is_side_right ? nCols : nRows;
153 ViewType A("A", N, K, K), B_actual("B_actual", N, nRows, nCols), B_expected("B_expected", N, nRows, nCols);
154 typename ViewType::HostMirror A_host = Kokkos::create_mirror_view(A);
155 typename ViewType::HostMirror B_actual_host = Kokkos::create_mirror_view(B_actual);
156 typename ViewType::HostMirror B_expected_host = Kokkos::create_mirror_view(B_expected);
157 uint64_t seed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
158
159 using ViewTypeSubA = decltype(Kokkos::subview(A, 0, Kokkos::ALL(), Kokkos::ALL()));
160 using ViewTypeSubB = decltype(Kokkos::subview(B_actual, 0, Kokkos::ALL(), Kokkos::ALL()));
161
162 Kokkos::Random_XorShift64_Pool<execution_space> rand_pool(seed);
163
164 if (std::is_same<typename ParamTagType::diag, Diag::NonUnit>::value) {
165 // Initialize A with deterministic random numbers
166 Kokkos::fill_random(A, rand_pool, Kokkos::rand<Kokkos::Random_XorShift64<execution_space>, ScalarType>::max());
167 using functor_type = UnitDiagTRMM<ViewTypeSubA, execution_space>;
168 for (int k = 0; k < N; ++k) {
169 functor_type udtrmm(Kokkos::subview(A, k, Kokkos::ALL(), Kokkos::ALL()));
170 // Initialize As diag with 1s
171 Kokkos::parallel_for("KokkosBlas::Test::UnitDiagTRMM", Kokkos::RangePolicy<execution_space>(0, K), udtrmm);
172 }
173 } else { //(diag[0]=='N')||(diag[0]=='n')
174 // Initialize A with random numbers
175 Kokkos::fill_random(A, rand_pool, Kokkos::rand<Kokkos::Random_XorShift64<execution_space>, ScalarType>::max());
176 using functor_type = NonUnitDiagTRMM<ViewTypeSubA, execution_space>;
177 for (int k = 0; k < N; ++k) {
178 functor_type nudtrmm(Kokkos::subview(A, k, Kokkos::ALL(), Kokkos::ALL()));
179 // Initialize As diag with A(i,i)+10
180 Kokkos::parallel_for("KokkosBlas::Test::NonUnitDiagTRMM", Kokkos::RangePolicy<execution_space>(0, K), nudtrmm);
181 }
182 }
183 Kokkos::fill_random(B_actual, rand_pool, Kokkos::rand<Kokkos::Random_XorShift64<execution_space>, ScalarType>::max());
184 Kokkos::fence();
185
186 Kokkos::deep_copy(B_expected, B_actual);
187 Kokkos::fence();
188
189 Kokkos::deep_copy(A_host, A);
190 // Make A_host a lower triangle
191 for (int k = 0; k < N; k++) {
192 if (is_A_lower) {
193 for (int i = 0; i < K - 1; i++)
194 for (int j = i + 1; j < K; j++) A_host(k, i, j) = ScalarType(0);
195 } else {
196 // Make A_host a upper triangle
197 for (int i = 1; i < K; i++)
198 for (int j = 0; j < i; j++) A_host(k, i, j) = ScalarType(0);
199 }
200 }
201 Kokkos::deep_copy(A, A_host);
202
203 if (!is_side_right) {
204 // B_expected = alpha * op(A) * B + beta * C = 1 * op(A) * B + 0 * C
205 struct VanillaGEMM<ViewTypeSubA, ViewTypeSubB, ViewTypeSubB, execution_space> vgemm;
206 vgemm.A_t = (trans[0] != 'N') && (trans[0] != 'n');
207 vgemm.B_t = false;
208 vgemm.A_c = (trans[0] == 'C') || (trans[0] == 'c');
209 vgemm.B_c = false;
210 vgemm.N = nCols;
211 vgemm.K = K;
212 vgemm.alpha = alpha;
213 vgemm.beta = beta;
214 for (int i = 0; i < N; i++) {
215 vgemm.A = Kokkos::subview(A, i, Kokkos::ALL(), Kokkos::ALL());
216 vgemm.B = Kokkos::subview(B_actual, i, Kokkos::ALL(), Kokkos::ALL());
217 ;
218 vgemm.C = Kokkos::subview(B_expected, i, Kokkos::ALL(), Kokkos::ALL());
219 ;
220 Kokkos::parallel_for("KokkosBlas::Test::VanillaGEMM",
221 Kokkos::TeamPolicy<execution_space>(nRows, Kokkos::AUTO, 16), vgemm);
222 }
223 } else {
224 // B_expected = alpha * B * op(A) + beta * C = 1 * B * op(A) + 0 * C
225 struct VanillaGEMM<ViewTypeSubB, ViewTypeSubA, ViewTypeSubB, execution_space> vgemm;
226 vgemm.A_t = false;
227 vgemm.B_t = (trans[0] != 'N') && (trans[0] != 'n');
228 vgemm.A_c = false;
229 vgemm.B_c = (trans[0] == 'C') || (trans[0] == 'c');
230 vgemm.N = nCols;
231 vgemm.K = K;
232 vgemm.alpha = alpha;
233 vgemm.beta = beta;
234 for (int i = 0; i < N; i++) {
235 vgemm.A = Kokkos::subview(B_actual, i, Kokkos::ALL(), Kokkos::ALL());
236 vgemm.B = Kokkos::subview(A, i, Kokkos::ALL(), Kokkos::ALL());
237 ;
238 vgemm.C = Kokkos::subview(B_expected, i, Kokkos::ALL(), Kokkos::ALL());
239 ;
240 Kokkos::parallel_for("KokkosBlas::Test::VanillaGEMM",
241 Kokkos::TeamPolicy<execution_space>(nRows, Kokkos::AUTO, 16), vgemm);
242 }
243 }
244
245 Functor_TestBatchedSerialTrmm<DeviceType, ViewType, ScalarType, ParamTagType, Algo::Trmm::Unblocked>(alpha, A,
246 B_actual)
247 .run();
248
249 Kokkos::fence();
250
251 Kokkos::deep_copy(B_actual_host, B_actual);
252 Kokkos::deep_copy(B_expected_host, B_expected);
253
254 Kokkos::fence();
255
256 // eps is ~ 10^-13 for double
257 typedef typename ats::mag_type mag_type;
258 const mag_type eps = 1.0e8 * ats::epsilon();
259 bool fail_flag = false;
260
261 for (int k = 0; k < N; ++k) {
262 for (int i = 0; i < nRows; ++i) {
263 for (int j = 0; j < nCols; ++j) {
264 if (ats::abs(B_actual_host(k, i, j) - B_expected_host(k, i, j)) > eps) {
265 // printf(" Error: eps ( %g ), abs_result( %.15lf ) != abs_solution(
266 // %.15lf ) (abs result-solution %g) at (k %d, i %d, j %d)\n", eps,
267 // ats::abs(B_actual_host(k,i,j)), ats::abs(B_expected_host(k,i,j)),
268 // ats::abs(B_actual_host(k,i,j) - B_expected_host(k,i,j)), k, i, j);
269 fail_flag = true;
270 }
271 }
272 }
273 }
274
275 ASSERT_EQ(fail_flag, false);
276 }
277 } // namespace Trmm
278 } // namespace Test
279
280 template <typename DeviceType, typename ValueType, typename ScalarType, typename ParamTagType, typename AlgoTagType>
281 int test_batched_trmm(int batchSize = 512) {
282 char trans = std::is_same<typename ParamTagType::trans, Trans::NoTranspose>::value ? 'N'
283 : std::is_same<typename ParamTagType::trans, Trans::Transpose>::value ? 'T'
284 : std::is_same<typename ParamTagType::trans, Trans::ConjTranspose>::value ? 'C'
285 : 'E';
286 #if defined(KOKKOSKERNELS_INST_LAYOUTLEFT)
287 {
288 typedef Kokkos::View<ValueType***, Kokkos::LayoutLeft, DeviceType> ViewType;
289 Test::Trmm::impl_test_batched_trmm<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(0, 10, 4, &trans);
290 for (int i = 0; i < 10; ++i) {
291 // printf("Testing: LayoutLeft, Blksize %d\n", i);
292 Test::Trmm::impl_test_batched_trmm<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(batchSize, i, 4,
293 &trans);
294 Test::Trmm::impl_test_batched_trmm<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(batchSize, i, 1,
295 &trans);
296 }
297 }
298 #endif
299 #if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT)
300 {
301 typedef Kokkos::View<ValueType***, Kokkos::LayoutRight, DeviceType> ViewType;
302 Test::Trmm::impl_test_batched_trmm<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(0, 10, 4, &trans);
303 for (int i = 0; i < 10; ++i) {
304 // printf("Testing: LayoutRight, Blksize %d\n", i);
305 Test::Trmm::impl_test_batched_trmm<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(batchSize, i, 4,
306 &trans);
307 Test::Trmm::impl_test_batched_trmm<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(batchSize, i, 1,
308 &trans);
309 }
310 }
311 #endif
312
313 return 0;
314 }