Debian Codegraph @

packages/kokkos-kernels/batched/dense/unit_test/Test_Batched_SerialTrtri.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #include "gtest/gtest.h"
17 #include "Kokkos_Core.hpp"
18 #include "Kokkos_Random.hpp"
19
20 #include "KokkosBatched_Trtri_Decl.hpp"
21 #include "KokkosBatched_Trtri_Serial_Impl.hpp"
22
23 #include "KokkosKernels_TestUtils.hpp"
24
25 #include <chrono>
26
27 #define PRINT_MAT 0
28
29 using namespace KokkosBatched;
30
31 namespace Test {
32 namespace Trtri {
33
34 template <class ViewTypeA, class ExecutionSpace>
35 struct UnitDiagTRTRI {
36 ViewTypeA A_;
37 using ScalarA = typename ViewTypeA::value_type;
38
39 UnitDiagTRTRI(const ViewTypeA& A) : A_(A) {}
40
41 KOKKOS_INLINE_FUNCTION
42 void operator()(const int& i) const { A_(i, i) = ScalarA(1); }
43 };
44 template <class ViewTypeA, class ExecutionSpace>
45 struct NonUnitDiagTRTRI {
46 ViewTypeA A_;
47 using ScalarA = typename ViewTypeA::value_type;
48
49 NonUnitDiagTRTRI(const ViewTypeA& A) : A_(A) {}
50
51 KOKKOS_INLINE_FUNCTION
52 void operator()(const int& i) const { A_(i, i) = A_(i, i) + 10; }
53 };
54 template <class ViewTypeA, class ViewTypeB, class ViewTypeC, class ExecutionSpace>
55 struct VanillaGEMM {
56 bool A_t, B_t, A_c, B_c;
57 int N, K;
58 ViewTypeA A;
59 ViewTypeB B;
60 ViewTypeC C;
61
62 typedef typename ViewTypeA::value_type ScalarA;
63 typedef typename ViewTypeB::value_type ScalarB;
64 typedef typename ViewTypeC::value_type ScalarC;
65 typedef Kokkos::ArithTraits<ScalarC> APT;
66 typedef typename APT::mag_type mag_type;
67 ScalarA alpha;
68 ScalarC beta;
69
70 KOKKOS_INLINE_FUNCTION
71 void operator()(const typename Kokkos::TeamPolicy<ExecutionSpace>::member_type& team) const {
72 // GNU COMPILER BUG WORKAROUND
73 #if defined(KOKKOS_COMPILER_GNU) && !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
74 int i = team.league_rank();
75 #else
76 const int i = team.league_rank();
77 #endif
78 Kokkos::parallel_for(Kokkos::TeamThreadRange(team, N), [&](const int& j) {
79 ScalarC C_ij = 0.0;
80
81 // GNU 5.3, 5.4 and 6.1 (and maybe more) crash with another nested lambda
82 // here
83
84 #if defined(KOKKOS_COMPILER_GNU) && !defined(KOKKOS_COMPILER_NVCC)
85 for (int k = 0; k < K; k++) {
86 ScalarA A_ik = A_t ? (A_c ? APT::conj(A(k, i)) : A(k, i)) : A(i, k);
87 ScalarB B_kj = B_t ? (B_c ? APT::conj(B(j, k)) : B(j, k)) : B(k, j);
88 C_ij += A_ik * B_kj;
89 }
90 #else
91 Kokkos::parallel_reduce(Kokkos::ThreadVectorRange(team,K), [&] (const int& k, ScalarC& lsum) {
92 ScalarA A_ik = A_t?(A_c?APT::conj(A(k,i)):A(k,i)):A(i,k);
93 ScalarB B_kj = B_t?(B_c?APT::conj(B(j,k)):B(j,k)):B(k,j);
94 lsum += A_ik*B_kj;
95 },C_ij);
96 #endif
97
98 C(i, j) = beta * C(i, j) + alpha * C_ij;
99 });
100 }
101 };
102
103 template <typename U, typename D>
104 struct ParamTag {
105 typedef U uplo;
106 typedef D diag;
107 };
108
109 template <typename DeviceType, typename ViewType, typename ParamTagType, typename AlgoTagType>
110 struct Functor_TestBatchedSerialTrtri {
111 using execution_space = typename DeviceType::execution_space;
112 ViewType _a;
113
114 KOKKOS_INLINE_FUNCTION
115 Functor_TestBatchedSerialTrtri(const ViewType& a) : _a(a) {}
116
117 KOKKOS_INLINE_FUNCTION
118 void operator()(const ParamTagType&, const int k) const {
119 auto aa = Kokkos::subview(_a, k, Kokkos::ALL(), Kokkos::ALL());
120
121 SerialTrtri<typename ParamTagType::uplo, typename ParamTagType::diag, AlgoTagType>::invoke(aa);
122 }
123
124 inline void run() {
125 typedef typename ViewType::value_type value_type;
126 std::string name_region("KokkosBatched::Test::SerialTrtri");
127 const std::string name_value_type = Test::value_type_name<value_type>();
128 std::string name = name_region + name_value_type;
129 Kokkos::Profiling::pushRegion(name.c_str());
130 Kokkos::RangePolicy<execution_space, ParamTagType> policy(0, _a.extent(0));
131 Kokkos::parallel_for("Functor_TestBatchedSerialTrtri", policy, *this);
132 Kokkos::Profiling::popRegion();
133 }
134 };
135
136 template <typename DeviceType, typename ViewType, typename ScalarType, typename ParamTagType, typename AlgoTagType>
137 void impl_test_batched_trtri(const int N, const int K) {
138 typedef typename ViewType::value_type value_type;
139 typedef typename DeviceType::execution_space execution_space;
140 typedef Kokkos::ArithTraits<value_type> ats;
141
142 ScalarType alpha(1.0);
143 ScalarType beta(0.0);
144
145 // eps is ~ 10^-13 for double
146 typedef typename ats::mag_type mag_type;
147 const mag_type eps = 1.0e8 * ats::epsilon();
148 bool fail_flag = false;
149 ScalarType cur_check_val; // Either 1 or 0, to check A_I
150
151 const bool is_A_lower = std::is_same<typename ParamTagType::uplo, Uplo::Lower>::value;
152 ViewType A("A", N, K, K);
153 ViewType A_original("A_original", N, K, K);
154 ViewType A_I("A_I", N, K, K);
155
156 typename ViewType::HostMirror I_host = Kokkos::create_mirror_view(A_I);
157 typename ViewType::HostMirror A_host = Kokkos::create_mirror_view(A);
158
159 uint64_t seed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
160
161 using ViewTypeSubA = decltype(Kokkos::subview(A, 0, Kokkos::ALL(), Kokkos::ALL()));
162
163 Kokkos::Random_XorShift64_Pool<execution_space> rand_pool(seed);
164
165 if (std::is_same<typename ParamTagType::diag, Diag::Unit>::value) {
166 // Initialize A with deterministic random numbers
167 Kokkos::fill_random(A, rand_pool, Kokkos::rand<Kokkos::Random_XorShift64<execution_space>, ScalarType>::max());
168 using functor_type = UnitDiagTRTRI<ViewTypeSubA, execution_space>;
169 for (int k = 0; k < N; ++k) {
170 functor_type udtrtri(Kokkos::subview(A, k, Kokkos::ALL(), Kokkos::ALL()));
171 // Initialize As diag with 1s
172 Kokkos::parallel_for("KokkosBlas::Test::UnitDiagTRTRI", Kokkos::RangePolicy<execution_space>(0, K), udtrtri);
173 }
174 } else { //(diag[0]=='N')||(diag[0]=='n')
175 // Initialize A with random numbers
176 Kokkos::fill_random(A, rand_pool, Kokkos::rand<Kokkos::Random_XorShift64<execution_space>, ScalarType>::max());
177 using functor_type = NonUnitDiagTRTRI<ViewTypeSubA, execution_space>;
178 for (int k = 0; k < N; ++k) {
179 functor_type nudtrtri(Kokkos::subview(A, k, Kokkos::ALL(), Kokkos::ALL()));
180 // Initialize As diag with A(i,i)+10
181 Kokkos::parallel_for("KokkosBlas::Test::NonUnitDiagTRTRI", Kokkos::RangePolicy<execution_space>(0, K), nudtrtri);
182 }
183 }
184 Kokkos::fence();
185
186 Kokkos::deep_copy(A_host, A);
187 // Make A_host a lower triangle
188 for (int k = 0; k < N; k++) {
189 if (is_A_lower) {
190 for (int i = 0; i < K - 1; i++)
191 for (int j = i + 1; j < K; j++) A_host(k, i, j) = ScalarType(0);
192 } else {
193 // Make A_host a upper triangle
194 for (int i = 1; i < K; i++)
195 for (int j = 0; j < i; j++) A_host(k, i, j) = ScalarType(0);
196 }
197 }
198 Kokkos::deep_copy(A, A_host);
199 Kokkos::deep_copy(A_original, A);
200 Kokkos::fence();
201
202 #if PRINT_MAT
203 printf("A_original:\n");
204 for (int k = 0; k < N; ++k) {
205 for (int i = 0; i < K; i++) {
206 for (int j = 0; j < K; j++) {
207 printf("%*.13lf ", 20, A_original(k, i, j));
208 }
209 printf("\n");
210 }
211 }
212 #endif
213
214 #if PRINT_MAT
215 printf("A:\n");
216 for (int k = 0; k < N; ++k) {
217 for (int i = 0; i < K; i++) {
218 for (int j = 0; j < K; j++) {
219 printf("%*.13lf ", 20, A(k, i, j));
220 }
221 printf("\n");
222 }
223 }
224 #endif
225
226 Functor_TestBatchedSerialTrtri<DeviceType, ViewType, ParamTagType, Algo::Trtri::Unblocked>(A).run();
227
228 #if PRINT_MAT
229 printf("A_original:\n");
230 for (int k = 0; k < N; ++k) {
231 for (int i = 0; i < K; i++) {
232 for (int j = 0; j < K; j++) {
233 printf("%*.13lf ", 20, A_original(k, i, j));
234 }
235 printf("\n");
236 }
237 }
238 #endif
239
240 #if PRINT_MAT
241 printf("A:\n");
242 for (int k = 0; k < N; ++k) {
243 for (int i = 0; i < K; i++) {
244 for (int j = 0; j < K; j++) {
245 printf("%*.13lf ", 20, A(k, i, j));
246 }
247 printf("\n");
248 }
249 }
250 #endif
251
252 Kokkos::fence();
253
254 struct VanillaGEMM<ViewTypeSubA, ViewTypeSubA, ViewTypeSubA, execution_space> vgemm;
255 vgemm.A_t = false;
256 vgemm.B_t = false;
257 vgemm.A_c = false;
258 vgemm.B_c = false;
259 vgemm.N = K;
260 vgemm.K = K;
261 vgemm.alpha = alpha;
262 vgemm.beta = beta;
263 for (int i = 0; i < N; i++) {
264 vgemm.A = Kokkos::subview(A, i, Kokkos::ALL(), Kokkos::ALL());
265 vgemm.B = Kokkos::subview(A_original, i, Kokkos::ALL(), Kokkos::ALL());
266 ;
267 vgemm.C = Kokkos::subview(A_I, i, Kokkos::ALL(), Kokkos::ALL());
268 ;
269 Kokkos::parallel_for("KokkosBlas::Test::VanillaGEMM", Kokkos::TeamPolicy<execution_space>(K, Kokkos::AUTO, 16),
270 vgemm);
271 }
272
273 Kokkos::fence();
274 Kokkos::deep_copy(I_host, A_I);
275 Kokkos::fence();
276
277 #if PRINT_MAT
278 printf("I_host:\n");
279 for (int k = 0; k < N; ++k) {
280 for (int i = 0; i < K; i++) {
281 for (int j = 0; j < K; j++) {
282 printf("%*.13lf ", 20, I_host(k, i, j));
283 }
284 printf("\n");
285 }
286 }
287 #endif
288
289 for (int k = 0; k < N; ++k) {
290 for (int i = 0; i < K; ++i) {
291 for (int j = 0; j < K; ++j) {
292 cur_check_val = (i == j) ? ScalarType(1) : ScalarType(0); // ats::abs(host_A(i,j));
293 if (ats::abs(ats::abs(I_host(k, i, j)) - cur_check_val) > eps) {
294 fail_flag = true;
295 // printf(" Error: eps ( %g ), I_host ( %.15f ) != cur_check_val
296 // (%.15f) (abs result-cur_check_val %g) at (k %d, i %d, j %d)\n",
297 // eps, I_host(k,i,j), cur_check_val, ats::abs(I_host(k,i,j) -
298 // cur_check_val), k, i, j);
299 }
300 }
301 }
302 }
303
304 ASSERT_EQ(fail_flag, false);
305 }
306 } // namespace Trtri
307 } // namespace Test
308
309 template <typename DeviceType, typename ValueType, typename ScalarType, typename ParamTagType, typename AlgoTagType>
310 int test_batched_trtri(int batchSize = 512) {
311 #if defined(KOKKOSKERNELS_INST_LAYOUTLEFT)
312 {
313 typedef Kokkos::View<ValueType***, Kokkos::LayoutLeft, DeviceType> ViewType;
314 Test::Trtri::impl_test_batched_trtri<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(0, 10);
315 // Test::impl_test_batched_trtri<DeviceType,ViewType,ScalarType,ParamTagType,AlgoTagType>(
316 // 1, 2);
317 for (int i = 0; i < 10; ++i) {
318 // printf("Testing: LayoutLeft, Blksize %d\n", i);
319 Test::Trtri::impl_test_batched_trtri<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(batchSize, i);
320 Test::Trtri::impl_test_batched_trtri<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(batchSize, i);
321 }
322 }
323 #endif
324 #if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT)
325 {
326 typedef Kokkos::View<ValueType***, Kokkos::LayoutRight, DeviceType> ViewType;
327 Test::Trtri::impl_test_batched_trtri<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(0, 10);
328 for (int i = 0; i < 10; ++i) {
329 // printf("Testing: LayoutRight, Blksize %d\n", i);
330 Test::Trtri::impl_test_batched_trtri<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(batchSize, i);
331 Test::Trtri::impl_test_batched_trtri<DeviceType, ViewType, ScalarType, ParamTagType, AlgoTagType>(batchSize, i);
332 }
333 }
334 #endif
335
336 return 0;
337 }