Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_ApplyPivot_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_APPLY_PIVOT_INTERNAL_HPP
17 #define KOKKOSBATCHED_APPLY_PIVOT_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 namespace KokkosBatched {
24
25 ///
26 /// TeamVector Internal Impl
27 /// ========================
28
29 ///
30 /// Forward
31 ///
32 struct TeamVectorApplyPivotVectorForwardInternal {
33 template <typename MemberType, typename ValueType>
34 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int piv,
35 /* */ ValueType *KOKKOS_RESTRICT A, const int as0) {
36 if (piv != 0) {
37 Kokkos::single(Kokkos::PerTeam(member), [&]() {
38 const int idx_p = piv * as0;
39 const ValueType tmp = A[0];
40 A[0] = A[idx_p];
41 A[idx_p] = tmp;
42 });
43 }
44 return 0;
45 }
46
47 template <typename MemberType, typename IntType, typename ValueType>
48 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int plen, const IntType *KOKKOS_RESTRICT p,
49 const int ps0,
50 /* */ ValueType *KOKKOS_RESTRICT A, const int as0) {
51 Kokkos::single(Kokkos::PerTeam(member), [&]() {
52 for (int i = 0; i < plen; ++i) {
53 const int piv = p[i * ps0];
54 if (piv != 0) {
55 const int idx_i = i * as0, idx_p = (i + piv) * as0;
56 const ValueType tmp = A[idx_i];
57 A[idx_i] = A[idx_p];
58 A[idx_p] = tmp;
59 }
60 }
61 });
62 return 0;
63 }
64 };
65
66 /// Pivot a row
67 struct TeamVectorApplyPivotMatrixForwardInternal {
68 template <typename MemberType, typename ValueType>
69 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int n, const int piv,
70 /* */ ValueType *KOKKOS_RESTRICT A, const int as0, const int as1) {
71 if (piv != 0) {
72 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, n), [&](const int &j) {
73 ValueType *KOKKOS_RESTRICT A_at_j = A + j * as1;
74 const int idx_p = piv * as0;
75 const ValueType tmp = A_at_j[0];
76 A_at_j[0] = A_at_j[idx_p];
77 A_at_j[idx_p] = tmp;
78 });
79 }
80 return 0;
81 }
82
83 template <typename MemberType, typename IntType, typename ValueType>
84 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int n, const int plen,
85 const IntType *KOKKOS_RESTRICT p, const int ps0,
86 /* */ ValueType *KOKKOS_RESTRICT A, const int as0, const int as1) {
87 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, n), [&](const int &j) {
88 ValueType *KOKKOS_RESTRICT A_at_j = A + j * as1;
89 for (int i = 0; i < plen; ++i) {
90 const int piv = p[i * ps0];
91 if (piv != 0) {
92 const int idx_i = i * as0, idx_p = (i + piv) * as0;
93 const ValueType tmp = A_at_j[idx_i];
94 A_at_j[idx_i] = A_at_j[idx_p];
95 A_at_j[idx_p] = tmp;
96 }
97 }
98 });
99 return 0;
100 }
101 };
102
103 ///
104 /// Backward
105 ///
106 struct TeamVectorApplyPivotVectorBackwardInternal {
107 template <typename MemberType, typename ValueType>
108 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int piv,
109 /* */ ValueType *KOKKOS_RESTRICT A, const int as0) {
110 if (piv != 0) {
111 Kokkos::single(Kokkos::PerTeam(member), [&]() {
112 const int idx_p = piv * as0;
113 const ValueType tmp = A[0];
114 A[0] = A[idx_p];
115 A[idx_p] = tmp;
116 });
117 }
118 return 0;
119 }
120
121 template <typename MemberType, typename IntType, typename ValueType>
122 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int plen, const IntType *KOKKOS_RESTRICT p,
123 const int ps0,
124 /* */ ValueType *KOKKOS_RESTRICT A, const int as0) {
125 Kokkos::single(Kokkos::PerTeam(member), [&]() {
126 for (int i = (plen - 1); i >= 0; --i) {
127 const int piv = p[i * ps0];
128 if (piv != 0) {
129 const int idx_i = i * as0, idx_p = (i + piv) * as0;
130 const ValueType tmp = A[idx_i];
131 A[idx_i] = A[idx_p];
132 A[idx_p] = tmp;
133 }
134 }
135 });
136 return 0;
137 }
138 };
139
140 /// Pivot a row
141 struct TeamVectorApplyPivotMatrixBackwardInternal {
142 template <typename MemberType, typename ValueType>
143 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int n, const int piv,
144 /* */ ValueType *KOKKOS_RESTRICT A, const int as0, const int as1) {
145 if (piv != 0) {
146 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, n), [&](const int &j) {
147 ValueType *KOKKOS_RESTRICT A_at_j = A + j * as1;
148 const int idx_p = piv * as0;
149 const ValueType tmp = A_at_j[0];
150 A_at_j[0] = A_at_j[idx_p];
151 A_at_j[idx_p] = tmp;
152 });
153 }
154 return 0;
155 }
156
157 template <typename MemberType, typename IntType, typename ValueType>
158 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int n, const int plen,
159 const IntType *KOKKOS_RESTRICT p, const int ps0,
160 /* */ ValueType *KOKKOS_RESTRICT A, const int as0, const int as1) {
161 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, n), [&](const int &j) {
162 ValueType *KOKKOS_RESTRICT A_at_j = A + j * as1;
163 for (int i = (plen - 1); i >= 0; --i) {
164 const int piv = p[i * ps0];
165 if (piv != 0) {
166 const int idx_i = i * as0, idx_p = (i + piv) * as0;
167 const ValueType tmp = A_at_j[idx_i];
168 A_at_j[idx_i] = A_at_j[idx_p];
169 A_at_j[idx_p] = tmp;
170 }
171 }
172 });
173 return 0;
174 }
175 };
176
177 } // namespace KokkosBatched
178
179 #endif