Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Vector_SIMD_Arith.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_VECTOR_SIMD_ARITH_HPP
17 #define KOKKOSBATCHED_VECTOR_SIMD_ARITH_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "Kokkos_Complex.hpp"
22 #include "KokkosKernels_Macros.hpp"
23
24 namespace KokkosBatched {
25
26 #define KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) Vector<SIMD<T>, l>
27 #define KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) Vector<SIMD<T>, l> &
28
29 /// simd, simd
30 #if defined(__KOKKOSBATCHED_ENABLE_AVX__)
31 #if defined(__AVX512F__)
32 KOKKOS_FORCEINLINE_FUNCTION
33 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 8) operator+(const Vector<SIMD<double>, 8> &a,
34 const Vector<SIMD<double>, 8> &b) {
35 return _mm512_add_pd(a, b);
36 }
37
38 #if !defined(KOKKOS_COMPILER_GNU)
39 KOKKOS_FORCEINLINE_FUNCTION
40 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 4) operator+(
41 const Vector<SIMD<Kokkos::complex<double> >, 4> &a, const Vector<SIMD<Kokkos::complex<double> >, 4> &b) {
42 return _mm512_add_pd(a, b);
43 }
44 #endif
45
46 #endif
47 #if defined(__AVX__) || defined(__AVX2__)
48 KOKKOS_FORCEINLINE_FUNCTION
49 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 4) operator+(const Vector<SIMD<double>, 4> &a,
50 const Vector<SIMD<double>, 4> &b) {
51 return _mm256_add_pd(a, b);
52 }
53
54 #if !defined(KOKKOS_COMPILER_GNU)
55 KOKKOS_FORCEINLINE_FUNCTION
56 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 2) operator+(
57 const Vector<SIMD<Kokkos::complex<double> >, 2> &a, const Vector<SIMD<Kokkos::complex<double> >, 2> &b) {
58 return _mm256_add_pd(a, b);
59 }
60 #endif
61
62 #endif
63 #endif
64
65 template <typename T, int l>
66 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator+(const Vector<SIMD<T>, l> &a,
67 const Vector<SIMD<T>, l> &b) {
68 Vector<SIMD<T>, l> r_val;
69 if (std::is_fundamental<T>::value) {
70 KOKKOSKERNELS_FORCE_SIMD
71 for (int i = 0; i < l; ++i) r_val[i] = a[i] + b[i];
72 } else {
73 for (int i = 0; i < l; ++i) r_val[i] = a[i] + b[i];
74 }
75 return r_val;
76 }
77
78 #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
79 KOKKOS_FORCEINLINE_FUNCTION
80 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(float, 2) operator+(const Vector<SIMD<float>, 2> &a,
81 const Vector<SIMD<float>, 2> &b) {
82 float2 r_val;
83 r_val.x = a.float2().x + b.float2().x;
84 r_val.y = a.float2().y + b.float2().y;
85 return r_val;
86 }
87 KOKKOS_FORCEINLINE_FUNCTION
88 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 2) operator+(const Vector<SIMD<double>, 2> &a,
89 const Vector<SIMD<double>, 2> &b) {
90 double2 r_val;
91 r_val.x = a.double2().x + b.double2().x;
92 r_val.y = a.double2().y + b.double2().y;
93 return r_val;
94 }
95 KOKKOS_FORCEINLINE_FUNCTION
96 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(float, 4) operator+(const Vector<SIMD<float>, 4> &a,
97 const Vector<SIMD<float>, 4> &b) {
98 float4 r_val;
99 r_val.x = a.float4().x + b.float4().x;
100 r_val.y = a.float4().y + b.float4().y;
101 r_val.z = a.float4().z + b.float4().z;
102 r_val.w = a.float4().w + b.float4().w;
103 return r_val;
104 }
105 KOKKOS_FORCEINLINE_FUNCTION
106 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 4) operator+(const Vector<SIMD<double>, 4> &a,
107 const Vector<SIMD<double>, 4> &b) {
108 double4 r_val;
109 r_val.x = a.double4().x + b.double4().x;
110 r_val.y = a.double4().y + b.double4().y;
111 r_val.z = a.double4().z + b.double4().z;
112 r_val.w = a.double4().w + b.double4().w;
113 return r_val;
114 }
115
116 #endif
117
118 template <typename T, int l>
119 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator+=(
120 Vector<SIMD<T>, l> &a, const Vector<SIMD<T>, l> &b) {
121 a = a + b;
122 return a;
123 }
124
125 /// simd, real
126
127 template <typename T, int l>
128 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator+(const Vector<SIMD<T>, l> &a,
129 const T b) {
130 return a + Vector<SIMD<T>, l>(b);
131 }
132
133 template <typename T, int l>
134 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator+(const T a,
135 const Vector<SIMD<T>, l> &b) {
136 return Vector<SIMD<T>, l>(a) + b;
137 }
138
139 template <typename T, int l>
140 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator+=(
141 Vector<SIMD<T>, l> &a, const T b) {
142 a = a + b;
143 return a;
144 }
145
146 template <typename T, int l>
147 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator++(Vector<SIMD<T>, l> &a, int) {
148 Vector<SIMD<T>, l> a0 = a;
149 a = a + typename Kokkos::ArithTraits<T>::mag_type(1);
150 return a0;
151 }
152
153 template <typename T, int l>
154 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator++(
155 Vector<SIMD<T>, l> &a) {
156 a = a + typename Kokkos::ArithTraits<T>::mag_type(1);
157 return a;
158 }
159
160 /// simd complex, real
161
162 template <typename T, int l>
163 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator+(
164 const Vector<SIMD<Kokkos::complex<T> >, l> &a, const T b) {
165 return a + Vector<SIMD<Kokkos::complex<T> >, l>(b);
166 }
167
168 template <typename T, int l>
169 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator+(
170 const T a, const Vector<SIMD<Kokkos::complex<T> >, l> &b) {
171 return Vector<SIMD<Kokkos::complex<T> >, l>(a) + b;
172 }
173
174 template <typename T, int l>
175 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(Kokkos::complex<T>, l) operator+=(
176 Vector<SIMD<Kokkos::complex<T> >, l> &a, const T b) {
177 a = a + b;
178 return a;
179 }
180
181 /// simd complex, complex
182
183 template <typename T, int l>
184 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator+(
185 const Vector<SIMD<Kokkos::complex<T> >, l> &a, const Kokkos::complex<T> b) {
186 return a + Vector<SIMD<Kokkos::complex<T> >, l>(b);
187 }
188
189 template <typename T, int l>
190 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator+(
191 const Kokkos::complex<T> a, const Vector<SIMD<Kokkos::complex<T> >, l> &b) {
192 return Vector<SIMD<Kokkos::complex<T> >, l>(a) + b;
193 }
194
195 template <typename T, int l>
196 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(Kokkos::complex<T>, l) operator+=(
197 Vector<SIMD<Kokkos::complex<T> >, l> &a, const Kokkos::complex<T> b) {
198 a = a + b;
199 return a;
200 }
201
202 /// ---------------------------------------------------------------------------------------------------
203
204 /// simd, simd
205
206 #if defined(__KOKKOSBATCHED_ENABLE_AVX__)
207 #if defined(__AVX512F__)
208 KOKKOS_FORCEINLINE_FUNCTION
209 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 8) operator-(const Vector<SIMD<double>, 8> &a,
210 const Vector<SIMD<double>, 8> &b) {
211 return _mm512_sub_pd(a, b);
212 }
213
214 #if !defined(KOKKOS_COMPILER_GNU)
215 KOKKOS_FORCEINLINE_FUNCTION
216 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 4) operator-(
217 const Vector<SIMD<Kokkos::complex<double> >, 4> &a, const Vector<SIMD<Kokkos::complex<double> >, 4> &b) {
218 return _mm512_sub_pd(a, b);
219 }
220 #endif
221
222 #endif
223 #if defined(__AVX__) || defined(__AVX2__)
224 KOKKOS_FORCEINLINE_FUNCTION
225 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 4) operator-(const Vector<SIMD<double>, 4> &a,
226 const Vector<SIMD<double>, 4> &b) {
227 return _mm256_sub_pd(a, b);
228 }
229
230 #if !defined(KOKKOS_COMPILER_GNU)
231 KOKKOS_FORCEINLINE_FUNCTION
232 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 2) operator-(
233 const Vector<SIMD<Kokkos::complex<double> >, 2> &a, const Vector<SIMD<Kokkos::complex<double> >, 2> &b) {
234 return _mm256_sub_pd(a, b);
235 }
236 #endif
237
238 #endif
239 #endif
240
241 template <typename T, int l>
242 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator-(const Vector<SIMD<T>, l> &a,
243 const Vector<SIMD<T>, l> &b) {
244 Vector<SIMD<T>, l> r_val;
245 if (std::is_fundamental<T>::value) {
246 KOKKOSKERNELS_FORCE_SIMD
247 for (int i = 0; i < l; ++i) r_val[i] = a[i] - b[i];
248 } else {
249 for (int i = 0; i < l; ++i) r_val[i] = a[i] - b[i];
250 }
251 return r_val;
252 }
253
254 #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
255 KOKKOS_FORCEINLINE_FUNCTION
256 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(float, 2) operator-(const Vector<SIMD<float>, 2> &a,
257 const Vector<SIMD<float>, 2> &b) {
258 float2 r_val;
259 r_val.x = a.float2().x - b.float2().x;
260 r_val.y = a.float2().y - b.float2().y;
261 return r_val;
262 }
263 KOKKOS_FORCEINLINE_FUNCTION
264 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 2) operator-(const Vector<SIMD<double>, 2> &a,
265 const Vector<SIMD<double>, 2> &b) {
266 double2 r_val;
267 r_val.x = a.double2().x - b.double2().x;
268 r_val.y = a.double2().y - b.double2().y;
269 return r_val;
270 }
271 KOKKOS_FORCEINLINE_FUNCTION
272 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(float, 4) operator-(const Vector<SIMD<float>, 4> &a,
273 const Vector<SIMD<float>, 4> &b) {
274 float4 r_val;
275 r_val.x = a.float4().x - b.float4().x;
276 r_val.y = a.float4().y - b.float4().y;
277 r_val.z = a.float4().z - b.float4().z;
278 r_val.w = a.float4().w - b.float4().w;
279 return r_val;
280 }
281 KOKKOS_FORCEINLINE_FUNCTION
282 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 4) operator-(const Vector<SIMD<double>, 4> &a,
283 const Vector<SIMD<double>, 4> &b) {
284 double4 r_val;
285 r_val.x = a.double4().x - b.double4().x;
286 r_val.y = a.double4().y - b.double4().y;
287 r_val.z = a.double4().z - b.double4().z;
288 r_val.w = a.double4().w - b.double4().w;
289 return r_val;
290 }
291 #endif
292
293 template <typename T, int l>
294 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator-(const Vector<SIMD<T>, l> &a) {
295 Vector<SIMD<T>, l> r_val;
296 if (std::is_fundamental<T>::value) {
297 KOKKOSKERNELS_FORCE_SIMD
298 for (int i = 0; i < l; ++i) r_val[i] = -a[i];
299 } else {
300 for (int i = 0; i < l; ++i) r_val[i] = -a[i];
301 }
302 return r_val;
303 }
304
305 template <typename T, int l>
306 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator-=(
307 Vector<SIMD<T>, l> &a, const Vector<SIMD<T>, l> &b) {
308 a = a - b;
309 return a;
310 }
311
312 /// simd, real
313
314 template <typename T, int l>
315 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator-(const Vector<SIMD<T>, l> &a,
316 const T b) {
317 return a - Vector<SIMD<T>, l>(b);
318 }
319
320 template <typename T, int l>
321 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator-(const T a,
322 const Vector<SIMD<T>, l> &b) {
323 return Vector<SIMD<T>, l>(a) - b;
324 }
325
326 template <typename T, int l>
327 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator-=(
328 Vector<SIMD<T>, l> &a, const T b) {
329 a = a - b;
330 return a;
331 }
332
333 template <typename T, int l>
334 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator--(Vector<SIMD<T>, l> &a, int) {
335 Vector<SIMD<T>, l> a0 = a;
336 a = a - typename Kokkos::ArithTraits<T>::mag_type(1);
337 return a0;
338 }
339
340 template <typename T, int l>
341 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator--(
342 Vector<SIMD<T>, l> &a) {
343 a = a - typename Kokkos::ArithTraits<T>::mag_type(1);
344 return a;
345 }
346
347 /// simd complex, real
348
349 template <typename T, int l>
350 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator-(
351 const Vector<SIMD<Kokkos::complex<T> >, l> &a, const T b) {
352 return a - Vector<SIMD<Kokkos::complex<T> >, l>(b);
353 }
354
355 template <typename T, int l>
356 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator-(
357 const T a, const Vector<SIMD<Kokkos::complex<T> >, l> &b) {
358 return Vector<SIMD<Kokkos::complex<T> >, l>(a) - b;
359 }
360
361 template <typename T, int l>
362 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(Kokkos::complex<T>, l) operator-=(
363 Vector<SIMD<Kokkos::complex<T> >, l> &a, const T b) {
364 a = a - b;
365 return a;
366 }
367
368 /// simd complex, complex
369
370 template <typename T, int l>
371 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator-(
372 const Vector<SIMD<Kokkos::complex<T> >, l> &a, const Kokkos::complex<T> b) {
373 return a - Vector<SIMD<Kokkos::complex<T> >, l>(b);
374 }
375
376 template <typename T, int l>
377 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator-(
378 const Kokkos::complex<T> a, const Vector<SIMD<Kokkos::complex<T> >, l> &b) {
379 return Vector<SIMD<Kokkos::complex<T> >, l>(a) - b;
380 }
381
382 template <typename T, int l>
383 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(Kokkos::complex<T>, l) operator-=(
384 Vector<SIMD<Kokkos::complex<T> >, l> &a, const Kokkos::complex<T> b) {
385 a = a - b;
386 return a;
387 }
388
389 /// ---------------------------------------------------------------------------------------------------
390
391 /// simd, simd
392
393 #if defined(__KOKKOSBATCHED_ENABLE_AVX__)
394 #if defined(__AVX512F__)
395 KOKKOS_FORCEINLINE_FUNCTION
396 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 8) operator*(const Vector<SIMD<double>, 8> &a,
397 const Vector<SIMD<double>, 8> &b) {
398 return _mm512_mul_pd(a, b);
399 }
400
401 #if !defined(KOKKOS_COMPILER_GNU)
402 KOKKOS_FORCEINLINE_FUNCTION
403 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 4) operator*(
404 const Vector<SIMD<Kokkos::complex<double> >, 4> &a, const Vector<SIMD<Kokkos::complex<double> >, 4> &b) {
405 const __m512d as = _mm512_permute_pd(a, 0x55), br = _mm512_permute_pd(b, 0x00), bi = _mm512_permute_pd(b, 0xff);
406
407 #if defined(__FMA__)
408 // latency 7, throughput 0.5
409 return _mm512_fmaddsub_pd(a, br, _mm512_mul_pd(as, bi));
410 #else
411 return _mm512_add_pd(_mm512_mul_pd(a, br),
412 _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(_mm512_mul_pd(as, bi)),
413 _mm512_castpd_si512(_mm512_mask_broadcast_f64x4(
414 _mm512_setzero_pd(), 0x55, _mm256_set1_pd(-0.0))))));
415 // const __mm512d cc = _mm512_mul_pd(as, bi);
416 // return _mm512_mask_sub_pd(_mm512_mask_add_pd(_mm512_mul_pd(a, br), 0x55,
417 // cc), 0xaa, cc);
418 #endif
419 }
420 #endif
421
422 #endif
423 #if defined(__AVX__) || defined(__AVX2__)
424 KOKKOS_FORCEINLINE_FUNCTION
425 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 4) operator*(const Vector<SIMD<double>, 4> &a,
426 const Vector<SIMD<double>, 4> &b) {
427 return _mm256_mul_pd(a, b);
428 }
429
430 #if !defined(KOKKOS_COMPILER_GNU)
431 KOKKOS_FORCEINLINE_FUNCTION
432 static Vector<SIMD<Kokkos::complex<double> >, 2> operator*(const Vector<SIMD<Kokkos::complex<double> >, 2> &a,
433 const Vector<SIMD<Kokkos::complex<double> >, 2> &b) {
434 const __m256d as = _mm256_permute_pd(a, 0x5), br = _mm256_permute_pd(b, 0x0), bi = _mm256_permute_pd(b, 0xf);
435
436 #if defined(__FMA__)
437 return _mm256_fmaddsub_pd(a, br, _mm256_mul_pd(as, bi));
438 #else
439 return _mm256_add_pd(_mm256_mul_pd(a, br), _mm256_xor_pd(_mm256_mul_pd(as, bi), _mm256_set_pd(0.0, -0.0, 0.0, -0.0)));
440 #endif
441 }
442 #endif
443
444 #endif
445 #endif
446
447 template <typename T, int l>
448 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator*(const Vector<SIMD<T>, l> &a,
449 const Vector<SIMD<T>, l> &b) {
450 Vector<SIMD<T>, l> r_val;
451 if (std::is_fundamental<T>::value) {
452 KOKKOSKERNELS_FORCE_SIMD
453 for (int i = 0; i < l; ++i) r_val[i] = a[i] * b[i];
454 } else {
455 for (int i = 0; i < l; ++i) r_val[i] = a[i] * b[i];
456 }
457 return r_val;
458 }
459
460 #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
461 KOKKOS_FORCEINLINE_FUNCTION
462 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(float, 2) operator*(const Vector<SIMD<float>, 2> &a,
463 const Vector<SIMD<float>, 2> &b) {
464 float2 r_val;
465 r_val.x = a.float2().x * b.float2().x;
466 r_val.y = a.float2().y * b.float2().y;
467 return r_val;
468 }
469 KOKKOS_FORCEINLINE_FUNCTION
470 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 2) operator*(const Vector<SIMD<double>, 2> &a,
471 const Vector<SIMD<double>, 2> &b) {
472 double2 r_val;
473 r_val.x = a.double2().x * b.double2().x;
474 r_val.y = a.double2().y * b.double2().y;
475 return r_val;
476 }
477 KOKKOS_FORCEINLINE_FUNCTION
478 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(float, 4) operator*(const Vector<SIMD<float>, 4> &a,
479 const Vector<SIMD<float>, 4> &b) {
480 float4 r_val;
481 r_val.x = a.float4().x * b.float4().x;
482 r_val.y = a.float4().y * b.float4().y;
483 r_val.z = a.float4().z * b.float4().z;
484 r_val.w = a.float4().w * b.float4().w;
485 return r_val;
486 }
487 KOKKOS_FORCEINLINE_FUNCTION
488 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 4) operator*(const Vector<SIMD<double>, 4> &a,
489 const Vector<SIMD<double>, 4> &b) {
490 double4 r_val;
491 r_val.x = a.double4().x * b.double4().x;
492 r_val.y = a.double4().y * b.double4().y;
493 r_val.z = a.double4().z * b.double4().z;
494 r_val.w = a.double4().w * b.double4().w;
495 return r_val;
496 }
497 #endif
498
499 template <typename T, int l>
500 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator*=(
501 Vector<SIMD<T>, l> &a, const Vector<SIMD<T>, l> &b) {
502 a = a * b;
503 return a;
504 }
505
506 /// simd, real
507
508 template <typename T, int l>
509 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator*(const Vector<SIMD<T>, l> &a,
510 const T b) {
511 return a * Vector<SIMD<T>, l>(b);
512 }
513
514 template <typename T, int l>
515 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator*(const T a,
516 const Vector<SIMD<T>, l> &b) {
517 return Vector<SIMD<T>, l>(a) * b;
518 }
519
520 template <typename T, int l>
521 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator*=(
522 Vector<SIMD<T>, l> &a, const T b) {
523 a = a * b;
524 return a;
525 }
526
527 /// simd complex, real
528
529 #if defined(__KOKKOSBATCHED_ENABLE_AVX__)
530 #if defined(__AVX512F__)
531
532 #if !defined(KOKKOS_COMPILER_GNU)
533 KOKKOS_FORCEINLINE_FUNCTION
534 KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 4)
535 operator*(const Vector<SIMD<Kokkos::complex<double> >, 4> &a, const double b) {
536 return _mm512_mul_pd(a, _mm512_set1_pd(b));
537 }
538 #endif
539
540 #endif
541 #if defined(__AVX__) || defined(__AVX2__)
542
543 #if !defined(KOKKOS_COMPILER_GNU)
544 KOKKOS_FORCEINLINE_FUNCTION
545 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 2) operator*(
546 const Vector<SIMD<Kokkos::complex<double> >, 2> &a, const double b) {
547 return _mm256_mul_pd(a, _mm256_set1_pd(b));
548 }
549 #endif
550
551 #endif
552 #endif
553
554 template <typename T, int l>
555 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator*(
556 const Vector<SIMD<Kokkos::complex<T> >, l> &a, const T b) {
557 return a * Vector<SIMD<Kokkos::complex<T> >, l>(b);
558 }
559
560 #if defined(__KOKKOSBATCHED_ENABLE_AVX__)
561 #if defined(__AVX512F__)
562
563 #if !defined(KOKKOS_COMPILER_GNU)
564 KOKKOS_FORCEINLINE_FUNCTION
565 KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 4)
566 operator*(const double a, const Vector<SIMD<Kokkos::complex<double> >, 4> &b) {
567 return _mm512_mul_pd(_mm512_set1_pd(a), b);
568 }
569 #endif
570
571 #endif
572 #if defined(__AVX__) || defined(__AVX2__)
573
574 #if !defined(KOKKOS_COMPILER_GNU)
575 KOKKOS_FORCEINLINE_FUNCTION
576 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 2) operator*(
577 const double a, const Vector<SIMD<Kokkos::complex<double> >, 2> &b) {
578 return _mm256_mul_pd(_mm256_set1_pd(a), b);
579 }
580 #endif
581
582 #endif
583 #endif
584
585 template <typename T, int l>
586 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator*(
587 const T a, const Vector<SIMD<Kokkos::complex<T> >, l> &b) {
588 return Vector<SIMD<Kokkos::complex<T> >, l>(a) * b;
589 }
590
591 template <typename T, int l>
592 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(Kokkos::complex<T>, l) operator*=(
593 Vector<SIMD<Kokkos::complex<T> >, l> &a, const T b) {
594 a = a * b;
595 return a;
596 }
597
598 /// simd complex, complex
599
600 template <typename T, int l>
601 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator*(
602 const Vector<SIMD<Kokkos::complex<T> >, l> &a, const Kokkos::complex<T> b) {
603 return a * Vector<SIMD<Kokkos::complex<T> >, l>(b);
604 }
605
606 template <typename T, int l>
607 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator*(
608 const Kokkos::complex<T> a, const Vector<SIMD<Kokkos::complex<T> >, l> &b) {
609 return Vector<SIMD<Kokkos::complex<T> >, l>(a) * b;
610 }
611
612 template <typename T, int l>
613 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(Kokkos::complex<T>, l) operator*=(
614 Vector<SIMD<Kokkos::complex<T> >, l> &a, const Kokkos::complex<T> b) {
615 a = a * b;
616 return a;
617 }
618
619 /// ---------------------------------------------------------------------------------------------------
620
621 /// simd, simd
622
623 #if defined(__KOKKOSBATCHED_ENABLE_AVX__)
624 #if defined(__AVX512F__)
625 KOKKOS_FORCEINLINE_FUNCTION
626 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 8) operator/(const Vector<SIMD<double>, 8> &a,
627 const Vector<SIMD<double>, 8> &b) {
628 return _mm512_div_pd(a, b);
629 }
630
631 #if !defined(KOKKOS_COMPILER_GNU)
632 KOKKOS_FORCEINLINE_FUNCTION
633 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 4) operator/(
634 const Vector<SIMD<Kokkos::complex<double> >, 4> &a, const Vector<SIMD<Kokkos::complex<double> >, 4> &b) {
635 const __m512d as = _mm512_permute_pd(a, 0x55),
636 cb = _mm512_castsi512_pd(_mm512_xor_si512(
637 _mm512_castpd_si512(b),
638 _mm512_castpd_si512(_mm512_mask_broadcast_f64x4(_mm512_setzero_pd(), 0xAA, _mm256_set1_pd(-0.0))))),
639 br = _mm512_permute_pd(cb, 0x00), bi = _mm512_permute_pd(cb, 0xff);
640
641 #if defined(__FMA__)
642 return _mm512_div_pd(_mm512_fmaddsub_pd(a, br, _mm512_mul_pd(as, bi)),
643 _mm512_fmadd_pd(br, br, _mm512_mul_pd(bi, bi)));
644 #else
645 return _mm512_div_pd(_mm512_add_pd(_mm512_mul_pd(a, br), _mm512_castsi512_pd(_mm512_xor_si512(
646 _mm512_castpd_si512(_mm512_mul_pd(as, bi)),
647 _mm512_castpd_si512(_mm512_mask_broadcast_f64x4(
648 _mm512_setzero_pd(), 0xAA, _mm256_set1_pd(-0.0)))))),
649 _mm512_add_pd(_mm512_mul_pd(br, br), _mm512_mul_pd(bi, bi)));
650 // const __mm512d cc = _mm512_mul_pd(as, bi);
651 // return _mm512_div_pd(_mm512_mask_sub_pd(_mm512_mask_add_pd(_mm512_mul_pd(a,
652 // br), 0x55, cc), 0xaa, cc),
653 // _mm512_add_pd(_mm512_mul_pd(br, br), _mm512_mul_pd(bi,
654 // bi)));
655 #endif
656 }
657 #endif
658
659 #endif
660
661 #if defined(__AVX__) || defined(__AVX2__)
662 KOKKOS_FORCEINLINE_FUNCTION
663 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 4) operator/(const Vector<SIMD<double>, 4> &a,
664 const Vector<SIMD<double>, 4> &b) {
665 return _mm256_div_pd(a, b);
666 }
667
668 #if !defined(KOKKOS_COMPILER_GNU)
669 KOKKOS_FORCEINLINE_FUNCTION
670 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 2) operator/(
671 Vector<SIMD<Kokkos::complex<double> >, 2> const &a, Vector<SIMD<Kokkos::complex<double> >, 2> const &b) {
672 const __m256d as = _mm256_permute_pd(a, 0x5), cb = _mm256_xor_pd(b, _mm256_set_pd(-0.0, 0.0, -0.0, 0.0)),
673 br = _mm256_permute_pd(cb, 0x0), bi = _mm256_permute_pd(cb, 0xf);
674
675 #if defined(__FMA__)
676 return _mm256_div_pd(_mm256_fmaddsub_pd(a, br, _mm256_mul_pd(as, bi)),
677 _mm256_add_pd(_mm256_mul_pd(br, br), _mm256_mul_pd(bi, bi)));
678 #else
679 return _mm256_div_pd(
680 _mm256_add_pd(_mm256_mul_pd(a, br), _mm256_xor_pd(_mm256_mul_pd(as, bi), _mm256_set_pd(0.0, -0.0, 0.0, -0.0))),
681 _mm256_add_pd(_mm256_mul_pd(br, br), _mm256_mul_pd(bi, bi)));
682 #endif
683 }
684 #endif
685
686 #endif
687 #endif
688
689 template <typename T, int l>
690 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator/(const Vector<SIMD<T>, l> &a,
691 const Vector<SIMD<T>, l> &b) {
692 Vector<SIMD<T>, l> r_val;
693 if (std::is_fundamental<T>::value) {
694 KOKKOSKERNELS_FORCE_SIMD
695 for (int i = 0; i < l; ++i) r_val[i] = a[i] / b[i];
696 } else {
697 for (int i = 0; i < l; ++i) r_val[i] = a[i] / b[i];
698 }
699 return r_val;
700 }
701
702 #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
703 KOKKOS_FORCEINLINE_FUNCTION
704 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(float, 2) operator/(const Vector<SIMD<float>, 2> &a,
705 const Vector<SIMD<float>, 2> &b) {
706 float2 r_val;
707 r_val.x = a.float2().x / b.float2().x;
708 r_val.y = a.float2().y / b.float2().y;
709 return r_val;
710 }
711 KOKKOS_FORCEINLINE_FUNCTION
712 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 2) operator/(const Vector<SIMD<double>, 2> &a,
713 const Vector<SIMD<double>, 2> &b) {
714 double2 r_val;
715 r_val.x = a.double2().x / b.double2().x;
716 r_val.y = a.double2().y / b.double2().y;
717 return r_val;
718 }
719 KOKKOS_FORCEINLINE_FUNCTION
720 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(float, 4) operator/(const Vector<SIMD<float>, 4> &a,
721 const Vector<SIMD<float>, 4> &b) {
722 float4 r_val;
723 r_val.x = a.float4().x / b.float4().x;
724 r_val.y = a.float4().y / b.float4().y;
725 r_val.z = a.float4().z / b.float4().z;
726 r_val.w = a.float4().w / b.float4().w;
727 return r_val;
728 }
729 KOKKOS_FORCEINLINE_FUNCTION
730 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(double, 4) operator/(const Vector<SIMD<double>, 4> &a,
731 const Vector<SIMD<double>, 4> &b) {
732 double4 r_val;
733 r_val.x = a.double4().x / b.double4().x;
734 r_val.y = a.double4().y / b.double4().y;
735 r_val.z = a.double4().z / b.double4().z;
736 r_val.w = a.double4().w / b.double4().w;
737 return r_val;
738 }
739 #endif
740
741 template <typename T, int l>
742 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator/=(
743 Vector<SIMD<T>, l> &a, const Vector<SIMD<T>, l> &b) {
744 a = a / b;
745 return a;
746 }
747
748 /// simd, real
749 #if defined(__KOKKOSBATCHED_ENABLE_AVX__)
750 #if defined(__AVX512F__)
751
752 #if !defined(KOKKOS_COMPILER_GNU)
753 KOKKOS_FORCEINLINE_FUNCTION
754 static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<double>, 4) operator/(
755 const Vector<SIMD<Kokkos::complex<double> >, 4> &a, const double b) {
756 return _mm512_div_pd(a, _mm512_set1_pd(b));
757 }
758 #endif
759
760 #endif
761 #endif
762
763 template <typename T, int l>
764 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator/(const Vector<SIMD<T>, l> &a,
765 const T b) {
766 return a / Vector<SIMD<T>, l>(b);
767 }
768
769 template <typename T, int l>
770 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(T, l) operator/(const T a,
771 const Vector<SIMD<T>, l> &b) {
772 return Vector<SIMD<T>, l>(a) / b;
773 }
774
775 template <typename T, int l>
776 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(T, l) operator/=(
777 Vector<SIMD<T>, l> &a, const T b) {
778 a = a / b;
779 return a;
780 }
781
782 /// simd complex, real
783
784 template <typename T, int l>
785 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator/(
786 const Vector<SIMD<Kokkos::complex<T> >, l> &a, const T b) {
787 return a / Vector<SIMD<Kokkos::complex<T> >, l>(b);
788 }
789
790 template <typename T, int l>
791 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator/(
792 const T a, const Vector<SIMD<Kokkos::complex<T> >, l> &b) {
793 return Vector<SIMD<Kokkos::complex<T> >, l>(a) / b;
794 }
795
796 template <typename T, int l>
797 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(Kokkos::complex<T>, l) operator/=(
798 Vector<SIMD<Kokkos::complex<T> >, l> &a, const T b) {
799 a = a / b;
800 return a;
801 }
802
803 /// simd complex, complex
804
805 template <typename T, int l>
806 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator/(
807 const Vector<SIMD<Kokkos::complex<T> >, l> &a, const Kokkos::complex<T> b) {
808 return a / Vector<SIMD<Kokkos::complex<T> >, l>(b);
809 }
810
811 template <typename T, int l>
812 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE(Kokkos::complex<T>, l) operator/(
813 const Kokkos::complex<T> a, const Vector<SIMD<Kokkos::complex<T> >, l> &b) {
814 return Vector<SIMD<Kokkos::complex<T> >, l>(a) / b;
815 }
816
817 template <typename T, int l>
818 KOKKOS_FORCEINLINE_FUNCTION static KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE(Kokkos::complex<T>, l) operator/=(
819 Vector<SIMD<Kokkos::complex<T> >, l> &a, const Kokkos::complex<T> b) {
820 a = a / b;
821 return a;
822 }
823 #undef KOKKOSKERNELS_SIMD_ARITH_RETURN_TYPE
824 #undef KOKKOSKERNELS_SIMD_ARITH_RETURN_REFERENCE_TYPE
825 } // namespace KokkosBatched
826
827 #endif