Debian Codegraph @

packages/kokkos-kernels/batched/dense/src/KokkosBatched_Vector_SIMD.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_VECTOR_SIMD_HPP
17 #define KOKKOSBATCHED_VECTOR_SIMD_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include <Kokkos_Complex.hpp>
22 #include <KokkosBatched_Vector.hpp>
23 #include "KokkosKernels_Macros.hpp"
24
25 #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
26 #undef __KOKKOSBATCHED_ENABLE_AVX__
27 #else
28 // compiler bug with AVX in some architectures
29 #define __KOKKOSBATCHED_ENABLE_AVX__
30 #endif
31
32 namespace KokkosBatched {
33
34 template <typename T, int l>
35 class Vector<SIMD<T>, l> {
36 public:
37 using type = Vector<SIMD<T>, l>;
38 using value_type = T;
39 using mag_type = typename Kokkos::ArithTraits<T>::mag_type;
40
41 enum : int { vector_length = l };
42
43 typedef value_type data_type[vector_length];
44
45 KOKKOS_INLINE_FUNCTION
46 static const char *label() { return "SIMD"; }
47
48 template <typename, int>
49 friend class Vector;
50
51 private:
52 mutable data_type _data;
53
54 public:
55 KOKKOS_INLINE_FUNCTION Vector() {
56 // NOTE Not meant to be instantiated for CUDA
57 KOKKOSKERNELS_FORCE_SIMD
58 for (int i = 0; i < vector_length; ++i) _data[i] = 0;
59 }
60 template <typename ArgValueType>
61 KOKKOS_INLINE_FUNCTION Vector(const ArgValueType &val) {
62 KOKKOSKERNELS_FORCE_SIMD
63 for (int i = 0; i < vector_length; ++i) _data[i] = val;
64 }
65 template <typename ArgValueType>
66 KOKKOS_INLINE_FUNCTION Vector(const Vector<SIMD<ArgValueType>, vector_length> &b) {
67 KOKKOSKERNELS_FORCE_SIMD
68 for (int i = 0; i < vector_length; ++i) _data[i] = b[i];
69 }
70
71 KOKKOS_INLINE_FUNCTION
72 type &loadAligned(const value_type *p) {
73 KOKKOSKERNELS_FORCE_SIMD
74 for (int i = 0; i < vector_length; ++i) _data[i] = p[i];
75 return *this;
76 }
77
78 KOKKOS_INLINE_FUNCTION
79 type &loadUnaligned(const value_type *p) { return loadAligned(p); }
80
81 KOKKOS_INLINE_FUNCTION
82 void storeAligned(value_type *p) const {
83 KOKKOSKERNELS_FORCE_SIMD
84 for (int i = 0; i < vector_length; ++i) p[i] = _data[i];
85 }
86
87 KOKKOS_INLINE_FUNCTION
88 void storeUnaligned(value_type *p) const { storeAligned(p); }
89
90 KOKKOS_INLINE_FUNCTION
91 value_type &operator[](const int &i) const { return _data[i]; }
92 };
93 } // namespace KokkosBatched
94
95 #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
96 namespace KokkosBatched {
97
98 template <>
99 class Vector<SIMD<float>, 2> {
100 public:
101 using type = Vector<SIMD<float>, 2>;
102 using value_type = float;
103 using mag_type = float;
104
105 enum : int { vector_length = 2 };
106 typedef float2 data_type;
107
108 KOKKOS_INLINE_FUNCTION
109 static const char *label() { return "GpuFloat2"; }
110
111 template <typename, int>
112 friend class Vector;
113
114 private:
115 mutable data_type _data;
116
117 public:
118 KOKKOS_INLINE_FUNCTION Vector() {
119 _data.x = 0;
120 _data.y = 0;
121 }
122 KOKKOS_INLINE_FUNCTION Vector(const value_type &val) {
123 _data.x = val;
124 _data.y = val;
125 }
126 KOKKOS_INLINE_FUNCTION Vector(const type &b) {
127 _data.x = b._data.x;
128 _data.y = b._data.y;
129 }
130 KOKKOS_INLINE_FUNCTION Vector(const float2 &val) {
131 _data.x = val.x;
132 _data.y = val.y;
133 }
134
135 template <typename ArgValueType>
136 KOKKOS_INLINE_FUNCTION Vector(const ArgValueType &val) {
137 _data.x = val;
138 _data.y = val;
139 }
140
141 template <typename ArgValueType>
142 KOKKOS_INLINE_FUNCTION Vector(const Vector<SIMD<ArgValueType>, vector_length> &b) {
143 _data.x = b[0];
144 _data.y = b[1];
145 }
146
147 KOKKOS_INLINE_FUNCTION
148 type &operator=(const float2 &val) {
149 _data.x = val.x;
150 _data.y = val.y;
151 return *this;
152 }
153
154 KOKKOS_INLINE_FUNCTION
155 float2 float2() const { return _data; }
156
157 KOKKOS_INLINE_FUNCTION
158 type &loadAligned(const value_type *p) {
159 _data.x = *(p);
160 _data.y = *(p + 1);
161 return *this;
162 }
163
164 KOKKOS_INLINE_FUNCTION
165 type &loadUnaligned(const value_type *p) {
166 _data.x = *(p);
167 _data.y = *(p + 1);
168 return *this;
169 }
170
171 KOKKOS_INLINE_FUNCTION
172 void storeAligned(value_type *p) const {
173 *(p) = _data.x;
174 *(p + 1) = _data.y;
175 }
176
177 KOKKOS_INLINE_FUNCTION
178 void storeUnaligned(value_type *p) const {
179 *(p) = _data.x;
180 *(p + 1) = _data.y;
181 }
182
183 KOKKOS_INLINE_FUNCTION
184 value_type &operator[](const int &i) const { return reinterpret_cast<value_type *>(&_data)[i]; }
185 };
186
187 template <>
188 class Vector<SIMD<double>, 2> {
189 public:
190 using type = Vector<SIMD<double>, 2>;
191 using value_type = double;
192 using mag_type = double;
193
194 enum : int { vector_length = 2 };
195 typedef double2 data_type;
196
197 KOKKOS_INLINE_FUNCTION
198 static const char *label() { return "GpuDouble2"; }
199
200 template <typename, int>
201 friend class Vector;
202
203 private:
204 mutable data_type _data;
205
206 public:
207 KOKKOS_INLINE_FUNCTION Vector() {
208 _data.x = 0;
209 _data.y = 0;
210 }
211 KOKKOS_INLINE_FUNCTION Vector(const value_type &val) {
212 _data.x = val;
213 _data.y = val;
214 }
215 KOKKOS_INLINE_FUNCTION Vector(const type &b) {
216 _data.x = b._data.x;
217 _data.y = b._data.y;
218 }
219 KOKKOS_INLINE_FUNCTION Vector(const double2 &val) {
220 _data.x = val.x;
221 _data.y = val.y;
222 }
223
224 template <typename ArgValueType>
225 KOKKOS_INLINE_FUNCTION Vector(const ArgValueType &val) {
226 _data.x = val;
227 _data.y = val;
228 }
229
230 template <typename ArgValueType>
231 KOKKOS_INLINE_FUNCTION Vector(const Vector<SIMD<ArgValueType>, vector_length> &b) {
232 _data.x = b[0];
233 _data.y = b[1];
234 }
235
236 KOKKOS_INLINE_FUNCTION
237 type &operator=(const double2 &val) {
238 _data.x = val.x;
239 _data.y = val.y;
240 return *this;
241 }
242
243 KOKKOS_INLINE_FUNCTION
244 double2 double2() const { return _data; }
245
246 KOKKOS_INLINE_FUNCTION
247 type &loadAligned(const value_type *p) {
248 _data.x = *(p);
249 _data.y = *(p + 1);
250 return *this;
251 }
252
253 KOKKOS_INLINE_FUNCTION
254 type &loadUnaligned(const value_type *p) {
255 _data.x = *(p);
256 _data.y = *(p + 1);
257 return *this;
258 }
259
260 KOKKOS_INLINE_FUNCTION
261 void storeAligned(value_type *p) const {
262 *(p) = _data.x;
263 *(p + 1) = _data.y;
264 }
265
266 KOKKOS_INLINE_FUNCTION
267 void storeUnaligned(value_type *p) const {
268 *(p) = _data.x;
269 *(p + 1) = _data.y;
270 }
271
272 KOKKOS_INLINE_FUNCTION
273 value_type &operator[](const int &i) const { return reinterpret_cast<value_type *>(&_data)[i]; }
274 };
275
276 template <>
277 class Vector<SIMD<float>, 4> {
278 public:
279 using type = Vector<SIMD<float>, 4>;
280 using value_type = float;
281 using mag_type = float;
282
283 enum : int { vector_length = 4 };
284 typedef float4 data_type;
285
286 KOKKOS_INLINE_FUNCTION
287 static const char *label() { return "GpuFloat4"; }
288
289 template <typename, int>
290 friend class Vector;
291
292 private:
293 mutable data_type _data;
294
295 public:
296 KOKKOS_INLINE_FUNCTION Vector() {
297 _data.x = 0;
298 _data.y = 0;
299 _data.z = 0;
300 _data.w = 0;
301 }
302 KOKKOS_INLINE_FUNCTION Vector(const value_type &val) {
303 _data.x = val;
304 _data.y = val;
305 _data.z = val;
306 _data.w = val;
307 }
308 KOKKOS_INLINE_FUNCTION Vector(const type &b) {
309 _data.x = b._data.x;
310 _data.y = b._data.y;
311 _data.z = b._data.z;
312 _data.w = b._data.w;
313 }
314 KOKKOS_INLINE_FUNCTION Vector(const float4 &val) {
315 _data.x = val.x;
316 _data.y = val.y;
317 _data.z = val.z;
318 _data.w = val.w;
319 }
320
321 template <typename ArgValueType>
322 KOKKOS_INLINE_FUNCTION Vector(const ArgValueType &val) {
323 _data.x = val;
324 _data.y = val;
325 _data.z = val;
326 _data.w = val;
327 }
328
329 template <typename ArgValueType>
330 KOKKOS_INLINE_FUNCTION Vector(const Vector<SIMD<ArgValueType>, vector_length> &b) {
331 _data.x = b[0];
332 _data.y = b[1];
333 _data.z = b[2];
334 _data.w = b[3];
335 }
336
337 KOKKOS_INLINE_FUNCTION
338 type &operator=(const float4 &val) {
339 _data.x = val.x;
340 _data.y = val.y;
341 _data.z = val.z;
342 _data.w = val.w;
343 return *this;
344 }
345
346 KOKKOS_INLINE_FUNCTION
347 float4 float4() const { return _data; }
348
349 KOKKOS_INLINE_FUNCTION
350 type &loadAligned(const value_type *p) {
351 _data.x = *(p);
352 _data.y = *(p + 1);
353 _data.z = *(p + 2);
354 _data.w = *(p + 3);
355 return *this;
356 }
357
358 KOKKOS_INLINE_FUNCTION
359 type &loadUnaligned(const value_type *p) {
360 _data.x = *(p);
361 _data.y = *(p + 1);
362 _data.z = *(p + 2);
363 _data.w = *(p + 3);
364 return *this;
365 }
366
367 KOKKOS_INLINE_FUNCTION
368 void storeAligned(value_type *p) const {
369 *(p) = _data.x;
370 *(p + 1) = _data.y;
371 *(p + 2) = _data.z;
372 *(p + 3) = _data.w;
373 }
374
375 KOKKOS_INLINE_FUNCTION
376 void storeUnaligned(value_type *p) const {
377 *(p) = _data.x;
378 *(p + 1) = _data.y;
379 *(p + 2) = _data.z;
380 *(p + 3) = _data.w;
381 }
382
383 KOKKOS_INLINE_FUNCTION
384 value_type &operator[](const int &i) const { return reinterpret_cast<value_type *>(&_data)[i]; }
385 };
386
387 template <>
388 class Vector<SIMD<double>, 4> {
389 public:
390 using type = Vector<SIMD<double>, 4>;
391 using value_type = double;
392 using mag_type = double;
393
394 enum : int { vector_length = 4 };
395 typedef double4 data_type;
396
397 KOKKOS_INLINE_FUNCTION
398 static const char *label() { return "GpuDouble4"; }
399
400 template <typename, int>
401 friend class Vector;
402
403 private:
404 mutable data_type _data;
405
406 public:
407 KOKKOS_INLINE_FUNCTION Vector() {
408 _data.x = 0;
409 _data.y = 0;
410 _data.z = 0;
411 _data.w = 0;
412 }
413 KOKKOS_INLINE_FUNCTION Vector(const value_type &val) {
414 _data.x = val;
415 _data.y = val;
416 _data.z = val;
417 _data.w = val;
418 }
419 KOKKOS_INLINE_FUNCTION Vector(const type &b) {
420 _data.x = b._data.x;
421 _data.y = b._data.y;
422 _data.z = b._data.z;
423 _data.w = b._data.w;
424 }
425 KOKKOS_INLINE_FUNCTION Vector(const double4 &val) {
426 _data.x = val.x;
427 _data.y = val.y;
428 _data.z = val.z;
429 _data.w = val.w;
430 }
431
432 template <typename ArgValueType>
433 KOKKOS_INLINE_FUNCTION Vector(const ArgValueType &val) {
434 _data.x = val;
435 _data.y = val;
436 _data.z = val;
437 _data.w = val;
438 }
439
440 template <typename ArgValueType>
441 KOKKOS_INLINE_FUNCTION Vector(const Vector<SIMD<ArgValueType>, vector_length> &b) {
442 _data.x = b[0];
443 _data.y = b[1];
444 _data.z = b[2];
445 _data.w = b[3];
446 }
447
448 KOKKOS_INLINE_FUNCTION
449 type &operator=(const double4 &val) {
450 _data.x = val.x;
451 _data.y = val.y;
452 _data.z = val.z;
453 _data.w = val.w;
454 return *this;
455 }
456
457 KOKKOS_INLINE_FUNCTION
458 double4 double4() const { return _data; }
459
460 KOKKOS_INLINE_FUNCTION
461 type &loadAligned(const value_type *p) {
462 _data.x = *(p);
463 _data.y = *(p + 1);
464 _data.z = *(p + 2);
465 _data.w = *(p + 3);
466 return *this;
467 }
468
469 KOKKOS_INLINE_FUNCTION
470 type &loadUnaligned(const value_type *p) {
471 _data.x = *(p);
472 _data.y = *(p + 1);
473 _data.z = *(p + 2);
474 _data.w = *(p + 3);
475 return *this;
476 }
477
478 KOKKOS_INLINE_FUNCTION
479 void storeAligned(value_type *p) const {
480 *(p) = _data.x;
481 *(p + 1) = _data.y;
482 *(p + 2) = _data.z;
483 *(p + 3) = _data.w;
484 }
485
486 KOKKOS_INLINE_FUNCTION
487 void storeUnaligned(value_type *p) const {
488 *(p) = _data.x;
489 *(p + 1) = _data.y;
490 *(p + 2) = _data.z;
491 *(p + 3) = _data.w;
492 }
493
494 KOKKOS_INLINE_FUNCTION
495 value_type &operator[](const int &i) const { return reinterpret_cast<value_type *>(&_data)[i]; }
496 };
497
498 } // namespace KokkosBatched
499 #endif
500
501 #if defined(__KOKKOSBATCHED_ENABLE_AVX__)
502 #if defined(__AVX__) || defined(__AVX2__)
503
504 #if CUDA_VERSION < 12022
505 #undef _Float16
506 #endif
507
508 #include <immintrin.h>
509
510 namespace KokkosBatched {
511
512 template <>
513 class Vector<SIMD<double>, 4> {
514 public:
515 using type = Vector<SIMD<double>, 4>;
516 using value_type = double;
517 using mag_type = double;
518
519 enum : int { vector_length = 4 };
520 typedef __m256d data_type __attribute__((aligned(32)));
521
522 inline static const char *label() { return "AVX256"; }
523
524 template <typename, int>
525 friend class Vector;
526
527 private:
528 mutable data_type _data;
529
530 public:
531 inline Vector() { _data = _mm256_setzero_pd(); }
532 inline Vector(const value_type &val) { _data = _mm256_set1_pd(val); }
533 inline Vector(const type &b) { _data = b._data; }
534 inline Vector(const __m256d &val) { _data = val; }
535
536 template <typename ArgValueType>
537 inline Vector(const ArgValueType &val) {
538 auto d = reinterpret_cast<value_type *>(&_data);
539 KOKKOSKERNELS_FORCE_SIMD
540 for (int i = 0; i < vector_length; ++i) d[i] = val;
541 }
542
543 template <typename ArgValueType>
544 inline Vector(const Vector<SIMD<ArgValueType>, vector_length> &b) {
545 auto dd = reinterpret_cast<value_type *>(&_data);
546 auto bb = reinterpret_cast<ArgValueType *>(&b._data);
547 KOKKOSKERNELS_FORCE_SIMD
548 for (int i = 0; i < vector_length; ++i) dd[i] = bb[i];
549 }
550
551 inline type &operator=(const __m256d &val) {
552 _data = val;
553 return *this;
554 }
555
556 inline operator __m256d() const { return _data; }
557
558 inline type &loadAligned(const value_type *p) {
559 _data = _mm256_load_pd(p);
560 return *this;
561 }
562
563 inline type &loadUnaligned(const value_type *p) {
564 _data = _mm256_loadu_pd(p);
565 return *this;
566 }
567
568 inline void storeAligned(value_type *p) const { _mm256_store_pd(p, _data); }
569
570 inline void storeUnaligned(value_type *p) const { _mm256_storeu_pd(p, _data); }
571
572 inline value_type &operator[](const int &i) const { return reinterpret_cast<value_type *>(&_data)[i]; }
573 };
574
575 template <>
576 class Vector<SIMD<Kokkos::complex<double> >, 2> {
577 public:
578 using type = Vector<SIMD<Kokkos::complex<double> >, 2>;
579 using value_type = Kokkos::complex<double>;
580 using mag_type = double;
581
582 static const int vector_length = 2;
583 typedef __m256d data_type __attribute__((aligned(32)));
584
585 inline static const char *label() { return "AVX256"; }
586
587 template <typename, int>
588 friend class Vector;
589
590 private:
591 mutable data_type _data;
592
593 public:
594 inline Vector() { _data = _mm256_setzero_pd(); }
595 inline Vector(const value_type &val) {
596 _data = _mm256_broadcast_pd((const __m128d *)&val);
597 KOKKOSKERNELS_GNU_COMPILER_FENCE
598 }
599 inline Vector(const mag_type &val) {
600 const value_type a(val);
601 _data = _mm256_broadcast_pd((__m128d const *)&a);
602 KOKKOSKERNELS_GNU_COMPILER_FENCE
603 }
604 inline Vector(const type &b) { _data = b._data; }
605 inline Vector(const __m256d &val) { _data = val; }
606
607 // template<typename ArgValueType>
608 // inline Vector(const ArgValueType val) {
609 // #if defined( KOKKOS_ENABLE_PRAGMA_IVDEP )
610 // #pragma ivdep
611 // #endif
612 // #if defined( KOKKOS_ENABLE_PRAGMA_VECTOR )
613 // #pragma vector always
614 // #endif
615 // for (int i=0;i<vector_length;++i)
616 // _data.d[i] = value_type(val);
617 // }
618 template <typename ArgValueType>
619 inline Vector(const Vector<SIMD<ArgValueType>, vector_length> &b) {
620 auto dd = reinterpret_cast<value_type *>(&_data);
621 auto bb = reinterpret_cast<ArgValueType *>(&b._data);
622 KOKKOSKERNELS_FORCE_SIMD
623 for (int i = 0; i < vector_length; ++i) dd[i] = bb[i];
624 }
625
626 inline type &operator=(const __m256d &val) {
627 _data = val;
628 return *this;
629 }
630
631 inline operator __m256d() const { return _data; }
632
633 inline type &loadAligned(const value_type *p) {
634 _data = _mm256_load_pd((mag_type *)p);
635 return *this;
636 }
637
638 inline type &loadUnaligned(const value_type *p) {
639 _data = _mm256_loadu_pd((mag_type *)p);
640 return *this;
641 }
642
643 inline void storeAligned(value_type *p) const { _mm256_store_pd((mag_type *)p, _data); }
644
645 inline void storeUnaligned(value_type *p) const { _mm256_storeu_pd((mag_type *)p, _data); }
646
647 inline value_type &operator[](const int &i) const { return reinterpret_cast<value_type *>(&_data)[i]; }
648 };
649 } // namespace KokkosBatched
650 #endif /* #if defined(__AVX__) || defined(__AVX2__) */
651
652 #if defined(__AVX512F__)
653 #if CUDA_VERSION < 12022
654 #undef _Float16
655 #endif
656 #include <immintrin.h>
657
658 namespace KokkosBatched {
659
660 template <>
661 class Vector<SIMD<double>, 8> {
662 public:
663 using type = Vector<SIMD<double>, 8>;
664 using value_type = double;
665 using mag_type = double;
666
667 enum : int { vector_length = 8 };
668 typedef __m512d data_type __attribute__((aligned(64)));
669
670 inline static const char *label() { return "AVX512"; }
671
672 template <typename, int>
673 friend class Vector;
674
675 private:
676 mutable data_type _data;
677
678 public:
679 inline Vector() { _data = _mm512_setzero_pd(); }
680 inline Vector(const value_type &val) { _data = _mm512_set1_pd(val); }
681 inline Vector(const type &b) { _data = b._data; }
682 inline Vector(const __m512d &val) { _data = val; }
683
684 template <typename ArgValueType>
685 inline Vector(const ArgValueType &val) {
686 auto d = reinterpret_cast<value_type *>(&_data);
687 KOKKOSKERNELS_FORCE_SIMD
688 for (int i = 0; i < vector_length; ++i) d[i] = val;
689 }
690 template <typename ArgValueType>
691 inline Vector(const Vector<SIMD<ArgValueType>, vector_length> &b) {
692 auto dd = reinterpret_cast<value_type *>(&_data);
693 auto bb = reinterpret_cast<ArgValueType *>(&b._data);
694 KOKKOSKERNELS_FORCE_SIMD
695 for (int i = 0; i < vector_length; ++i) dd[i] = bb[i];
696 }
697
698 inline type &operator=(const __m512d &val) {
699 _data = val;
700 return *this;
701 }
702
703 inline operator __m512d() const { return _data; }
704
705 inline type &loadAligned(const value_type *p) {
706 _data = _mm512_load_pd(p);
707 return *this;
708 }
709
710 inline type &loadUnaligned(const value_type *p) {
711 _data = _mm512_loadu_pd(p);
712 return *this;
713 }
714
715 inline void storeAligned(value_type *p) const { _mm512_store_pd(p, _data); }
716
717 inline void storeUnaligned(value_type *p) const { _mm512_storeu_pd(p, _data); }
718
719 inline value_type &operator[](const int &i) const { return reinterpret_cast<value_type *>(&_data)[i]; }
720 };
721
722 template <>
723 class Vector<SIMD<Kokkos::complex<double> >, 4> {
724 public:
725 using type = Vector<SIMD<Kokkos::complex<double> >, 4>;
726 using value_type = Kokkos::complex<double>;
727 using mag_type = double;
728
729 enum : int { vector_length = 4 };
730 typedef __m512d data_type __attribute__((aligned(64)));
731
732 inline static const char *label() { return "AVX512"; }
733
734 template <typename, int>
735 friend class Vector;
736
737 private:
738 mutable data_type _data;
739
740 public:
741 inline Vector() { _data = _mm512_setzero_pd(); }
742 inline Vector(const value_type &val) {
743 _data = _mm512_mask_broadcast_f64x4(_mm512_set1_pd(val.imag()), 0x55, _mm256_set1_pd(val.real()));
744 KOKKOSKERNELS_GNU_COMPILER_FENCE
745 }
746 inline Vector(const mag_type &val) {
747 _data = _mm512_mask_broadcast_f64x4(_mm512_setzero_pd(), 0x55, _mm256_set1_pd(val));
748 KOKKOSKERNELS_GNU_COMPILER_FENCE
749 }
750 inline Vector(const type &b) { _data = b._data; }
751 inline Vector(const __m512d &val) { _data = val; }
752
753 template <typename ArgValueType>
754 inline Vector(const ArgValueType &val) {
755 auto d = reinterpret_cast<value_type *>(&_data);
756 KOKKOSKERNELS_FORCE_SIMD
757 for (int i = 0; i < vector_length; ++i) d[i] = val;
758 }
759 template <typename ArgValueType>
760 inline Vector(const Vector<SIMD<ArgValueType>, vector_length> &b) {
761 auto dd = reinterpret_cast<value_type *>(&_data);
762 auto bb = reinterpret_cast<value_type *>(&b._data);
763 KOKKOSKERNELS_FORCE_SIMD
764 for (int i = 0; i < vector_length; ++i) dd[i] = bb[i];
765 }
766
767 inline type &operator=(const __m512d &val) {
768 _data = val;
769 return *this;
770 }
771
772 inline operator __m512d() const { return _data; }
773
774 inline type &loadAligned(const value_type *p) {
775 _data = _mm512_load_pd((mag_type *)p);
776 return *this;
777 }
778
779 inline type &loadUnaligned(const value_type *p) {
780 _data = _mm512_loadu_pd((mag_type *)p);
781 return *this;
782 }
783
784 inline void storeAligned(value_type *p) const { _mm512_store_pd((mag_type *)p, _data); }
785
786 inline void storeUnaligned(value_type *p) const { _mm512_storeu_pd((mag_type *)p, _data); }
787
788 inline value_type &operator[](const int &i) const { return reinterpret_cast<value_type *>(&_data)[i]; }
789 };
790 } // namespace KokkosBatched
791
792 #endif /* #if defined(__AVX512F__) */
793 #endif /* #if defined(__KOKKOSBATCHED_ENABLE_AVX__) */
794
795 #include "KokkosBatched_Vector_SIMD_Arith.hpp"
796 #include "KokkosBatched_Vector_SIMD_Logical.hpp"
797 #include "KokkosBatched_Vector_SIMD_Relation.hpp"
798 #include "KokkosBatched_Vector_SIMD_Math.hpp"
799 #include "KokkosBatched_Vector_SIMD_Misc.hpp"
800 #include "KokkosBatched_Vector_SIMD_View.hpp"
801
802 #endif