Debian Codegraph @

include/deal.II/base/vectorization.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2012 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15
16 #ifndef dealii_vectorization_h
17 #define dealii_vectorization_h
18
19 #include <deal.II/base/config.h>
20
21 #include <deal.II/base/exceptions.h>
22 #include <deal.II/base/numbers.h>
23 #include <deal.II/base/template_constraints.h>
24
25 #include <algorithm>
26 #include <array>
27 #include <cmath>
28
29 // Note:
30 // The flag DEAL_II_VECTORIZATION_WIDTH_IN_BITS is essentially constructed
31 // according to the following scheme (on x86-based architectures)
32 // #ifdef __AVX512F__
33 // #define DEAL_II_VECTORIZATION_WIDTH_IN_BITS 512
34 // #elif defined (__AVX__)
35 // #define DEAL_II_VECTORIZATION_WIDTH_IN_BITS 256
36 // #elif defined (__SSE2__)
37 // #define DEAL_II_VECTORIZATION_WIDTH_IN_BITS 128
38 // #else
39 // #define DEAL_II_VECTORIZATION_WIDTH_IN_BITS 0
40 // #endif
41 // In addition to checking the flags __AVX512F__, __AVX__ and __SSE2__, a CMake
42 // test, 'check_01_cpu_features.cmake', ensures that these feature are not only
43 // present in the compilation unit but also working properly.
44
45 #if DEAL_II_VECTORIZATION_WIDTH_IN_BITS > 0
46
47 // These error messages try to detect the case that deal.II was compiled with
48 // a wider instruction set extension as the current compilation unit, for
49 // example because deal.II was compiled with AVX, but a user project does not
50 // add -march=native or similar flags, making it fall to SSE2. This leads to
51 // very strange errors as the size of data structures differs between the
52 // compiled deal.II code sitting in libdeal_II.so and the user code if not
53 // detected.
54 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 256 && !defined(__AVX__)
55 # error \
56 "Mismatch in vectorization capabilities: AVX was detected during configuration of deal.II and switched on, but it is apparently not available for the file you are trying to compile at the moment. Check compilation flags controlling the instruction set, such as -march=native."
57 # endif
58 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 512 && !defined(__AVX512F__)
59 # error \
60 "Mismatch in vectorization capabilities: AVX-512F was detected during configuration of deal.II and switched on, but it is apparently not available for the file you are trying to compile at the moment. Check compilation flags controlling the instruction set, such as -march=native."
61 # endif
62
63 # ifdef _MSC_VER
64 # include <intrin.h>
65 # elif defined(__ALTIVEC__)
66 # include <altivec.h>
67
68 // altivec.h defines vector, pixel, bool, but we do not use them, so undefine
69 // them before they make trouble
70 # undef vector
71 # undef pixel
72 # undef bool
73 # elif defined(__ARM_NEON)
74 # include <arm_neon.h>
75 # elif defined(__x86_64__)
76 # include <x86intrin.h>
77 # endif
78
79 #endif
80
81
82 DEAL_II_NAMESPACE_OPEN
83
84
85 // Enable the EnableIfScalar type trait for VectorizedArray<Number> such
86 // that it can be used as a Number type in Tensor<rank,dim,Number>, etc.
87
88 template <typename Number, std::size_t width>
89 struct EnableIfScalar<VectorizedArray<Number, width>>
90 {
91 using type = VectorizedArray<typename EnableIfScalar<Number>::type, width>;
92 };
93
94
95
96 /**
97 * An iterator for VectorizedArray.
98 */
99 template <typename T>
100 class VectorizedArrayIterator
101 {
102 public:
103 /**
104 * Constructor.
105 *
106 * @param data The actual VectorizedArray.
107 * @param lane A pointer to the current lane.
108 */
109 constexpr VectorizedArrayIterator(T &data, const std::size_t lane)
110 : data(&data)
111 , lane(lane)
112 {}
113
114 /**
115 * Compare for equality.
116 */
117 constexpr bool
118 operator==(const VectorizedArrayIterator<T> &other) const
119 {
120 Assert(this->data == other.data,
121 ExcMessage(
122 "You are trying to compare iterators into different arrays."));
123 return this->lane == other.lane;
124 }
125
126 /**
127 * Compare for inequality.
128 */
129 constexpr bool
130 operator!=(const VectorizedArrayIterator<T> &other) const
131 {
132 Assert(this->data == other.data,
133 ExcMessage(
134 "You are trying to compare iterators into different arrays."));
135 return this->lane != other.lane;
136 }
137
138 /**
139 * Dereferencing operator (const version): returns the value of the current
140 * lane.
141 */
142 constexpr const typename T::value_type &
143 operator*() const
144 {
145 AssertIndexRange(lane, T::size());
146 return (*data)[lane];
147 }
148
149
150 /**
151 * Dereferencing operator (non-@p const version): returns the value of the
152 * current lane.
153 */
154 template <typename U = T>
155 constexpr std::enable_if_t<!std::is_same_v<U, const U>,
156 typename T::value_type> &
157 operator*()
158 {
159 AssertIndexRange(lane, T::size());
160 return (*data)[lane];
161 }
162
163 /**
164 * Prefix <tt>++</tt> operator: <tt>++iterator</tt>. This operator advances
165 * the iterator to the next lane and returns a reference to
166 * <tt>*this</tt>.
167 */
168 constexpr VectorizedArrayIterator<T> &
169 operator++()
170 {
171 AssertIndexRange(lane + 1, T::size() + 1);
172 ++lane;
173 return *this;
174 }
175
176 /**
177 * This operator advances the iterator by @p offset lanes and returns a
178 * reference to <tt>*this</tt>.
179 */
180 constexpr VectorizedArrayIterator<T> &
181 operator+=(const std::size_t offset)
182 {
183 AssertIndexRange(lane + offset, T::size() + 1);
184 lane += offset;
185 return *this;
186 }
187
188 /**
189 * Prefix <tt>--</tt> operator: <tt>--iterator</tt>. This operator advances
190 * the iterator to the previous lane and returns a reference to
191 * <tt>*this</tt>.
192 */
193 constexpr VectorizedArrayIterator<T> &
194 operator--()
195 {
196 Assert(
197 lane > 0,
198 ExcMessage(
199 "You can't decrement an iterator that is already at the beginning of the range."));
200 --lane;
201 return *this;
202 }
203
204 /**
205 * Create new iterator, which is shifted by @p offset.
206 */
207 constexpr VectorizedArrayIterator<T>
208 operator+(const std::size_t &offset) const
209 {
210 AssertIndexRange(lane + offset, T::size() + 1);
211 return VectorizedArrayIterator<T>(*data, lane + offset);
212 }
213
214 /**
215 * Compute distance between this iterator and iterator @p other.
216 */
217 constexpr std::ptrdiff_t
218 operator-(const VectorizedArrayIterator<T> &other) const
219 {
220 return static_cast<std::ptrdiff_t>(lane) -
221 static_cast<std::ptrdiff_t>(other.lane);
222 }
223
224 private:
225 /**
226 * Pointer to the actual VectorizedArray.
227 */
228 T *data;
229
230 /**
231 * Pointer to the current lane.
232 */
233 std::size_t lane;
234 };
235
236
237
238 /**
239 * A base class for the various VectorizedArray template specializations,
240 * containing common functionalities.
241 *
242 * @tparam VectorizedArrayType Type of the actual vectorized array this
243 * class is operating on. We are using the
244 * Curiously Recurring Template Pattern (see
245 * https://en.wikipedia.org/wiki/Curiously_recurring_template_pattern) in this
246 * class to avoid having to resort to `virtual` member functions. In
247 * other words, `VectorizedArrayType` is a class derived from the
248 * current class.
249 */
250 template <typename VectorizedArrayType, std::size_t width>
251 class VectorizedArrayBase
252 {
253 public:
254 /**
255 * Default constructor.
256 */
257 constexpr VectorizedArrayBase() = default;
258
259 /**
260 * Construct an array with the given initializer list.
261 *
262 * The initializer list must have at most as many elements as the
263 * vector length. Elements not listed in the initializer list are
264 * zero-initialized.
265 */
266 template <typename U>
267 constexpr VectorizedArrayBase(const std::initializer_list<U> &list)
268 {
269 const unsigned int n_initializers = list.size();
270 Assert(n_initializers <= size(),
271 ExcMessage("The initializer list must have at most "
272 "as many elements as the vector length."));
273
274 // Copy what's in the list.
275 std::copy_n(list.begin(), n_initializers, this->begin());
276
277 // Then add zero padding where necessary.
278 if (n_initializers < size())
279 std::fill(this->begin() + n_initializers, this->end(), 0.0);
280 }
281
282 /**
283 * Return the number of elements in the array.
284 */
285 static constexpr std::size_t
286 size()
287 {
288 return width;
289 }
290
291 /**
292 * @return An iterator pointing to the beginning of the underlying data.
293 */
294 constexpr VectorizedArrayIterator<VectorizedArrayType>
295 begin()
296 {
297 return VectorizedArrayIterator<VectorizedArrayType>(
298 static_cast<VectorizedArrayType &>(*this), 0);
299 }
300
301 /**
302 * @return An iterator pointing to the beginning of the underlying data (`const`
303 * version).
304 */
305 constexpr VectorizedArrayIterator<const VectorizedArrayType>
306 begin() const
307 {
308 return VectorizedArrayIterator<const VectorizedArrayType>(
309 static_cast<const VectorizedArrayType &>(*this), 0);
310 }
311
312 /**
313 * @return An iterator pointing to the end of the underlying data.
314 */
315 constexpr VectorizedArrayIterator<VectorizedArrayType>
316 end()
317 {
318 return VectorizedArrayIterator<VectorizedArrayType>(
319 static_cast<VectorizedArrayType &>(*this), width);
320 }
321
322 /**
323 * @return An iterator pointing to the end of the underlying data (`const`
324 * version).
325 */
326 constexpr VectorizedArrayIterator<const VectorizedArrayType>
327 end() const
328 {
329 return VectorizedArrayIterator<const VectorizedArrayType>(
330 static_cast<const VectorizedArrayType &>(*this), width);
331 }
332
333 /**
334 * A default implementation for computing the dot product between two
335 * vectorized arrays. It first forms the lane-by-lane product between
336 * the current object and the argument, and then the across-lanes
337 * sum of the product. The function returns the kind of object that
338 * the VectorizedArray::sum() function returns.
339 *
340 * This function is inherited by all derived classes and provides
341 * the dot product to them unless they override it with their own
342 * implementation (presumably using a more efficient approach).
343 */
344 auto
345 dot_product(const VectorizedArrayType &v) const
346 {
347 VectorizedArrayType p = static_cast<const VectorizedArrayType &>(*this);
348 p *= v;
349 return p.sum();
350 }
351 };
352
353
354
355 /**
356 * This generic class defines a unified interface to a vectorized data type.
357 * For general template arguments, this class simply corresponds to the
358 * template argument. For example, VectorizedArray<long double> is nothing
359 * else but a wrapper around <tt>long double</tt> with exactly one data field
360 * of type <tt>long double</tt> and overloaded arithmetic operations. This
361 * means that <tt>VectorizedArray<ComplicatedType></tt> has a similar layout
362 * as ComplicatedType, provided that ComplicatedType defines basic arithmetic
363 * operations. For floats and doubles, an array of numbers are packed together
364 * with the goal to be processed in a single-instruction/multiple-data (SIMD)
365 * fashion. In the SIMD context, the elements of such a short vector are often
366 * called "lanes". The number of elements packed together, i.e., the number of
367 * lanes, depends on the computer system and compiler flags that are used for
368 * compilation of deal.II. The fundamental idea of these packed data types is
369 * to use one single CPU instruction to perform arithmetic operations on the
370 * whole array using the processor's vector (SIMD) units. Most computer
371 * systems by 2010 standards will use an array of two doubles or four floats,
372 * respectively (this corresponds to the SSE/SSE2 data sets) when compiling
373 * deal.II on 64-bit operating systems. On later processors (such as Intel Sandy
374 * Bridge and newer, or AMD Bulldozer processors and newer), four
375 * doubles or eight floats are used when deal.II is configured using gcc with
376 * `--with-cpu=native` or `--with-cpu=corei7-avx`. On compilations with
377 * AVX-512 support (e.g., Intel Skylake Server from 2017), eight doubles
378 * or sixteen floats are used.
379 *
380 * The behavior of this class is made similar to the basic data types double
381 * and float. The definition of a vectorized array does not initialize the
382 * data field but rather leaves it undefined, as is the case for double and
383 * float. However, when calling something like `VectorizedArray<double> a =
384 * VectorizedArray<double>()` or `VectorizedArray<double> a = 0.`, it sets all
385 * numbers in this field to zero. This class is of standard layout type
386 * according to the C++11 standard, which means that there is an equivalent C
387 * representation and the class can e.g. be safely copied with std::memcpy.
388 * (See also https://en.cppreference.com/w/cpp/named_req/StandardLayoutType.)
389 * The standard layout is also necessary for ensuring correct alignment of
390 * data with address boundaries when collected in a vector (i.e., when the
391 * first element in a vector is properly aligned, all subsequent elements will
392 * be correctly aligned, too).
393 *
394 * Note that for proper functioning of this class, certain data alignment
395 * rules must be respected. This is because the computer expects the starting
396 * address of a VectorizedArray<double> field at specific addresses in memory
397 * (usually, the address of the vectorized array should be a multiple of the
398 * length of the array in bytes). Otherwise, a segmentation fault or a severe
399 * loss of performance might occur. When creating a single data field on the
400 * stack like `VectorizedArray<double> a = 5.;`, the compiler will take care
401 * of data alignment automatically. However, when allocating a long vector of
402 * VectorizedArray<double> data, one needs to respect these rules. Use the
403 * class AlignedVector or data containers based on AlignedVector (such as
404 * Table) for this purpose. It is a class very similar to std::vector
405 * otherwise but always makes sure that data is correctly aligned.
406 *
407 * The user can explicitly control the width of a particular instruction set
408 * architecture (ISA) extension by specifying the number of lanes via the second
409 * template parameter of this wrapper class. For example on Intel Skylake
410 * Server, you have the following options for the data type double:
411 * - VectorizedArray<double, 1> // no vectorization (auto-optimization)
412 * - VectorizedArray<double, 2> // SSE2
413 * - VectorizedArray<double, 4> // AVX
414 * - VectorizedArray<double, 8> // AVX-512 (default)
415 *
416 * and for Intel Sandy Bridge, Haswell, Broadwell, AMD Bulldozer and Zen/Ryzen:
417 * - VectorizedArray<double, 1> // no vectorization (auto-optimization)
418 * - VectorizedArray<double, 2> // SSE2
419 * - VectorizedArray<double, 4> // AVX (default)
420 *
421 * and for processors with AltiVec support:
422 * - VectorizedArray<double, 1>
423 * - VectorizedArray<double, 2>
424 *
425 * For older x86 processors or in case no processor-specific compilation flags
426 * were added (i.e., without `-D CMAKE_CXX_FLAGS=-march=native` or similar
427 * flags):
428 * - VectorizedArray<double, 1> // no vectorization (auto-optimization)
429 * - VectorizedArray<double, 2> // SSE2
430 *
431 * Similar considerations also apply to the data type `float`.
432 *
433 * Wrongly selecting the width, e.g., width=3 or width=8 on a processor which
434 * does not support AVX-512 leads to a failing `static_assert`, i.e., you cannot
435 * compile code using a VectorizedArray class with a number of lanes that
436 * is not supported by the platform you are on. In particular, all platforms
437 * we are aware of only ever support widths that are powers of two.
438 *
439 * @tparam Number The underlying scalar data type.
440 * @tparam width Vector length. (Optional; if not set, the maximal width of the
441 * architecture is used.)
442 */
443 template <typename Number, std::size_t width>
444 class VectorizedArray
445 : public VectorizedArrayBase<VectorizedArray<Number, width>, 1>
446 {
447 public:
448 /**
449 * The scalar type of the array elements.
450 */
451 using value_type = Number;
452
453 /**
454 * A constexpr boolean indicating whether the VectorizedArray with the
455 * given choice of template parameters @p Number and @p width is indeed
456 * implemented. The generic implementation is only implemented for
457 * @p width equal to one. For specializations of this class (which are
458 * defined depending on the instruction sets available) the boolean is
459 * set to true as well.
460 */
461 static constexpr bool is_implemented = (width == 1);
462
463 /**
464 * Default empty constructor, leaving the data in an uninitialized state
465 * similar to float/double.
466 */
467 VectorizedArray() = default;
468
469 /**
470 * Construct an array with the given scalar broadcast to all lanes.
471 */
472 VectorizedArray(const Number scalar)
473 {
474 static_assert(width == 1,
475 "You specified an illegal width that is not supported.");
476
477 this->operator=(scalar);
478 }
479
480 /**
481 * Construct an array with the given initializer list.
482 */
483 template <typename U>
484 VectorizedArray(const std::initializer_list<U> &list)
485 : VectorizedArrayBase<VectorizedArray<Number, width>, 1>(list)
486 {
487 static_assert(width == 1,
488 "You specified an illegal width that is not supported.");
489 }
490
491 /**
492 * This function assigns a scalar to the current object.
493 */
494 DEAL_II_ALWAYS_INLINE
495 VectorizedArray &
496 operator=(const Number scalar) &
497 {
498 data = scalar;
499 return *this;
500 }
501
502 /**
503 * Assign a scalar to the current object. This overload is used for
504 * rvalue references; because it does not make sense to assign
505 * something to a temporary, the function is deleted.
506 */
507 VectorizedArray &
508 operator=(const Number scalar) && = delete;
509
510 /**
511 * Access operator (only valid with component 0 in the base class without
512 * specialization).
513 */
514 DEAL_II_ALWAYS_INLINE
515 Number &
516 operator[](const unsigned int comp)
517 {
518 (void)comp;
519 AssertIndexRange(comp, 1);
520 return data;
521 }
522
523 /**
524 * Constant access operator (only valid with component 0 in the base class
525 * without specialization).
526 */
527 DEAL_II_ALWAYS_INLINE
528 const Number &
529 operator[](const unsigned int comp) const
530 {
531 (void)comp;
532 AssertIndexRange(comp, 1);
533 return data;
534 }
535
536 /**
537 * Element-wise addition of two arrays of numbers.
538 */
539 DEAL_II_ALWAYS_INLINE
540 VectorizedArray &
541 operator+=(const VectorizedArray &vec)
542 {
543 data += vec.data;
544 return *this;
545 }
546
547 /**
548 * Element-wise subtraction of two arrays of numbers.
549 */
550 DEAL_II_ALWAYS_INLINE
551 VectorizedArray &
552 operator-=(const VectorizedArray &vec)
553 {
554 data -= vec.data;
555 return *this;
556 }
557
558 /**
559 * Element-wise multiplication of two arrays of numbers.
560 */
561 DEAL_II_ALWAYS_INLINE
562 VectorizedArray &
563 operator*=(const VectorizedArray &vec)
564 {
565 data *= vec.data;
566 return *this;
567 }
568
569 /**
570 * Element-wise division of two arrays of numbers.
571 */
572 DEAL_II_ALWAYS_INLINE
573 VectorizedArray &
574 operator/=(const VectorizedArray &vec)
575 {
576 data /= vec.data;
577 return *this;
578 }
579
580 /**
581 * Load size() data items from memory into the calling class, starting at
582 * the given address. The pointer `ptr` needs not be aligned by the amount
583 * of bytes in the vectorized array, as opposed to casting a double address
584 * to VectorizedArray<double>*.
585 */
586 template <typename OtherNumber>
587 DEAL_II_ALWAYS_INLINE void
588 load(const OtherNumber *ptr)
589 {
590 data = *ptr;
591 }
592
593 /**
594 * Write the content of the calling class into memory in form of
595 * size() data items to the given address. The pointer `ptr` needs not be
596 * aligned by the amount of bytes in the vectorized array, as opposed to
597 * casting a double address to VectorizedArray<double>*.
598 */
599 template <typename OtherNumber>
600 DEAL_II_ALWAYS_INLINE void
601 store(OtherNumber *ptr) const
602 {
603 *ptr = data;
604 }
605
606 /**
607 * Write the content of the calling class into memory in form of
608 * size() data items to the given address using non-temporal stores that
609 * bypass the processor's caches, using @p _mm_stream_pd store intrinsics on
610 * supported CPUs. The destination of the store @p ptr must be aligned by
611 * the amount of bytes in the vectorized array.
612 *
613 * This store operation can be faster than usual store operations in case
614 * the store is streaming because it avoids the read-for-ownership transfer
615 * typically invoked in standard stores. This approximately works as follows
616 * (see the literature on computer architecture for details): When an
617 * algorithm stores some results to a memory address, a processor typically
618 * wants to move it into some of its caches as it expects the data to be
619 * re-used again at some point. Since caches are organized in lines of sizes
620 * either 64 byte or 128 byte but writes are usually smaller, a processor
621 * must first load in the destination cache line upon a write because only
622 * part of the cache line is overwritten initially. If a series of stores
623 * write data in a chunk bigger than any of its caches could handle, the
624 * data finally has to be moved out from the caches to main memory. But
625 * since all addressed have first been read, this doubles the load on main
626 * memory, which can incur a performance penalty. Furthermore, the
627 * organization of caches in a multicore context also requires reading an
628 * address before something can be written to cache to that address, see
629 * e.g. the <a href="https://en.wikipedia.org/wiki/MESI_protocol">Wikipedia
630 * article on the MESI protocol</a> for details. The instruction underlying
631 * this function call signals to the processor that these two prerequisites
632 * on a store are relaxed: Firstly, one expects the whole cache line to be
633 * overwritten (meaning that the memory subsystem makes sure that
634 * consecutive stores that together span a cache line are merged, and
635 * appropriately handling the case where only part of a cache line is
636 * written), so there is no need to first read the "remainder" of the cache
637 * line. Secondly, the data behind that particular memory will not be
638 * subject to cache coherency protocol as it will be in main memory both
639 * when the same processor wants to access it again as well as any other
640 * processors in a multicore chip. Due to this particular setup, any
641 * subsequent access to the data written by this function will need to query
642 * main memory, which is slower than an access from a cache both
643 * latency-wise and throughput-wise. Thus, this command should only be used
644 * for storing large arrays that will collectively not fit into caches, as
645 * performance will be degraded otherwise. For a typical use case, see also
646 * <a href="https://blogs.fau.de/hager/archives/2103">this blog article</a>.
647 *
648 * Note that streaming stores are only available in the specialized SSE/AVX
649 * classes of VectorizedArray of type @p double or @p float, not in the
650 * generic base class.
651 */
652 DEAL_II_ALWAYS_INLINE
653 void
654 streaming_store(Number *ptr) const
655 {
656 *ptr = data;
657 }
658
659 /**
660 * Load size() data items from memory into the calling class, starting at
661 * the given address and with given offsets, each entry from the offset
662 * providing one element of the vectorized array.
663 *
664 * This operation corresponds to the following code (but uses a more
665 * efficient implementation in case the hardware allows for that):
666 * @code
667 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
668 * this->operator[](v) = base_ptr[offsets[v]];
669 * @endcode
670 */
671 DEAL_II_ALWAYS_INLINE
672 void
673 gather(const Number *base_ptr, const unsigned int *offsets)
674 {
675 data = base_ptr[offsets[0]];
676 }
677
678 /**
679 * Write the content of the calling class into memory in form of
680 * size() data items to the given address and the given offsets, filling the
681 * elements of the vectorized array into each offset.
682 *
683 * This operation corresponds to the following code (but uses a more
684 * efficient implementation in case the hardware allows for that):
685 * @code
686 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
687 * base_ptr[offsets[v]] = this->operator[](v);
688 * @endcode
689 */
690 DEAL_II_ALWAYS_INLINE
691 void
692 scatter(const unsigned int *offsets, Number *base_ptr) const
693 {
694 base_ptr[offsets[0]] = data;
695 }
696
697 /**
698 * Returns sum over entries of the data field, $\sum_{i=1}^{\text{size}()}
699 * this->data[i]$.
700 */
701 DEAL_II_ALWAYS_INLINE
702 Number
703 sum() const
704 {
705 return data;
706 }
707
708 /**
709 * Actual data field. To be consistent with the standard layout type and to
710 * enable interaction with external SIMD functionality, this member is
711 * declared public.
712 */
713 Number data;
714
715 private:
716 /**
717 * Return the square root of this field. Not for use in user code. Use
718 * sqrt(x) instead.
719 */
720 DEAL_II_ALWAYS_INLINE
721 VectorizedArray
722 get_sqrt() const
723 {
724 VectorizedArray res;
725 res.data = std::sqrt(data);
726 return res;
727 }
728
729 /**
730 * Return the absolute value of this field. Not for use in user code. Use
731 * abs(x) instead.
732 */
733 DEAL_II_ALWAYS_INLINE
734 VectorizedArray
735 get_abs() const
736 {
737 VectorizedArray res;
738 res.data = std::fabs(data);
739 return res;
740 }
741
742 /**
743 * Return the component-wise maximum of this field and another one. Not for
744 * use in user code. Use max(x,y) instead.
745 */
746 DEAL_II_ALWAYS_INLINE
747 VectorizedArray
748 get_max(const VectorizedArray &other) const
749 {
750 VectorizedArray res;
751 res.data = std::max(data, other.data);
752 return res;
753 }
754
755 /**
756 * Return the component-wise minimum of this field and another one. Not for
757 * use in user code. Use min(x,y) instead.
758 */
759 DEAL_II_ALWAYS_INLINE
760 VectorizedArray
761 get_min(const VectorizedArray &other) const
762 {
763 VectorizedArray res;
764 res.data = std::min(data, other.data);
765 return res;
766 }
767
768 // Make a few functions friends.
769 template <typename Number2, std::size_t width2>
770 friend VectorizedArray<Number2, width2>
771 std::sqrt(const VectorizedArray<Number2, width2> &);
772 template <typename Number2, std::size_t width2>
773 friend VectorizedArray<Number2, width2>
774 std::abs(const VectorizedArray<Number2, width2> &);
775 template <typename Number2, std::size_t width2>
776 friend VectorizedArray<Number2, width2>
777 std::max(const VectorizedArray<Number2, width2> &,
778 const VectorizedArray<Number2, width2> &);
779 template <typename Number2, std::size_t width2>
780 friend VectorizedArray<Number2, width2>
781 std::min(const VectorizedArray<Number2, width2> &,
782 const VectorizedArray<Number2, width2> &);
783 };
784
785
786
787 /**
788 * @name Packing and unpacking of a VectorizedArray
789 * @{
790 */
791
792 /**
793 * Create a vectorized array that sets all entries in the array to the given
794 * scalar, i.e., broadcasts the scalar to all array elements.
795 *
796 * @relatesalso VectorizedArray
797 */
798 template <typename Number,
799 std::size_t width =
800 internal::VectorizedArrayWidthSpecifier<Number>::max_width>
801 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
802 make_vectorized_array(const Number &u)
803 {
804 VectorizedArray<Number, width> result = u;
805 return result;
806 }
807
808
809
810 /**
811 * Create a vectorized array of given type and broadcast the scalar value
812 * to all array elements.
813 *
814 * @relatesalso VectorizedArray
815 */
816 template <typename VectorizedArrayType>
817 inline DEAL_II_ALWAYS_INLINE VectorizedArrayType
818 make_vectorized_array(const typename VectorizedArrayType::value_type &u)
819 {
820 static_assert(
821 std::is_same_v<VectorizedArrayType,
822 VectorizedArray<typename VectorizedArrayType::value_type,
823 VectorizedArrayType::size()>>,
824 "VectorizedArrayType is not a VectorizedArray.");
825
826 VectorizedArrayType result = u;
827 return result;
828 }
829
830
831
832 /**
833 * Load size() data items from memory into the VectorizedArray @p out,
834 * starting at the given addresses and with given offset, each entry from the
835 * offset providing one element of the vectorized array.
836 *
837 * This operation corresponds to the following code:
838 * @code
839 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
840 * out.data[v] = ptrs[v][offset];
841 * @endcode
842 */
843 template <typename Number, std::size_t width>
844 inline DEAL_II_ALWAYS_INLINE void
845 gather(VectorizedArray<Number, width> &out,
846 const std::array<Number *, width> &ptrs,
847 const unsigned int offset)
848 {
849 for (unsigned int v = 0; v < width; ++v)
850 out.data[v] = ptrs[v][offset];
851 }
852
853
854
855 /**
856 * This method loads VectorizedArray::size() data streams from the
857 * given array @p in. The offsets to the input array are given by the array @p
858 * offsets. From each stream, n_entries are read. The data is then transposed
859 * and stored it into an array of VectorizedArray type. The output array @p
860 * out is expected to be an array of size @p n_entries. This method operates
861 * on plain arrays, so no checks for valid data access are made. It is the
862 * user's responsibility to ensure that the given arrays are valid according
863 * to the access layout below.
864 *
865 * This operation corresponds to a transformation of an array-of-struct
866 * (input) into a struct-of-array (output) according to the following formula:
867 *
868 * @code
869 * for (unsigned int i=0; i<n_entries; ++i)
870 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
871 * out[i][v] = in[offsets[v]+i];
872 * @endcode
873 *
874 * A more optimized version of this code will be used for supported types.
875 *
876 * This is the inverse operation to vectorized_transpose_and_store().
877 *
878 * @relatesalso VectorizedArray
879 */
880 template <typename Number, std::size_t width>
881 inline DEAL_II_ALWAYS_INLINE void
882 vectorized_load_and_transpose(const unsigned int n_entries,
883 const Number *in,
884 const unsigned int *offsets,
885 VectorizedArray<Number, width> *out)
886 {
887 for (unsigned int i = 0; i < n_entries; ++i)
888 for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
889 out[i][v] = in[offsets[v] + i];
890 }
891
892
893 /**
894 * The same as above with the difference that an array of pointers are
895 * passed in as input argument @p in.
896 *
897 * In analogy to the function above, one can consider that
898 * `in+offset[v]` is precomputed and passed as input argument.
899 *
900 * However, this function can also be used if some function returns an array
901 * of pointers and no assumption can be made that they belong to the same array,
902 * i.e., they can have their origin in different memory allocations.
903 */
904 template <typename Number, std::size_t width>
905 inline DEAL_II_ALWAYS_INLINE void
906 vectorized_load_and_transpose(const unsigned int n_entries,
907 const std::array<Number *, width> &in,
908 VectorizedArray<Number, width> *out)
909 {
910 for (unsigned int i = 0; i < n_entries; ++i)
911 for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
912 out[i][v] = in[v][i];
913 }
914
915
916
917 /**
918 * This method stores the vectorized arrays in transposed form into the given
919 * output array @p out with the given offsets @p offsets. This operation
920 * corresponds to a transformation of a struct-of-array (input) into an
921 * array-of-struct (output). This method operates on plain array, so no checks
922 * for valid data access are made. It is the user's responsibility to ensure
923 * that the given arrays are valid according to the access layout below.
924 *
925 * This method assumes that the specified offsets do not overlap. Otherwise,
926 * the behavior is undefined in the vectorized case. It is the user's
927 * responsibility to make sure that the access does not overlap and avoid
928 * undefined behavior.
929 *
930 * The argument @p add_into selects where the entries should only be written
931 * into the output arrays or the result should be added into the existing
932 * entries in the output. For <code>add_into == false</code>, the following
933 * code is assumed:
934 *
935 * @code
936 * for (unsigned int i=0; i<n_entries; ++i)
937 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
938 * out[offsets[v]+i] = in[i][v];
939 * @endcode
940 *
941 * For <code>add_into == true</code>, the code implements the following
942 * action:
943 * @code
944 * for (unsigned int i=0; i<n_entries; ++i)
945 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
946 * out[offsets[v]+i] += in[i][v];
947 * @endcode
948 *
949 * A more optimized version of this code will be used for supported types.
950 *
951 * This is the inverse operation to vectorized_load_and_transpose().
952 *
953 * @relatesalso VectorizedArray
954 */
955 template <typename Number, std::size_t width>
956 inline DEAL_II_ALWAYS_INLINE void
957 vectorized_transpose_and_store(const bool add_into,
958 const unsigned int n_entries,
959 const VectorizedArray<Number, width> *in,
960 const unsigned int *offsets,
961 Number *out)
962 {
963 if (add_into)
964 for (unsigned int i = 0; i < n_entries; ++i)
965 for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
966 out[offsets[v] + i] += in[i][v];
967 else
968 for (unsigned int i = 0; i < n_entries; ++i)
969 for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
970 out[offsets[v] + i] = in[i][v];
971 }
972
973
974 /**
975 * The same as above with the difference that an array of pointers are
976 * passed in as input argument @p out.
977 *
978 * In analogy to the function above, one can consider that
979 * `out+offset[v]` is precomputed and passed as input argument.
980 *
981 * However, this function can also be used if some function returns an array
982 * of pointers and no assumption can be made that they belong to the same array,
983 * i.e., they can have their origin in different memory allocations.
984 */
985 template <typename Number, std::size_t width>
986 inline DEAL_II_ALWAYS_INLINE void
987 vectorized_transpose_and_store(const bool add_into,
988 const unsigned int n_entries,
989 const VectorizedArray<Number, width> *in,
990 std::array<Number *, width> &out)
991 {
992 if (add_into)
993 for (unsigned int i = 0; i < n_entries; ++i)
994 for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
995 out[v][i] += in[i][v];
996 else
997 for (unsigned int i = 0; i < n_entries; ++i)
998 for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
999 out[v][i] = in[i][v];
1000 }
1001
1002
1003 /** @} */
1004
1005 #ifndef DOXYGEN
1006
1007 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 128 && defined(__ARM_NEON)
1008
1009 /**
1010 * Specialization for double and ARM Neon.
1011 */
1012 template <>
1013 class VectorizedArray<double, 2>
1014 : public VectorizedArrayBase<VectorizedArray<double, 2>, 2>
1015 {
1016 public:
1017 /**
1018 * This gives the type of the array elements.
1019 */
1020 using value_type = double;
1021
1022 /**
1023 * Record the fact that the given specialization of VectorizedArray is
1024 * indeed implemented.
1025 */
1026 static constexpr bool is_implemented = true;
1027
1028 /**
1029 * Default empty constructor, leaving the data in an uninitialized state
1030 * similar to float/double.
1031 */
1032 VectorizedArray() = default;
1033
1034 /**
1035 * Construct an array with the given scalar broadcast to all lanes.
1036 */
1037 VectorizedArray(const double scalar)
1038 {
1039 this->operator=(scalar);
1040 }
1041
1042 /**
1043 * Construct an array with the given initializer list.
1044 */
1045 template <typename U>
1046 VectorizedArray(const std::initializer_list<U> &list)
1047 : VectorizedArrayBase<VectorizedArray<double, 2>, 2>(list)
1048 {}
1049
1050 /**
1051 * This function can be used to set all data fields to a given scalar.
1052 */
1053 VectorizedArray &
1054 operator=(const double x) &
1055 {
1056 data = vdupq_n_f64(x);
1057 return *this;
1058 }
1059
1060 /**
1061 * Assign a scalar to the current object. This overload is used for
1062 * rvalue references; because it does not make sense to assign
1063 * something to a temporary, the function is deleted.
1064 */
1065 VectorizedArray &
1066 operator=(const double scalar) && = delete;
1067
1068 /**
1069 * Access operator.
1070 */
1071 double &
1072 operator[](const unsigned int comp)
1073 {
1074 return *(reinterpret_cast<double *>(&data) + comp);
1075 }
1076
1077 /**
1078 * Constant access operator.
1079 */
1080 const double &
1081 operator[](const unsigned int comp) const
1082 {
1083 return *(reinterpret_cast<const double *>(&data) + comp);
1084 }
1085
1086 /**
1087 * Element-wise addition of two arrays of numbers.
1088 */
1089 VectorizedArray &
1090 operator+=(const VectorizedArray &vec)
1091 {
1092 data = vaddq_f64(data, vec.data);
1093 return *this;
1094 }
1095
1096 /**
1097 * Element-wise subtraction of two arrays of numbers.
1098 */
1099 VectorizedArray &
1100 operator-=(const VectorizedArray &vec)
1101 {
1102 data = vsubq_f64(data, vec.data);
1103 return *this;
1104 }
1105
1106 /**
1107 * Element-wise multiplication of two arrays of numbers.
1108 */
1109 VectorizedArray &
1110 operator*=(const VectorizedArray &vec)
1111 {
1112 data = vmulq_f64(data, vec.data);
1113 return *this;
1114 }
1115
1116 /**
1117 * Element-wise division of two arrays of numbers.
1118 */
1119 VectorizedArray &
1120 operator/=(const VectorizedArray &vec)
1121 {
1122 data = vdivq_f64(data, vec.data);
1123 return *this;
1124 }
1125
1126 /**
1127 * Load @p size() from memory into the calling class, starting at
1128 * the given address. The memory need not be aligned by 16 bytes, as opposed
1129 * to casting a double address to VectorizedArray<double>*.
1130 */
1131 void
1132 load(const double *ptr)
1133 {
1134 data = vld1q_f64(ptr);
1135 }
1136
1137 DEAL_II_ALWAYS_INLINE
1138 void
1139 load(const float *ptr)
1140 {
1141 DEAL_II_OPENMP_SIMD_PRAGMA
1142 for (unsigned int i = 0; i < 2; ++i)
1143 data[i] = ptr[i];
1144 }
1145
1146 /**
1147 * Write the content of the calling class into memory in form of @p
1148 * size() to the given address. The memory need not be aligned by
1149 * 16 bytes, as opposed to casting a double address to
1150 * VectorizedArray<double>*.
1151 */
1152 void
1153 store(double *ptr) const
1154 {
1155 vst1q_f64(ptr, data);
1156 }
1157
1158 DEAL_II_ALWAYS_INLINE
1159 void
1160 store(float *ptr) const
1161 {
1162 DEAL_II_OPENMP_SIMD_PRAGMA
1163 for (unsigned int i = 0; i < 2; ++i)
1164 ptr[i] = data[i];
1165 }
1166
1167 /**
1168 * @copydoc VectorizedArray<Number>::streaming_store()
1169 * @note Memory must be aligned by 16 bytes.
1170 */
1171 DEAL_II_ALWAYS_INLINE
1172 void
1173 streaming_store(double *ptr) const
1174 {
1175 Assert(reinterpret_cast<std::size_t>(ptr) % 16 == 0,
1176 ExcMessage("Memory not aligned"));
1177 vst1q_f64(ptr, data);
1178 }
1179
1180 /**
1181 * Load @p size() from memory into the calling class, starting at
1182 * the given address and with given offsets, each entry from the offset
1183 * providing one element of the vectorized array.
1184 *
1185 * This operation corresponds to the following code (but uses a more
1186 * efficient implementation in case the hardware allows for that):
1187 * @code
1188 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
1189 * this->operator[](v) = base_ptr[offsets[v]];
1190 * @endcode
1191 */
1192 void
1193 gather(const double *base_ptr, const unsigned int *offsets)
1194 {
1195 for (unsigned int i = 0; i < 2; ++i)
1196 *(reinterpret_cast<double *>(&data) + i) = base_ptr[offsets[i]];
1197 }
1198
1199 /**
1200 * Write the content of the calling class into memory in form of @p
1201 * size() to the given address and the given offsets, filling the
1202 * elements of the vectorized array into each offset.
1203 *
1204 * This operation corresponds to the following code (but uses a more
1205 * efficient implementation in case the hardware allows for that):
1206 * @code
1207 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
1208 * base_ptr[offsets[v]] = this->operator[](v);
1209 * @endcode
1210 */
1211 void
1212 scatter(const unsigned int *offsets, double *base_ptr) const
1213 {
1214 for (unsigned int i = 0; i < 2; ++i)
1215 base_ptr[offsets[i]] = *(reinterpret_cast<const double *>(&data) + i);
1216 }
1217
1218 /**
1219 * Returns sum over entries of the data field, $\sum_{i=1}^{\text{size}()}
1220 * this->data[i]$.
1221 */
1222 double
1223 sum() const
1224 {
1225 return vaddvq_f64(data);
1226 }
1227
1228 /**
1229 * Actual data field. To be consistent with the standard layout type and to
1230 * enable interaction with external SIMD functionality, this member is
1231 * declared public.
1232 */
1233 mutable float64x2_t data;
1234
1235 private:
1236 /**
1237 * Return the square root of this field. Not for use in user code. Use
1238 * sqrt(x) instead.
1239 */
1240 VectorizedArray
1241 get_sqrt() const
1242 {
1243 VectorizedArray res;
1244 res.data = vsqrtq_f64(data);
1245 return res;
1246 }
1247
1248 /**
1249 * Return the absolute value of this field. Not for use in user code. Use
1250 * abs(x) instead.
1251 */
1252 VectorizedArray
1253 get_abs() const
1254 {
1255 VectorizedArray res;
1256 res.data = vabsq_f64(data);
1257 return res;
1258 }
1259
1260 /**
1261 * Return the component-wise maximum of this field and another one. Not for
1262 * use in user code. Use max(x,y) instead.
1263 */
1264 VectorizedArray
1265 get_max(const VectorizedArray &other) const
1266 {
1267 VectorizedArray res;
1268 res.data = vmaxq_f64(data, other.data);
1269 return res;
1270 }
1271
1272 /**
1273 * Return the component-wise minimum of this field and another one. Not for
1274 * use in user code. Use min(x,y) instead.
1275 */
1276 VectorizedArray
1277 get_min(const VectorizedArray &other) const
1278 {
1279 VectorizedArray res;
1280 res.data = vminq_f64(data, other.data);
1281 return res;
1282 }
1283
1284 // Make a few functions friends.
1285 template <typename Number2, std::size_t width2>
1286 friend VectorizedArray<Number2, width2>
1287 std::sqrt(const VectorizedArray<Number2, width2> &);
1288 template <typename Number2, std::size_t width2>
1289 friend VectorizedArray<Number2, width2>
1290 std::abs(const VectorizedArray<Number2, width2> &);
1291 template <typename Number2, std::size_t width2>
1292 friend VectorizedArray<Number2, width2>
1293 std::max(const VectorizedArray<Number2, width2> &,
1294 const VectorizedArray<Number2, width2> &);
1295 template <typename Number2, std::size_t width2>
1296 friend VectorizedArray<Number2, width2>
1297 std::min(const VectorizedArray<Number2, width2> &,
1298 const VectorizedArray<Number2, width2> &);
1299 };
1300
1301 /**
1302 * Specialization for float and ARM Neon.
1303 */
1304 template <>
1305 class VectorizedArray<float, 4>
1306 : public VectorizedArrayBase<VectorizedArray<float, 4>, 4>
1307 {
1308 public:
1309 /**
1310 * This gives the type of the array elements.
1311 */
1312 using value_type = float;
1313
1314 /**
1315 * Record the fact that the given specialization of VectorizedArray is
1316 * indeed implemented.
1317 */
1318 static constexpr bool is_implemented = true;
1319
1320 /**
1321 * Default empty constructor, leaving the data in an uninitialized state
1322 * similar to float/double.
1323 */
1324 VectorizedArray() = default;
1325
1326 /**
1327 * Construct an array with the given scalar broadcast to all lanes.
1328 */
1329 VectorizedArray(const float scalar)
1330 {
1331 this->operator=(scalar);
1332 }
1333
1334 /**
1335 * Construct an array with the given initializer list.
1336 */
1337 template <typename U>
1338 VectorizedArray(const std::initializer_list<U> &list)
1339 : VectorizedArrayBase<VectorizedArray<float, 4>, 4>(list)
1340 {}
1341
1342 /**
1343 * This function can be used to set all data fields to a given scalar.
1344 */
1345 VectorizedArray &
1346 operator=(const float x) &
1347 {
1348 data = vdupq_n_f32(x);
1349 return *this;
1350 }
1351
1352 /**
1353 * Assign a scalar to the current object. This overload is used for
1354 * rvalue references; because it does not make sense to assign
1355 * something to a temporary, the function is deleted.
1356 */
1357 VectorizedArray &
1358 operator=(const float scalar) && = delete;
1359
1360 /**
1361 * Access operator.
1362 */
1363 value_type &
1364 operator[](const unsigned int comp)
1365 {
1366 return *(reinterpret_cast<float *>(&data) + comp);
1367 }
1368
1369 /**
1370 * Constant access operator.
1371 */
1372 const value_type &
1373 operator[](const unsigned int comp) const
1374 {
1375 return *(reinterpret_cast<const float *>(&data) + comp);
1376 }
1377
1378 /**
1379 * Element-wise addition of two arrays of numbers.
1380 */
1381 VectorizedArray &
1382 operator+=(const VectorizedArray &vec)
1383 {
1384 data = vaddq_f32(data, vec.data);
1385 return *this;
1386 }
1387
1388 /**
1389 * Element-wise subtraction of two arrays of numbers.
1390 */
1391 VectorizedArray &
1392 operator-=(const VectorizedArray &vec)
1393 {
1394 data = vsubq_f32(data, vec.data);
1395 return *this;
1396 }
1397
1398 /**
1399 * Element-wise multiplication of two arrays of numbers.
1400 */
1401 VectorizedArray &
1402 operator*=(const VectorizedArray &vec)
1403 {
1404 data = vmulq_f32(data, vec.data);
1405 return *this;
1406 }
1407
1408 /**
1409 * Element-wise division of two arrays of numbers.
1410 */
1411 VectorizedArray &
1412 operator/=(const VectorizedArray &vec)
1413 {
1414 data = vdivq_f32(data, vec.data);
1415 return *this;
1416 }
1417
1418 /**
1419 * Load @p size() from memory into the calling class, starting at
1420 * the given address. The memory need not be aligned by 16 bytes, as opposed
1421 * to casting a float address to VectorizedArray<float>*.
1422 */
1423 void
1424 load(const float *ptr)
1425 {
1426 data = vld1q_f32(ptr);
1427 }
1428
1429 /**
1430 * Write the content of the calling class into memory in form of @p
1431 * size() to the given address. The memory need not be aligned by
1432 * 16 bytes, as opposed to casting a float address to
1433 * VectorizedArray<float>*.
1434 */
1435 void
1436 store(float *ptr) const
1437 {
1438 vst1q_f32(ptr, data);
1439 }
1440
1441 /**
1442 * @copydoc VectorizedArray<Number>::streaming_store()
1443 * @note Memory must be aligned by 16 bytes.
1444 */
1445 DEAL_II_ALWAYS_INLINE
1446 void
1447 streaming_store(float *ptr) const
1448 {
1449 Assert(reinterpret_cast<std::size_t>(ptr) % 16 == 0,
1450 ExcMessage("Memory not aligned"));
1451 vst1q_f32(ptr, data);
1452 }
1453
1454 /**
1455 * Load @p size() from memory into the calling class, starting at
1456 * the given address and with given offsets, each entry from the offset
1457 * providing one element of the vectorized array.
1458 *
1459 * This operation corresponds to the following code (but uses a more
1460 * efficient implementation in case the hardware allows for that):
1461 * @code
1462 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
1463 * this->operator[](v) = base_ptr[offsets[v]];
1464 * @endcode
1465 */
1466 void
1467 gather(const float *base_ptr, const unsigned int *offsets)
1468 {
1469 for (unsigned int i = 0; i < 4; ++i)
1470 *(reinterpret_cast<float *>(&data) + i) = base_ptr[offsets[i]];
1471 }
1472
1473 /**
1474 * Write the content of the calling class into memory in form of @p
1475 * size() to the given address and the given offsets, filling the
1476 * elements of the vectorized array into each offset.
1477 *
1478 * This operation corresponds to the following code (but uses a more
1479 * efficient implementation in case the hardware allows for that):
1480 * @code
1481 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
1482 * base_ptr[offsets[v]] = this->operator[](v);
1483 * @endcode
1484 */
1485 void
1486 scatter(const unsigned int *offsets, float *base_ptr) const
1487 {
1488 for (unsigned int i = 0; i < 4; ++i)
1489 base_ptr[offsets[i]] = *(reinterpret_cast<const float *>(&data) + i);
1490 }
1491
1492 /**
1493 * Returns sum over entries of the data field, $\sum_{i=1}^{\text{size}()}
1494 * this->data[i]$.
1495 */
1496 float
1497 sum() const
1498 {
1499 return vaddvq_f32(data);
1500 }
1501
1502 /**
1503 * Actual data field. To be consistent with the standard layout type and to
1504 * enable interaction with external SIMD functionality, this member is
1505 * declared public.
1506 */
1507 mutable float32x4_t data;
1508
1509 private:
1510 /**
1511 * Return the square root of this field. Not for use in user code. Use
1512 * sqrt(x) instead.
1513 */
1514 VectorizedArray
1515 get_sqrt() const
1516 {
1517 VectorizedArray res;
1518 res.data = vsqrtq_f32(data);
1519 return res;
1520 }
1521
1522 /**
1523 * Return the absolute value of this field. Not for use in user code. Use
1524 * abs(x) instead.
1525 */
1526 VectorizedArray
1527 get_abs() const
1528 {
1529 VectorizedArray res;
1530 res.data = vabsq_f32(data);
1531 return res;
1532 }
1533
1534 /**
1535 * Return the component-wise maximum of this field and another one. Not for
1536 * use in user code. Use max(x,y) instead.
1537 */
1538 VectorizedArray
1539 get_max(const VectorizedArray &other) const
1540 {
1541 VectorizedArray res;
1542 res.data = vmaxq_f32(data, other.data);
1543 return res;
1544 }
1545
1546 /**
1547 * Return the component-wise minimum of this field and another one. Not for
1548 * use in user code. Use min(x,y) instead.
1549 */
1550 VectorizedArray
1551 get_min(const VectorizedArray &other) const
1552 {
1553 VectorizedArray res;
1554 res.data = vminq_f32(data, other.data);
1555 return res;
1556 }
1557
1558 // Make a few functions friends.
1559 template <typename Number2, std::size_t width2>
1560 friend VectorizedArray<Number2, width2>
1561 std::sqrt(const VectorizedArray<Number2, width2> &);
1562 template <typename Number2, std::size_t width2>
1563 friend VectorizedArray<Number2, width2>
1564 std::abs(const VectorizedArray<Number2, width2> &);
1565 template <typename Number2, std::size_t width2>
1566 friend VectorizedArray<Number2, width2>
1567 std::max(const VectorizedArray<Number2, width2> &,
1568 const VectorizedArray<Number2, width2> &);
1569 template <typename Number2, std::size_t width2>
1570 friend VectorizedArray<Number2, width2>
1571 std::min(const VectorizedArray<Number2, width2> &,
1572 const VectorizedArray<Number2, width2> &);
1573 };
1574
1575
1576 # endif
1577
1578 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 128 && defined(__SSE2__)
1579
1580 /**
1581 * Specialization for double and SSE2.
1582 */
1583 template <>
1584 class VectorizedArray<double, 2>
1585 : public VectorizedArrayBase<VectorizedArray<double, 2>, 2>
1586 {
1587 public:
1588 /**
1589 * This gives the type of the array elements.
1590 */
1591 using value_type = double;
1592
1593 /**
1594 * Record the fact that the given specialization of VectorizedArray is
1595 * indeed implemented.
1596 */
1597 static constexpr bool is_implemented = true;
1598
1599 /**
1600 * Default empty constructor, leaving the data in an uninitialized state
1601 * similar to float/double.
1602 */
1603 VectorizedArray() = default;
1604
1605 /**
1606 * Construct an array with the given scalar broadcast to all lanes.
1607 */
1608 VectorizedArray(const double scalar)
1609 {
1610 this->operator=(scalar);
1611 }
1612
1613 /**
1614 * Construct an array with the given initializer list.
1615 */
1616 template <typename U>
1617 VectorizedArray(const std::initializer_list<U> &list)
1618 : VectorizedArrayBase<VectorizedArray<double, 2>, 2>(list)
1619 {}
1620
1621 /**
1622 * This function can be used to set all data fields to a given scalar.
1623 */
1624 DEAL_II_ALWAYS_INLINE
1625 VectorizedArray &
1626 operator=(const double x) &
1627 {
1628 data = _mm_set1_pd(x);
1629 return *this;
1630 }
1631
1632 /**
1633 * Assign a scalar to the current object. This overload is used for
1634 * rvalue references; because it does not make sense to assign
1635 * something to a temporary, the function is deleted.
1636 */
1637 VectorizedArray &
1638 operator=(const double scalar) && = delete;
1639
1640 /**
1641 * Access operator.
1642 */
1643 DEAL_II_ALWAYS_INLINE
1644 double &
1645 operator[](const unsigned int comp)
1646 {
1647 AssertIndexRange(comp, 2);
1648 return *(reinterpret_cast<double *>(&data) + comp);
1649 }
1650
1651 /**
1652 * Constant access operator.
1653 */
1654 DEAL_II_ALWAYS_INLINE
1655 const double &
1656 operator[](const unsigned int comp) const
1657 {
1658 AssertIndexRange(comp, 2);
1659 return *(reinterpret_cast<const double *>(&data) + comp);
1660 }
1661
1662 /**
1663 * Element-wise addition of two arrays of numbers.
1664 */
1665 DEAL_II_ALWAYS_INLINE
1666 VectorizedArray &
1667 operator+=(const VectorizedArray &vec)
1668 {
1669 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
1670 data += vec.data;
1671 # else
1672 data = _mm_add_pd(data, vec.data);
1673 # endif
1674 return *this;
1675 }
1676
1677 /**
1678 * Element-wise subtraction of two arrays of numbers.
1679 */
1680 DEAL_II_ALWAYS_INLINE
1681 VectorizedArray &
1682 operator-=(const VectorizedArray &vec)
1683 {
1684 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
1685 data -= vec.data;
1686 # else
1687 data = _mm_sub_pd(data, vec.data);
1688 # endif
1689 return *this;
1690 }
1691
1692 /**
1693 * Element-wise multiplication of two arrays of numbers.
1694 */
1695 DEAL_II_ALWAYS_INLINE
1696 VectorizedArray &
1697 operator*=(const VectorizedArray &vec)
1698 {
1699 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
1700 data *= vec.data;
1701 # else
1702 data = _mm_mul_pd(data, vec.data);
1703 # endif
1704 return *this;
1705 }
1706
1707 /**
1708 * Element-wise division of two arrays of numbers.
1709 */
1710 DEAL_II_ALWAYS_INLINE
1711 VectorizedArray &
1712 operator/=(const VectorizedArray &vec)
1713 {
1714 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
1715 data /= vec.data;
1716 # else
1717 data = _mm_div_pd(data, vec.data);
1718 # endif
1719 return *this;
1720 }
1721
1722 /**
1723 * Load @p size() from memory into the calling class, starting at
1724 * the given address. The memory need not be aligned by 16 bytes, as opposed
1725 * to casting a double address to VectorizedArray<double>*.
1726 */
1727 DEAL_II_ALWAYS_INLINE
1728 void
1729 load(const double *ptr)
1730 {
1731 data = _mm_loadu_pd(ptr);
1732 }
1733
1734 DEAL_II_ALWAYS_INLINE
1735 void
1736 load(const float *ptr)
1737 {
1738 DEAL_II_OPENMP_SIMD_PRAGMA
1739 for (unsigned int i = 0; i < 2; ++i)
1740 data[i] = ptr[i];
1741 }
1742
1743 /**
1744 * Write the content of the calling class into memory in form of @p
1745 * size() to the given address. The memory need not be aligned by
1746 * 16 bytes, as opposed to casting a double address to
1747 * VectorizedArray<double>*.
1748 */
1749 DEAL_II_ALWAYS_INLINE
1750 void
1751 store(double *ptr) const
1752 {
1753 _mm_storeu_pd(ptr, data);
1754 }
1755
1756 DEAL_II_ALWAYS_INLINE
1757 void
1758 store(float *ptr) const
1759 {
1760 DEAL_II_OPENMP_SIMD_PRAGMA
1761 for (unsigned int i = 0; i < 2; ++i)
1762 ptr[i] = data[i];
1763 }
1764
1765 /**
1766 * @copydoc VectorizedArray<Number>::streaming_store()
1767 * @note Memory must be aligned by 16 bytes.
1768 */
1769 DEAL_II_ALWAYS_INLINE
1770 void
1771 streaming_store(double *ptr) const
1772 {
1773 Assert(reinterpret_cast<std::size_t>(ptr) % 16 == 0,
1774 ExcMessage("Memory not aligned"));
1775 _mm_stream_pd(ptr, data);
1776 }
1777
1778 /**
1779 * Load @p size() from memory into the calling class, starting at
1780 * the given address and with given offsets, each entry from the offset
1781 * providing one element of the vectorized array.
1782 *
1783 * This operation corresponds to the following code (but uses a more
1784 * efficient implementation in case the hardware allows for that):
1785 * @code
1786 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
1787 * this->operator[](v) = base_ptr[offsets[v]];
1788 * @endcode
1789 */
1790 DEAL_II_ALWAYS_INLINE
1791 void
1792 gather(const double *base_ptr, const unsigned int *offsets)
1793 {
1794 for (unsigned int i = 0; i < 2; ++i)
1795 *(reinterpret_cast<double *>(&data) + i) = base_ptr[offsets[i]];
1796 }
1797
1798 /**
1799 * Write the content of the calling class into memory in form of @p
1800 * size() to the given address and the given offsets, filling the
1801 * elements of the vectorized array into each offset.
1802 *
1803 * This operation corresponds to the following code (but uses a more
1804 * efficient implementation in case the hardware allows for that):
1805 * @code
1806 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
1807 * base_ptr[offsets[v]] = this->operator[](v);
1808 * @endcode
1809 */
1810 DEAL_II_ALWAYS_INLINE
1811 void
1812 scatter(const unsigned int *offsets, double *base_ptr) const
1813 {
1814 for (unsigned int i = 0; i < 2; ++i)
1815 base_ptr[offsets[i]] = *(reinterpret_cast<const double *>(&data) + i);
1816 }
1817
1818 /**
1819 * Returns sum over entries of the data field, $\sum_{i=1}^{\text{size}()}
1820 * this->data[i]$.
1821 */
1822 double
1823 sum() const
1824 {
1825 __m128d t1 = _mm_unpackhi_pd(data, data);
1826 __m128d t2 = _mm_add_pd(data, t1);
1827 return _mm_cvtsd_f64(t2);
1828 }
1829
1830 /**
1831 * Actual data field. To be consistent with the standard layout type and to
1832 * enable interaction with external SIMD functionality, this member is
1833 * declared public.
1834 */
1835 __m128d data;
1836
1837 private:
1838 /**
1839 * Return the square root of this field. Not for use in user code. Use
1840 * sqrt(x) instead.
1841 */
1842 DEAL_II_ALWAYS_INLINE
1843 VectorizedArray
1844 get_sqrt() const
1845 {
1846 VectorizedArray res;
1847 res.data = _mm_sqrt_pd(data);
1848 return res;
1849 }
1850
1851 /**
1852 * Return the absolute value of this field. Not for use in user code. Use
1853 * abs(x) instead.
1854 */
1855 DEAL_II_ALWAYS_INLINE
1856 VectorizedArray
1857 get_abs() const
1858 {
1859 // to compute the absolute value, perform
1860 // bitwise andnot with -0. This will leave all
1861 // value and exponent bits unchanged but force
1862 // the sign value to +.
1863 __m128d mask = _mm_set1_pd(-0.);
1864 VectorizedArray res;
1865 res.data = _mm_andnot_pd(mask, data);
1866 return res;
1867 }
1868
1869 /**
1870 * Return the component-wise maximum of this field and another one. Not for
1871 * use in user code. Use max(x,y) instead.
1872 */
1873 DEAL_II_ALWAYS_INLINE
1874 VectorizedArray
1875 get_max(const VectorizedArray &other) const
1876 {
1877 VectorizedArray res;
1878 res.data = _mm_max_pd(data, other.data);
1879 return res;
1880 }
1881
1882 /**
1883 * Return the component-wise minimum of this field and another one. Not for
1884 * use in user code. Use min(x,y) instead.
1885 */
1886 DEAL_II_ALWAYS_INLINE
1887 VectorizedArray
1888 get_min(const VectorizedArray &other) const
1889 {
1890 VectorizedArray res;
1891 res.data = _mm_min_pd(data, other.data);
1892 return res;
1893 }
1894
1895 // Make a few functions friends.
1896 template <typename Number2, std::size_t width2>
1897 friend VectorizedArray<Number2, width2>
1898 std::sqrt(const VectorizedArray<Number2, width2> &);
1899 template <typename Number2, std::size_t width2>
1900 friend VectorizedArray<Number2, width2>
1901 std::abs(const VectorizedArray<Number2, width2> &);
1902 template <typename Number2, std::size_t width2>
1903 friend VectorizedArray<Number2, width2>
1904 std::max(const VectorizedArray<Number2, width2> &,
1905 const VectorizedArray<Number2, width2> &);
1906 template <typename Number2, std::size_t width2>
1907 friend VectorizedArray<Number2, width2>
1908 std::min(const VectorizedArray<Number2, width2> &,
1909 const VectorizedArray<Number2, width2> &);
1910 };
1911
1912
1913
1914 /**
1915 * Specialization for double and SSE2.
1916 */
1917 template <>
1918 inline DEAL_II_ALWAYS_INLINE void
1919 vectorized_load_and_transpose(const unsigned int n_entries,
1920 const double *in,
1921 const unsigned int *offsets,
1922 VectorizedArray<double, 2> *out)
1923 {
1924 const unsigned int n_chunks = n_entries / 2;
1925 for (unsigned int i = 0; i < n_chunks; ++i)
1926 {
1927 __m128d u0 = _mm_loadu_pd(in + 2 * i + offsets[0]);
1928 __m128d u1 = _mm_loadu_pd(in + 2 * i + offsets[1]);
1929 out[2 * i + 0].data = _mm_unpacklo_pd(u0, u1);
1930 out[2 * i + 1].data = _mm_unpackhi_pd(u0, u1);
1931 }
1932
1933 // remainder loop of work that does not divide by 2
1934 for (unsigned int i = 2 * n_chunks; i < n_entries; ++i)
1935 for (unsigned int v = 0; v < 2; ++v)
1936 out[i][v] = in[offsets[v] + i];
1937 }
1938
1939
1940
1941 /**
1942 * Specialization for double and SSE2.
1943 */
1944 template <>
1945 inline DEAL_II_ALWAYS_INLINE void
1946 vectorized_load_and_transpose(const unsigned int n_entries,
1947 const std::array<double *, 2> &in,
1948 VectorizedArray<double, 2> *out)
1949 {
1950 // see the comments in the vectorized_load_and_transpose above
1951
1952 const unsigned int n_chunks = n_entries / 2;
1953 for (unsigned int i = 0; i < n_chunks; ++i)
1954 {
1955 __m128d u0 = _mm_loadu_pd(in[0] + 2 * i);
1956 __m128d u1 = _mm_loadu_pd(in[1] + 2 * i);
1957 out[2 * i + 0].data = _mm_unpacklo_pd(u0, u1);
1958 out[2 * i + 1].data = _mm_unpackhi_pd(u0, u1);
1959 }
1960
1961 for (unsigned int i = 2 * n_chunks; i < n_entries; ++i)
1962 for (unsigned int v = 0; v < 2; ++v)
1963 out[i][v] = in[v][i];
1964 }
1965
1966
1967
1968 /**
1969 * Specialization for double and SSE2.
1970 */
1971 template <>
1972 inline DEAL_II_ALWAYS_INLINE void
1973 vectorized_transpose_and_store(const bool add_into,
1974 const unsigned int n_entries,
1975 const VectorizedArray<double, 2> *in,
1976 const unsigned int *offsets,
1977 double *out)
1978 {
1979 const unsigned int n_chunks = n_entries / 2;
1980 if (add_into)
1981 {
1982 for (unsigned int i = 0; i < n_chunks; ++i)
1983 {
1984 __m128d u0 = in[2 * i + 0].data;
1985 __m128d u1 = in[2 * i + 1].data;
1986 __m128d res0 = _mm_unpacklo_pd(u0, u1);
1987 __m128d res1 = _mm_unpackhi_pd(u0, u1);
1988 _mm_storeu_pd(out + 2 * i + offsets[0],
1989 _mm_add_pd(_mm_loadu_pd(out + 2 * i + offsets[0]),
1990 res0));
1991 _mm_storeu_pd(out + 2 * i + offsets[1],
1992 _mm_add_pd(_mm_loadu_pd(out + 2 * i + offsets[1]),
1993 res1));
1994 }
1995 // remainder loop of work that does not divide by 2
1996 for (unsigned int i = 2 * n_chunks; i < n_entries; ++i)
1997 for (unsigned int v = 0; v < 2; ++v)
1998 out[offsets[v] + i] += in[i][v];
1999 }
2000 else
2001 {
2002 for (unsigned int i = 0; i < n_chunks; ++i)
2003 {
2004 __m128d u0 = in[2 * i + 0].data;
2005 __m128d u1 = in[2 * i + 1].data;
2006 __m128d res0 = _mm_unpacklo_pd(u0, u1);
2007 __m128d res1 = _mm_unpackhi_pd(u0, u1);
2008 _mm_storeu_pd(out + 2 * i + offsets[0], res0);
2009 _mm_storeu_pd(out + 2 * i + offsets[1], res1);
2010 }
2011 // remainder loop of work that does not divide by 2
2012 for (unsigned int i = 2 * n_chunks; i < n_entries; ++i)
2013 for (unsigned int v = 0; v < 2; ++v)
2014 out[offsets[v] + i] = in[i][v];
2015 }
2016 }
2017
2018
2019
2020 /**
2021 * Specialization for double and SSE2.
2022 */
2023 template <>
2024 inline DEAL_II_ALWAYS_INLINE void
2025 vectorized_transpose_and_store(const bool add_into,
2026 const unsigned int n_entries,
2027 const VectorizedArray<double, 2> *in,
2028 std::array<double *, 2> &out)
2029 {
2030 // see the comments in the vectorized_transpose_and_store above
2031
2032 const unsigned int n_chunks = n_entries / 2;
2033 if (add_into)
2034 {
2035 for (unsigned int i = 0; i < n_chunks; ++i)
2036 {
2037 __m128d u0 = in[2 * i + 0].data;
2038 __m128d u1 = in[2 * i + 1].data;
2039 __m128d res0 = _mm_unpacklo_pd(u0, u1);
2040 __m128d res1 = _mm_unpackhi_pd(u0, u1);
2041 _mm_storeu_pd(out[0] + 2 * i,
2042 _mm_add_pd(_mm_loadu_pd(out[0] + 2 * i), res0));
2043 _mm_storeu_pd(out[1] + 2 * i,
2044 _mm_add_pd(_mm_loadu_pd(out[1] + 2 * i), res1));
2045 }
2046
2047 for (unsigned int i = 2 * n_chunks; i < n_entries; ++i)
2048 for (unsigned int v = 0; v < 2; ++v)
2049 out[v][i] += in[i][v];
2050 }
2051 else
2052 {
2053 for (unsigned int i = 0; i < n_chunks; ++i)
2054 {
2055 __m128d u0 = in[2 * i + 0].data;
2056 __m128d u1 = in[2 * i + 1].data;
2057 __m128d res0 = _mm_unpacklo_pd(u0, u1);
2058 __m128d res1 = _mm_unpackhi_pd(u0, u1);
2059 _mm_storeu_pd(out[0] + 2 * i, res0);
2060 _mm_storeu_pd(out[1] + 2 * i, res1);
2061 }
2062
2063 for (unsigned int i = 2 * n_chunks; i < n_entries; ++i)
2064 for (unsigned int v = 0; v < 2; ++v)
2065 out[v][i] = in[i][v];
2066 }
2067 }
2068
2069
2070
2071 /**
2072 * Specialization for float and SSE2.
2073 */
2074 template <>
2075 class VectorizedArray<float, 4>
2076 : public VectorizedArrayBase<VectorizedArray<float, 4>, 4>
2077 {
2078 public:
2079 /**
2080 * This gives the type of the array elements.
2081 */
2082 using value_type = float;
2083
2084 /**
2085 * Record the fact that the given specialization of VectorizedArray is
2086 * indeed implemented.
2087 */
2088 static constexpr bool is_implemented = true;
2089
2090 /**
2091 * Default empty constructor, leaving the data in an uninitialized state
2092 * similar to float/double.
2093 */
2094 VectorizedArray() = default;
2095
2096 /**
2097 * Construct an array with the given scalar broadcast to all lanes.
2098 */
2099 VectorizedArray(const float scalar)
2100 {
2101 this->operator=(scalar);
2102 }
2103
2104 /**
2105 * Construct an array with the given initializer list.
2106 */
2107 template <typename U>
2108 VectorizedArray(const std::initializer_list<U> &list)
2109 : VectorizedArrayBase<VectorizedArray<float, 4>, 4>(list)
2110 {}
2111
2112 /**
2113 * This function can be used to set all data fields to a given scalar.
2114 */
2115 DEAL_II_ALWAYS_INLINE
2116 VectorizedArray &
2117 operator=(const float x) &
2118 {
2119 data = _mm_set1_ps(x);
2120 return *this;
2121 }
2122
2123 /**
2124 * Assign a scalar to the current object. This overload is used for
2125 * rvalue references; because it does not make sense to assign
2126 * something to a temporary, the function is deleted.
2127 */
2128 VectorizedArray &
2129 operator=(const float scalar) && = delete;
2130
2131 /**
2132 * Access operator.
2133 */
2134 DEAL_II_ALWAYS_INLINE
2135 float &
2136 operator[](const unsigned int comp)
2137 {
2138 AssertIndexRange(comp, 4);
2139 return *(reinterpret_cast<float *>(&data) + comp);
2140 }
2141
2142 /**
2143 * Constant access operator.
2144 */
2145 DEAL_II_ALWAYS_INLINE
2146 const float &
2147 operator[](const unsigned int comp) const
2148 {
2149 AssertIndexRange(comp, 4);
2150 return *(reinterpret_cast<const float *>(&data) + comp);
2151 }
2152
2153 /**
2154 * Element-wise addition of two arrays of numbers.
2155 */
2156 DEAL_II_ALWAYS_INLINE
2157 VectorizedArray &
2158 operator+=(const VectorizedArray &vec)
2159 {
2160 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
2161 data += vec.data;
2162 # else
2163 data = _mm_add_ps(data, vec.data);
2164 # endif
2165 return *this;
2166 }
2167
2168 /**
2169 * Element-wise subtraction of two arrays of numbers.
2170 */
2171 DEAL_II_ALWAYS_INLINE
2172 VectorizedArray &
2173 operator-=(const VectorizedArray &vec)
2174 {
2175 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
2176 data -= vec.data;
2177 # else
2178 data = _mm_sub_ps(data, vec.data);
2179 # endif
2180 return *this;
2181 }
2182
2183 /**
2184 * Element-wise multiplication of two arrays of numbers.
2185 */
2186 DEAL_II_ALWAYS_INLINE
2187 VectorizedArray &
2188 operator*=(const VectorizedArray &vec)
2189 {
2190 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
2191 data *= vec.data;
2192 # else
2193 data = _mm_mul_ps(data, vec.data);
2194 # endif
2195 return *this;
2196 }
2197
2198 /**
2199 * Element-wise division of two arrays of numbers.
2200 */
2201 DEAL_II_ALWAYS_INLINE
2202 VectorizedArray &
2203 operator/=(const VectorizedArray &vec)
2204 {
2205 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
2206 data /= vec.data;
2207 # else
2208 data = _mm_div_ps(data, vec.data);
2209 # endif
2210 return *this;
2211 }
2212
2213 /**
2214 * Load @p size() from memory into the calling class, starting at
2215 * the given address. The memory need not be aligned by 16 bytes, as opposed
2216 * to casting a float address to VectorizedArray<float>*.
2217 */
2218 DEAL_II_ALWAYS_INLINE
2219 void
2220 load(const float *ptr)
2221 {
2222 data = _mm_loadu_ps(ptr);
2223 }
2224
2225 /**
2226 * Write the content of the calling class into memory in form of @p
2227 * size() to the given address. The memory need not be aligned by
2228 * 16 bytes, as opposed to casting a float address to
2229 * VectorizedArray<float>*.
2230 */
2231 DEAL_II_ALWAYS_INLINE
2232 void
2233 store(float *ptr) const
2234 {
2235 _mm_storeu_ps(ptr, data);
2236 }
2237
2238 /**
2239 * @copydoc VectorizedArray<Number>::streaming_store()
2240 * @note Memory must be aligned by 16 bytes.
2241 */
2242 DEAL_II_ALWAYS_INLINE
2243 void
2244 streaming_store(float *ptr) const
2245 {
2246 Assert(reinterpret_cast<std::size_t>(ptr) % 16 == 0,
2247 ExcMessage("Memory not aligned"));
2248 _mm_stream_ps(ptr, data);
2249 }
2250
2251 /**
2252 * Load @p size() from memory into the calling class, starting at
2253 * the given address and with given offsets, each entry from the offset
2254 * providing one element of the vectorized array.
2255 *
2256 * This operation corresponds to the following code (but uses a more
2257 * efficient implementation in case the hardware allows for that):
2258 * @code
2259 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
2260 * this->operator[](v) = base_ptr[offsets[v]];
2261 * @endcode
2262 */
2263 DEAL_II_ALWAYS_INLINE
2264 void
2265 gather(const float *base_ptr, const unsigned int *offsets)
2266 {
2267 for (unsigned int i = 0; i < 4; ++i)
2268 *(reinterpret_cast<float *>(&data) + i) = base_ptr[offsets[i]];
2269 }
2270
2271 /**
2272 * Write the content of the calling class into memory in form of @p
2273 * size() to the given address and the given offsets, filling the
2274 * elements of the vectorized array into each offset.
2275 *
2276 * This operation corresponds to the following code (but uses a more
2277 * efficient implementation in case the hardware allows for that):
2278 * @code
2279 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
2280 * base_ptr[offsets[v]] = this->operator[](v);
2281 * @endcode
2282 */
2283 DEAL_II_ALWAYS_INLINE
2284 void
2285 scatter(const unsigned int *offsets, float *base_ptr) const
2286 {
2287 for (unsigned int i = 0; i < 4; ++i)
2288 base_ptr[offsets[i]] = *(reinterpret_cast<const float *>(&data) + i);
2289 }
2290
2291 /**
2292 * Returns sum over entries of the data field, $\sum_{i=1}^{\text{size}()}
2293 * this->data[i]$.
2294 */
2295 float
2296 sum() const
2297 {
2298 __m128 t1 = _mm_movehl_ps(data, data);
2299 __m128 t2 = _mm_add_ps(data, t1);
2300 __m128 t3 = _mm_shuffle_ps(t2, t2, 1);
2301 __m128 t4 = _mm_add_ss(t2, t3);
2302 return _mm_cvtss_f32(t4);
2303 }
2304
2305 /**
2306 * Actual data field. To be consistent with the standard layout type and to
2307 * enable interaction with external SIMD functionality, this member is
2308 * declared public.
2309 */
2310 __m128 data;
2311
2312 private:
2313 /**
2314 * Return the square root of this field. Not for use in user code. Use
2315 * sqrt(x) instead.
2316 */
2317 DEAL_II_ALWAYS_INLINE
2318 VectorizedArray
2319 get_sqrt() const
2320 {
2321 VectorizedArray res;
2322 res.data = _mm_sqrt_ps(data);
2323 return res;
2324 }
2325
2326 /**
2327 * Return the absolute value of this field. Not for use in user code. Use
2328 * abs(x) instead.
2329 */
2330 DEAL_II_ALWAYS_INLINE
2331 VectorizedArray
2332 get_abs() const
2333 {
2334 // to compute the absolute value, perform bitwise andnot with -0. This
2335 // will leave all value and exponent bits unchanged but force the sign
2336 // value to +.
2337 __m128 mask = _mm_set1_ps(-0.f);
2338 VectorizedArray res;
2339 res.data = _mm_andnot_ps(mask, data);
2340 return res;
2341 }
2342
2343 /**
2344 * Return the component-wise maximum of this field and another one. Not for
2345 * use in user code. Use max(x,y) instead.
2346 */
2347 DEAL_II_ALWAYS_INLINE
2348 VectorizedArray
2349 get_max(const VectorizedArray &other) const
2350 {
2351 VectorizedArray res;
2352 res.data = _mm_max_ps(data, other.data);
2353 return res;
2354 }
2355
2356 /**
2357 * Return the component-wise minimum of this field and another one. Not for
2358 * use in user code. Use min(x,y) instead.
2359 */
2360 DEAL_II_ALWAYS_INLINE
2361 VectorizedArray
2362 get_min(const VectorizedArray &other) const
2363 {
2364 VectorizedArray res;
2365 res.data = _mm_min_ps(data, other.data);
2366 return res;
2367 }
2368
2369 // Make a few functions friends.
2370 template <typename Number2, std::size_t width2>
2371 friend VectorizedArray<Number2, width2>
2372 std::sqrt(const VectorizedArray<Number2, width2> &);
2373 template <typename Number2, std::size_t width2>
2374 friend VectorizedArray<Number2, width2>
2375 std::abs(const VectorizedArray<Number2, width2> &);
2376 template <typename Number2, std::size_t width2>
2377 friend VectorizedArray<Number2, width2>
2378 std::max(const VectorizedArray<Number2, width2> &,
2379 const VectorizedArray<Number2, width2> &);
2380 template <typename Number2, std::size_t width2>
2381 friend VectorizedArray<Number2, width2>
2382 std::min(const VectorizedArray<Number2, width2> &,
2383 const VectorizedArray<Number2, width2> &);
2384 };
2385
2386
2387
2388 /**
2389 * Specialization for float and SSE2.
2390 */
2391 template <>
2392 inline DEAL_II_ALWAYS_INLINE void
2393 vectorized_load_and_transpose(const unsigned int n_entries,
2394 const float *in,
2395 const unsigned int *offsets,
2396 VectorizedArray<float, 4> *out)
2397 {
2398 const unsigned int n_chunks = n_entries / 4;
2399 for (unsigned int i = 0; i < n_chunks; ++i)
2400 {
2401 __m128 u0 = _mm_loadu_ps(in + 4 * i + offsets[0]);
2402 __m128 u1 = _mm_loadu_ps(in + 4 * i + offsets[1]);
2403 __m128 u2 = _mm_loadu_ps(in + 4 * i + offsets[2]);
2404 __m128 u3 = _mm_loadu_ps(in + 4 * i + offsets[3]);
2405 __m128 v0 = _mm_shuffle_ps(u0, u1, 0x44);
2406 __m128 v1 = _mm_shuffle_ps(u0, u1, 0xee);
2407 __m128 v2 = _mm_shuffle_ps(u2, u3, 0x44);
2408 __m128 v3 = _mm_shuffle_ps(u2, u3, 0xee);
2409 out[4 * i + 0].data = _mm_shuffle_ps(v0, v2, 0x88);
2410 out[4 * i + 1].data = _mm_shuffle_ps(v0, v2, 0xdd);
2411 out[4 * i + 2].data = _mm_shuffle_ps(v1, v3, 0x88);
2412 out[4 * i + 3].data = _mm_shuffle_ps(v1, v3, 0xdd);
2413 }
2414
2415 // remainder loop of work that does not divide by 4
2416 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
2417 for (unsigned int v = 0; v < 4; ++v)
2418 out[i][v] = in[offsets[v] + i];
2419 }
2420
2421
2422
2423 /**
2424 * Specialization for float and SSE2.
2425 */
2426 template <>
2427 inline DEAL_II_ALWAYS_INLINE void
2428 vectorized_load_and_transpose(const unsigned int n_entries,
2429 const std::array<float *, 4> &in,
2430 VectorizedArray<float, 4> *out)
2431 {
2432 // see the comments in the vectorized_load_and_transpose above
2433
2434 const unsigned int n_chunks = n_entries / 4;
2435 for (unsigned int i = 0; i < n_chunks; ++i)
2436 {
2437 __m128 u0 = _mm_loadu_ps(in[0] + 4 * i);
2438 __m128 u1 = _mm_loadu_ps(in[1] + 4 * i);
2439 __m128 u2 = _mm_loadu_ps(in[2] + 4 * i);
2440 __m128 u3 = _mm_loadu_ps(in[3] + 4 * i);
2441 __m128 v0 = _mm_shuffle_ps(u0, u1, 0x44);
2442 __m128 v1 = _mm_shuffle_ps(u0, u1, 0xee);
2443 __m128 v2 = _mm_shuffle_ps(u2, u3, 0x44);
2444 __m128 v3 = _mm_shuffle_ps(u2, u3, 0xee);
2445 out[4 * i + 0].data = _mm_shuffle_ps(v0, v2, 0x88);
2446 out[4 * i + 1].data = _mm_shuffle_ps(v0, v2, 0xdd);
2447 out[4 * i + 2].data = _mm_shuffle_ps(v1, v3, 0x88);
2448 out[4 * i + 3].data = _mm_shuffle_ps(v1, v3, 0xdd);
2449 }
2450
2451 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
2452 for (unsigned int v = 0; v < 4; ++v)
2453 out[i][v] = in[v][i];
2454 }
2455
2456
2457
2458 /**
2459 * Specialization for float and SSE2.
2460 */
2461 template <>
2462 inline DEAL_II_ALWAYS_INLINE void
2463 vectorized_transpose_and_store(const bool add_into,
2464 const unsigned int n_entries,
2465 const VectorizedArray<float, 4> *in,
2466 const unsigned int *offsets,
2467 float *out)
2468 {
2469 const unsigned int n_chunks = n_entries / 4;
2470 for (unsigned int i = 0; i < n_chunks; ++i)
2471 {
2472 __m128 u0 = in[4 * i + 0].data;
2473 __m128 u1 = in[4 * i + 1].data;
2474 __m128 u2 = in[4 * i + 2].data;
2475 __m128 u3 = in[4 * i + 3].data;
2476 __m128 t0 = _mm_shuffle_ps(u0, u1, 0x44);
2477 __m128 t1 = _mm_shuffle_ps(u0, u1, 0xee);
2478 __m128 t2 = _mm_shuffle_ps(u2, u3, 0x44);
2479 __m128 t3 = _mm_shuffle_ps(u2, u3, 0xee);
2480 u0 = _mm_shuffle_ps(t0, t2, 0x88);
2481 u1 = _mm_shuffle_ps(t0, t2, 0xdd);
2482 u2 = _mm_shuffle_ps(t1, t3, 0x88);
2483 u3 = _mm_shuffle_ps(t1, t3, 0xdd);
2484
2485 // Cannot use the same store instructions in both paths of the 'if'
2486 // because the compiler cannot know that there is no aliasing between
2487 // pointers
2488 if (add_into)
2489 {
2490 u0 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[0]), u0);
2491 _mm_storeu_ps(out + 4 * i + offsets[0], u0);
2492 u1 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[1]), u1);
2493 _mm_storeu_ps(out + 4 * i + offsets[1], u1);
2494 u2 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[2]), u2);
2495 _mm_storeu_ps(out + 4 * i + offsets[2], u2);
2496 u3 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[3]), u3);
2497 _mm_storeu_ps(out + 4 * i + offsets[3], u3);
2498 }
2499 else
2500 {
2501 _mm_storeu_ps(out + 4 * i + offsets[0], u0);
2502 _mm_storeu_ps(out + 4 * i + offsets[1], u1);
2503 _mm_storeu_ps(out + 4 * i + offsets[2], u2);
2504 _mm_storeu_ps(out + 4 * i + offsets[3], u3);
2505 }
2506 }
2507
2508 // remainder loop of work that does not divide by 4
2509 if (add_into)
2510 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
2511 for (unsigned int v = 0; v < 4; ++v)
2512 out[offsets[v] + i] += in[i][v];
2513 else
2514 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
2515 for (unsigned int v = 0; v < 4; ++v)
2516 out[offsets[v] + i] = in[i][v];
2517 }
2518
2519
2520
2521 /**
2522 * Specialization for float and SSE2.
2523 */
2524 template <>
2525 inline DEAL_II_ALWAYS_INLINE void
2526 vectorized_transpose_and_store(const bool add_into,
2527 const unsigned int n_entries,
2528 const VectorizedArray<float, 4> *in,
2529 std::array<float *, 4> &out)
2530 {
2531 // see the comments in the vectorized_transpose_and_store above
2532
2533 const unsigned int n_chunks = n_entries / 4;
2534 for (unsigned int i = 0; i < n_chunks; ++i)
2535 {
2536 __m128 u0 = in[4 * i + 0].data;
2537 __m128 u1 = in[4 * i + 1].data;
2538 __m128 u2 = in[4 * i + 2].data;
2539 __m128 u3 = in[4 * i + 3].data;
2540 __m128 t0 = _mm_shuffle_ps(u0, u1, 0x44);
2541 __m128 t1 = _mm_shuffle_ps(u0, u1, 0xee);
2542 __m128 t2 = _mm_shuffle_ps(u2, u3, 0x44);
2543 __m128 t3 = _mm_shuffle_ps(u2, u3, 0xee);
2544 u0 = _mm_shuffle_ps(t0, t2, 0x88);
2545 u1 = _mm_shuffle_ps(t0, t2, 0xdd);
2546 u2 = _mm_shuffle_ps(t1, t3, 0x88);
2547 u3 = _mm_shuffle_ps(t1, t3, 0xdd);
2548
2549 if (add_into)
2550 {
2551 u0 = _mm_add_ps(_mm_loadu_ps(out[0] + 4 * i), u0);
2552 _mm_storeu_ps(out[0] + 4 * i, u0);
2553 u1 = _mm_add_ps(_mm_loadu_ps(out[1] + 4 * i), u1);
2554 _mm_storeu_ps(out[1] + 4 * i, u1);
2555 u2 = _mm_add_ps(_mm_loadu_ps(out[2] + 4 * i), u2);
2556 _mm_storeu_ps(out[2] + 4 * i, u2);
2557 u3 = _mm_add_ps(_mm_loadu_ps(out[3] + 4 * i), u3);
2558 _mm_storeu_ps(out[3] + 4 * i, u3);
2559 }
2560 else
2561 {
2562 _mm_storeu_ps(out[0] + 4 * i, u0);
2563 _mm_storeu_ps(out[1] + 4 * i, u1);
2564 _mm_storeu_ps(out[2] + 4 * i, u2);
2565 _mm_storeu_ps(out[3] + 4 * i, u3);
2566 }
2567 }
2568
2569 if (add_into)
2570 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
2571 for (unsigned int v = 0; v < 4; ++v)
2572 out[v][i] += in[i][v];
2573 else
2574 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
2575 for (unsigned int v = 0; v < 4; ++v)
2576 out[v][i] = in[i][v];
2577 }
2578
2579
2580
2581 # endif // if DEAL_II_VECTORIZATION_WIDTH_IN_BITS > 0 && defined(__SSE2__)
2582
2583 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 256 && defined(__AVX__)
2584
2585 /**
2586 * Specialization of VectorizedArray class for double and AVX.
2587 */
2588 template <>
2589 class VectorizedArray<double, 4>
2590 : public VectorizedArrayBase<VectorizedArray<double, 4>, 4>
2591 {
2592 public:
2593 /**
2594 * This gives the type of the array elements.
2595 */
2596 using value_type = double;
2597
2598 /**
2599 * Record the fact that the given specialization of VectorizedArray is
2600 * indeed implemented.
2601 */
2602 static constexpr bool is_implemented = true;
2603
2604 /**
2605 * Default empty constructor, leaving the data in an uninitialized state
2606 * similar to float/double.
2607 */
2608 VectorizedArray() = default;
2609
2610 /**
2611 * Construct an array with the given scalar broadcast to all lanes.
2612 */
2613 VectorizedArray(const double scalar)
2614 {
2615 this->operator=(scalar);
2616 }
2617
2618 /**
2619 * Construct an array with the given initializer list.
2620 */
2621 template <typename U>
2622 VectorizedArray(const std::initializer_list<U> &list)
2623 : VectorizedArrayBase<VectorizedArray<double, 4>, 4>(list)
2624 {}
2625
2626 /**
2627 * This function can be used to set all data fields to a given scalar.
2628 */
2629 DEAL_II_ALWAYS_INLINE
2630 VectorizedArray &
2631 operator=(const double x) &
2632 {
2633 data = _mm256_set1_pd(x);
2634 return *this;
2635 }
2636
2637 /**
2638 * Assign a scalar to the current object. This overload is used for
2639 * rvalue references; because it does not make sense to assign
2640 * something to a temporary, the function is deleted.
2641 */
2642 VectorizedArray &
2643 operator=(const double scalar) && = delete;
2644
2645 /**
2646 * Access operator.
2647 */
2648 DEAL_II_ALWAYS_INLINE
2649 double &
2650 operator[](const unsigned int comp)
2651 {
2652 AssertIndexRange(comp, 4);
2653 return *(reinterpret_cast<double *>(&data) + comp);
2654 }
2655
2656 /**
2657 * Constant access operator.
2658 */
2659 DEAL_II_ALWAYS_INLINE
2660 const double &
2661 operator[](const unsigned int comp) const
2662 {
2663 AssertIndexRange(comp, 4);
2664 return *(reinterpret_cast<const double *>(&data) + comp);
2665 }
2666
2667 /**
2668 * Element-wise addition of two arrays of numbers.
2669 */
2670 DEAL_II_ALWAYS_INLINE
2671 VectorizedArray &
2672 operator+=(const VectorizedArray &vec)
2673 {
2674 // if the compiler supports vector arithmetic, we can simply use +=
2675 // operator on the given data type. this allows the compiler to combine
2676 // additions with multiplication (fused multiply-add) if those
2677 // instructions are available. Otherwise, we need to use the built-in
2678 // intrinsic command for __m256d
2679 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
2680 data += vec.data;
2681 # else
2682 data = _mm256_add_pd(data, vec.data);
2683 # endif
2684 return *this;
2685 }
2686
2687 /**
2688 * Element-wise subtraction of two arrays of numbers.
2689 */
2690 DEAL_II_ALWAYS_INLINE
2691 VectorizedArray &
2692 operator-=(const VectorizedArray &vec)
2693 {
2694 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
2695 data -= vec.data;
2696 # else
2697 data = _mm256_sub_pd(data, vec.data);
2698 # endif
2699 return *this;
2700 }
2701 /**
2702 * Element-wise multiplication of two arrays of numbers.
2703 */
2704 DEAL_II_ALWAYS_INLINE
2705 VectorizedArray &
2706 operator*=(const VectorizedArray &vec)
2707 {
2708 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
2709 data *= vec.data;
2710 # else
2711 data = _mm256_mul_pd(data, vec.data);
2712 # endif
2713 return *this;
2714 }
2715
2716 /**
2717 * Element-wise division of two arrays of numbers.
2718 */
2719 DEAL_II_ALWAYS_INLINE
2720 VectorizedArray &
2721 operator/=(const VectorizedArray &vec)
2722 {
2723 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
2724 data /= vec.data;
2725 # else
2726 data = _mm256_div_pd(data, vec.data);
2727 # endif
2728 return *this;
2729 }
2730
2731 /**
2732 * Load @p size() from memory into the calling class, starting at
2733 * the given address. The memory need not be aligned by 32 bytes, as opposed
2734 * to casting a double address to VectorizedArray<double>*.
2735 */
2736 DEAL_II_ALWAYS_INLINE
2737 void
2738 load(const double *ptr)
2739 {
2740 data = _mm256_loadu_pd(ptr);
2741 }
2742
2743 DEAL_II_ALWAYS_INLINE
2744 void
2745 load(const float *ptr)
2746 {
2747 data = _mm256_cvtps_pd(_mm_loadu_ps(ptr));
2748 }
2749
2750 /**
2751 * Write the content of the calling class into memory in form of @p
2752 * size() to the given address. The memory need not be aligned by
2753 * 32 bytes, as opposed to casting a double address to
2754 * VectorizedArray<double>*.
2755 */
2756 DEAL_II_ALWAYS_INLINE
2757 void
2758 store(double *ptr) const
2759 {
2760 _mm256_storeu_pd(ptr, data);
2761 }
2762
2763 DEAL_II_ALWAYS_INLINE
2764 void
2765 store(float *ptr) const
2766 {
2767 _mm_storeu_ps(ptr, _mm256_cvtpd_ps(data));
2768 }
2769
2770 /**
2771 * @copydoc VectorizedArray<Number>::streaming_store()
2772 * @note Memory must be aligned by 32 bytes.
2773 */
2774 DEAL_II_ALWAYS_INLINE
2775 void
2776 streaming_store(double *ptr) const
2777 {
2778 Assert(reinterpret_cast<std::size_t>(ptr) % 32 == 0,
2779 ExcMessage("Memory not aligned"));
2780 _mm256_stream_pd(ptr, data);
2781 }
2782
2783 /**
2784 * Load @p size() from memory into the calling class, starting at
2785 * the given address and with given offsets, each entry from the offset
2786 * providing one element of the vectorized array.
2787 *
2788 * This operation corresponds to the following code (but uses a more
2789 * efficient implementation in case the hardware allows for that):
2790 * @code
2791 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
2792 * this->operator[](v) = base_ptr[offsets[v]];
2793 * @endcode
2794 */
2795 DEAL_II_ALWAYS_INLINE
2796 void
2797 gather(const double *base_ptr, const unsigned int *offsets)
2798 {
2799 # if defined(__AVX2__) && defined(DEAL_II_USE_VECTORIZATION_GATHER)
2800 // unfortunately, there does not appear to be a 128 bit integer load, so
2801 // do it by some reinterpret casts here. this is allowed because the Intel
2802 // API allows aliasing between different vector types.
2803 const __m128 index_val =
2804 _mm_loadu_ps(reinterpret_cast<const float *>(offsets));
2805 const __m128i index = *reinterpret_cast<const __m128i *>(&index_val);
2806
2807 // work around a warning with gcc-12 about an uninitialized initial state
2808 // for gather by starting with a zero guess, even though all lanes will be
2809 // overwritten
2810 __m256d zero = _mm256_setzero_pd();
2811 __m256d mask = _mm256_cmp_pd(zero, zero, _CMP_EQ_OQ);
2812
2813 data = _mm256_mask_i32gather_pd(zero, base_ptr, index, mask, 8);
2814 # else
2815 for (unsigned int i = 0; i < 4; ++i)
2816 *(reinterpret_cast<double *>(&data) + i) = base_ptr[offsets[i]];
2817 # endif
2818 }
2819
2820 /**
2821 * Write the content of the calling class into memory in form of @p
2822 * size() to the given address and the given offsets, filling the
2823 * elements of the vectorized array into each offset.
2824 *
2825 * This operation corresponds to the following code (but uses a more
2826 * efficient implementation in case the hardware allows for that):
2827 * @code
2828 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
2829 * base_ptr[offsets[v]] = this->operator[](v);
2830 * @endcode
2831 */
2832 DEAL_II_ALWAYS_INLINE
2833 void
2834 scatter(const unsigned int *offsets, double *base_ptr) const
2835 {
2836 // no scatter operation in AVX/AVX2
2837 for (unsigned int i = 0; i < 4; ++i)
2838 base_ptr[offsets[i]] = *(reinterpret_cast<const double *>(&data) + i);
2839 }
2840
2841 /**
2842 * Returns sum over entries of the data field, $\sum_{i=1}^{\text{size}()}
2843 * this->data[i]$.
2844 */
2845 double
2846 sum() const
2847 {
2848 VectorizedArray<double, 2> t1;
2849 t1.data = _mm_add_pd(this->get_lower(), this->get_upper());
2850 return t1.sum();
2851 }
2852
2853 /**
2854 * Actual data field. To be consistent with the standard layout type and to
2855 * enable interaction with external SIMD functionality, this member is
2856 * declared public.
2857 */
2858 __m256d data;
2859
2860 private:
2861 /**
2862 * Extract lower half of data field.
2863 */
2864 DEAL_II_ALWAYS_INLINE
2865 __m128d
2866 get_lower() const
2867 {
2868 return _mm256_castpd256_pd128(data);
2869 }
2870
2871 /**
2872 * Extract upper half of data field.
2873 */
2874 DEAL_II_ALWAYS_INLINE
2875 __m128d
2876 get_upper() const
2877 {
2878 return _mm256_extractf128_pd(data, 1);
2879 }
2880
2881 /**
2882 * Return the square root of this field. Not for use in user code. Use
2883 * sqrt(x) instead.
2884 */
2885 DEAL_II_ALWAYS_INLINE
2886 VectorizedArray
2887 get_sqrt() const
2888 {
2889 VectorizedArray res;
2890 res.data = _mm256_sqrt_pd(data);
2891 return res;
2892 }
2893
2894 /**
2895 * Return the absolute value of this field. Not for use in user code. Use
2896 * abs(x) instead.
2897 */
2898 DEAL_II_ALWAYS_INLINE
2899 VectorizedArray
2900 get_abs() const
2901 {
2902 // to compute the absolute value, perform bitwise andnot with -0. This
2903 // will leave all value and exponent bits unchanged but force the sign
2904 // value to +.
2905 __m256d mask = _mm256_set1_pd(-0.);
2906 VectorizedArray res;
2907 res.data = _mm256_andnot_pd(mask, data);
2908 return res;
2909 }
2910
2911 /**
2912 * Return the component-wise maximum of this field and another one. Not for
2913 * use in user code. Use max(x,y) instead.
2914 */
2915 DEAL_II_ALWAYS_INLINE
2916 VectorizedArray
2917 get_max(const VectorizedArray &other) const
2918 {
2919 VectorizedArray res;
2920 res.data = _mm256_max_pd(data, other.data);
2921 return res;
2922 }
2923
2924 /**
2925 * Return the component-wise minimum of this field and another one. Not for
2926 * use in user code. Use min(x,y) instead.
2927 */
2928 DEAL_II_ALWAYS_INLINE
2929 VectorizedArray
2930 get_min(const VectorizedArray &other) const
2931 {
2932 VectorizedArray res;
2933 res.data = _mm256_min_pd(data, other.data);
2934 return res;
2935 }
2936
2937 // Make a few functions friends.
2938 template <typename Number2, std::size_t width2>
2939 friend VectorizedArray<Number2, width2>
2940 std::sqrt(const VectorizedArray<Number2, width2> &);
2941 template <typename Number2, std::size_t width2>
2942 friend VectorizedArray<Number2, width2>
2943 std::abs(const VectorizedArray<Number2, width2> &);
2944 template <typename Number2, std::size_t width2>
2945 friend VectorizedArray<Number2, width2>
2946 std::max(const VectorizedArray<Number2, width2> &,
2947 const VectorizedArray<Number2, width2> &);
2948 template <typename Number2, std::size_t width2>
2949 friend VectorizedArray<Number2, width2>
2950 std::min(const VectorizedArray<Number2, width2> &,
2951 const VectorizedArray<Number2, width2> &);
2952 };
2953
2954
2955
2956 /**
2957 * Specialization for double and AVX.
2958 */
2959 template <>
2960 inline DEAL_II_ALWAYS_INLINE void
2961 vectorized_load_and_transpose(const unsigned int n_entries,
2962 const double *in,
2963 const unsigned int *offsets,
2964 VectorizedArray<double, 4> *out)
2965 {
2966 const unsigned int n_chunks = n_entries / 4;
2967 const double *in0 = in + offsets[0];
2968 const double *in1 = in + offsets[1];
2969 const double *in2 = in + offsets[2];
2970 const double *in3 = in + offsets[3];
2971
2972 for (unsigned int i = 0; i < n_chunks; ++i)
2973 {
2974 __m256d u0 = _mm256_loadu_pd(in0 + 4 * i);
2975 __m256d u1 = _mm256_loadu_pd(in1 + 4 * i);
2976 __m256d u2 = _mm256_loadu_pd(in2 + 4 * i);
2977 __m256d u3 = _mm256_loadu_pd(in3 + 4 * i);
2978 __m256d t0 = _mm256_permute2f128_pd(u0, u2, 0x20);
2979 __m256d t1 = _mm256_permute2f128_pd(u1, u3, 0x20);
2980 __m256d t2 = _mm256_permute2f128_pd(u0, u2, 0x31);
2981 __m256d t3 = _mm256_permute2f128_pd(u1, u3, 0x31);
2982 out[4 * i + 0].data = _mm256_unpacklo_pd(t0, t1);
2983 out[4 * i + 1].data = _mm256_unpackhi_pd(t0, t1);
2984 out[4 * i + 2].data = _mm256_unpacklo_pd(t2, t3);
2985 out[4 * i + 3].data = _mm256_unpackhi_pd(t2, t3);
2986 }
2987
2988 // remainder loop of work that does not divide by 4
2989 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
2990 out[i].gather(in + i, offsets);
2991 }
2992
2993
2994
2995 /**
2996 * Specialization for double and AVX.
2997 */
2998 template <>
2999 inline DEAL_II_ALWAYS_INLINE void
3000 vectorized_load_and_transpose(const unsigned int n_entries,
3001 const std::array<double *, 4> &in,
3002 VectorizedArray<double, 4> *out)
3003 {
3004 // see the comments in the vectorized_load_and_transpose above
3005
3006 const unsigned int n_chunks = n_entries / 4;
3007 const double *in0 = in[0];
3008 const double *in1 = in[1];
3009 const double *in2 = in[2];
3010 const double *in3 = in[3];
3011
3012 for (unsigned int i = 0; i < n_chunks; ++i)
3013 {
3014 __m256d u0 = _mm256_loadu_pd(in0 + 4 * i);
3015 __m256d u1 = _mm256_loadu_pd(in1 + 4 * i);
3016 __m256d u2 = _mm256_loadu_pd(in2 + 4 * i);
3017 __m256d u3 = _mm256_loadu_pd(in3 + 4 * i);
3018 __m256d t0 = _mm256_permute2f128_pd(u0, u2, 0x20);
3019 __m256d t1 = _mm256_permute2f128_pd(u1, u3, 0x20);
3020 __m256d t2 = _mm256_permute2f128_pd(u0, u2, 0x31);
3021 __m256d t3 = _mm256_permute2f128_pd(u1, u3, 0x31);
3022 out[4 * i + 0].data = _mm256_unpacklo_pd(t0, t1);
3023 out[4 * i + 1].data = _mm256_unpackhi_pd(t0, t1);
3024 out[4 * i + 2].data = _mm256_unpacklo_pd(t2, t3);
3025 out[4 * i + 3].data = _mm256_unpackhi_pd(t2, t3);
3026 }
3027
3028 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3029 gather(out[i], in, i);
3030 }
3031
3032
3033
3034 /**
3035 * Specialization for double and AVX.
3036 */
3037 template <>
3038 inline DEAL_II_ALWAYS_INLINE void
3039 vectorized_transpose_and_store(const bool add_into,
3040 const unsigned int n_entries,
3041 const VectorizedArray<double, 4> *in,
3042 const unsigned int *offsets,
3043 double *out)
3044 {
3045 const unsigned int n_chunks = n_entries / 4;
3046 double *out0 = out + offsets[0];
3047 double *out1 = out + offsets[1];
3048 double *out2 = out + offsets[2];
3049 double *out3 = out + offsets[3];
3050 for (unsigned int i = 0; i < n_chunks; ++i)
3051 {
3052 __m256d u0 = in[4 * i + 0].data;
3053 __m256d u1 = in[4 * i + 1].data;
3054 __m256d u2 = in[4 * i + 2].data;
3055 __m256d u3 = in[4 * i + 3].data;
3056 __m256d t0 = _mm256_permute2f128_pd(u0, u2, 0x20);
3057 __m256d t1 = _mm256_permute2f128_pd(u1, u3, 0x20);
3058 __m256d t2 = _mm256_permute2f128_pd(u0, u2, 0x31);
3059 __m256d t3 = _mm256_permute2f128_pd(u1, u3, 0x31);
3060 __m256d res0 = _mm256_unpacklo_pd(t0, t1);
3061 __m256d res1 = _mm256_unpackhi_pd(t0, t1);
3062 __m256d res2 = _mm256_unpacklo_pd(t2, t3);
3063 __m256d res3 = _mm256_unpackhi_pd(t2, t3);
3064
3065 // Cannot use the same store instructions in both paths of the 'if'
3066 // because the compiler cannot know that there is no aliasing between
3067 // pointers
3068 if (add_into)
3069 {
3070 res0 = _mm256_add_pd(_mm256_loadu_pd(out0 + 4 * i), res0);
3071 _mm256_storeu_pd(out0 + 4 * i, res0);
3072 res1 = _mm256_add_pd(_mm256_loadu_pd(out1 + 4 * i), res1);
3073 _mm256_storeu_pd(out1 + 4 * i, res1);
3074 res2 = _mm256_add_pd(_mm256_loadu_pd(out2 + 4 * i), res2);
3075 _mm256_storeu_pd(out2 + 4 * i, res2);
3076 res3 = _mm256_add_pd(_mm256_loadu_pd(out3 + 4 * i), res3);
3077 _mm256_storeu_pd(out3 + 4 * i, res3);
3078 }
3079 else
3080 {
3081 _mm256_storeu_pd(out0 + 4 * i, res0);
3082 _mm256_storeu_pd(out1 + 4 * i, res1);
3083 _mm256_storeu_pd(out2 + 4 * i, res2);
3084 _mm256_storeu_pd(out3 + 4 * i, res3);
3085 }
3086 }
3087
3088 // remainder loop of work that does not divide by 4
3089 if (add_into)
3090 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3091 for (unsigned int v = 0; v < 4; ++v)
3092 out[offsets[v] + i] += in[i][v];
3093 else
3094 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3095 for (unsigned int v = 0; v < 4; ++v)
3096 out[offsets[v] + i] = in[i][v];
3097 }
3098
3099
3100
3101 /**
3102 * Specialization for double and AVX.
3103 */
3104 template <>
3105 inline DEAL_II_ALWAYS_INLINE void
3106 vectorized_transpose_and_store(const bool add_into,
3107 const unsigned int n_entries,
3108 const VectorizedArray<double, 4> *in,
3109 std::array<double *, 4> &out)
3110 {
3111 // see the comments in the vectorized_transpose_and_store above
3112
3113 const unsigned int n_chunks = n_entries / 4;
3114 double *out0 = out[0];
3115 double *out1 = out[1];
3116 double *out2 = out[2];
3117 double *out3 = out[3];
3118 for (unsigned int i = 0; i < n_chunks; ++i)
3119 {
3120 __m256d u0 = in[4 * i + 0].data;
3121 __m256d u1 = in[4 * i + 1].data;
3122 __m256d u2 = in[4 * i + 2].data;
3123 __m256d u3 = in[4 * i + 3].data;
3124 __m256d t0 = _mm256_permute2f128_pd(u0, u2, 0x20);
3125 __m256d t1 = _mm256_permute2f128_pd(u1, u3, 0x20);
3126 __m256d t2 = _mm256_permute2f128_pd(u0, u2, 0x31);
3127 __m256d t3 = _mm256_permute2f128_pd(u1, u3, 0x31);
3128 __m256d res0 = _mm256_unpacklo_pd(t0, t1);
3129 __m256d res1 = _mm256_unpackhi_pd(t0, t1);
3130 __m256d res2 = _mm256_unpacklo_pd(t2, t3);
3131 __m256d res3 = _mm256_unpackhi_pd(t2, t3);
3132
3133 // Cannot use the same store instructions in both paths of the 'if'
3134 // because the compiler cannot know that there is no aliasing between
3135 // pointers
3136 if (add_into)
3137 {
3138 res0 = _mm256_add_pd(_mm256_loadu_pd(out0 + 4 * i), res0);
3139 _mm256_storeu_pd(out0 + 4 * i, res0);
3140 res1 = _mm256_add_pd(_mm256_loadu_pd(out1 + 4 * i), res1);
3141 _mm256_storeu_pd(out1 + 4 * i, res1);
3142 res2 = _mm256_add_pd(_mm256_loadu_pd(out2 + 4 * i), res2);
3143 _mm256_storeu_pd(out2 + 4 * i, res2);
3144 res3 = _mm256_add_pd(_mm256_loadu_pd(out3 + 4 * i), res3);
3145 _mm256_storeu_pd(out3 + 4 * i, res3);
3146 }
3147 else
3148 {
3149 _mm256_storeu_pd(out0 + 4 * i, res0);
3150 _mm256_storeu_pd(out1 + 4 * i, res1);
3151 _mm256_storeu_pd(out2 + 4 * i, res2);
3152 _mm256_storeu_pd(out3 + 4 * i, res3);
3153 }
3154 }
3155
3156 // remainder loop of work that does not divide by 4
3157 if (add_into)
3158 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3159 for (unsigned int v = 0; v < 4; ++v)
3160 out[v][i] += in[i][v];
3161 else
3162 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3163 for (unsigned int v = 0; v < 4; ++v)
3164 out[v][i] = in[i][v];
3165 }
3166
3167
3168
3169 /**
3170 * Specialization for float and AVX.
3171 */
3172 template <>
3173 class VectorizedArray<float, 8>
3174 : public VectorizedArrayBase<VectorizedArray<float, 8>, 8>
3175 {
3176 public:
3177 /**
3178 * This gives the type of the array elements.
3179 */
3180 using value_type = float;
3181
3182 /**
3183 * Record the fact that the given specialization of VectorizedArray is
3184 * indeed implemented.
3185 */
3186 static constexpr bool is_implemented = true;
3187
3188 /**
3189 * Default empty constructor, leaving the data in an uninitialized state
3190 * similar to float/double.
3191 */
3192 VectorizedArray() = default;
3193
3194 /**
3195 * Construct an array with the given scalar broadcast to all lanes.
3196 */
3197 VectorizedArray(const float scalar)
3198 {
3199 this->operator=(scalar);
3200 }
3201
3202 /**
3203 * Construct an array with the given initializer list.
3204 */
3205 template <typename U>
3206 VectorizedArray(const std::initializer_list<U> &list)
3207 : VectorizedArrayBase<VectorizedArray<float, 8>, 8>(list)
3208 {}
3209
3210 /**
3211 * This function can be used to set all data fields to a given scalar.
3212 */
3213 DEAL_II_ALWAYS_INLINE
3214 VectorizedArray &
3215 operator=(const float x) &
3216 {
3217 data = _mm256_set1_ps(x);
3218 return *this;
3219 }
3220
3221 /**
3222 * Assign a scalar to the current object. This overload is used for
3223 * rvalue references; because it does not make sense to assign
3224 * something to a temporary, the function is deleted.
3225 */
3226 VectorizedArray &
3227 operator=(const float scalar) && = delete;
3228
3229 /**
3230 * Access operator.
3231 */
3232 DEAL_II_ALWAYS_INLINE
3233 float &
3234 operator[](const unsigned int comp)
3235 {
3236 AssertIndexRange(comp, 8);
3237 return *(reinterpret_cast<float *>(&data) + comp);
3238 }
3239
3240 /**
3241 * Constant access operator.
3242 */
3243 DEAL_II_ALWAYS_INLINE
3244 const float &
3245 operator[](const unsigned int comp) const
3246 {
3247 AssertIndexRange(comp, 8);
3248 return *(reinterpret_cast<const float *>(&data) + comp);
3249 }
3250
3251 /**
3252 * Element-wise addition of two arrays of numbers.
3253 */
3254 DEAL_II_ALWAYS_INLINE
3255 VectorizedArray &
3256 operator+=(const VectorizedArray &vec)
3257 {
3258 // if the compiler supports vector arithmetic, we can simply use +=
3259 // operator on the given data type. this allows the compiler to combine
3260 // additions with multiplication (fused multiply-add) if those
3261 // instructions are available. Otherwise, we need to use the built-in
3262 // intrinsic command for __m256d
3263 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
3264 data += vec.data;
3265 # else
3266 data = _mm256_add_ps(data, vec.data);
3267 # endif
3268 return *this;
3269 }
3270
3271 /**
3272 * Element-wise subtraction of two arrays of numbers.
3273 */
3274 DEAL_II_ALWAYS_INLINE
3275 VectorizedArray &
3276 operator-=(const VectorizedArray &vec)
3277 {
3278 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
3279 data -= vec.data;
3280 # else
3281 data = _mm256_sub_ps(data, vec.data);
3282 # endif
3283 return *this;
3284 }
3285 /**
3286 * Element-wise multiplication of two arrays of numbers.
3287 */
3288 DEAL_II_ALWAYS_INLINE
3289 VectorizedArray &
3290 operator*=(const VectorizedArray &vec)
3291 {
3292 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
3293 data *= vec.data;
3294 # else
3295 data = _mm256_mul_ps(data, vec.data);
3296 # endif
3297 return *this;
3298 }
3299
3300 /**
3301 * Element-wise division of two arrays of numbers.
3302 */
3303 DEAL_II_ALWAYS_INLINE
3304 VectorizedArray &
3305 operator/=(const VectorizedArray &vec)
3306 {
3307 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
3308 data /= vec.data;
3309 # else
3310 data = _mm256_div_ps(data, vec.data);
3311 # endif
3312 return *this;
3313 }
3314
3315 /**
3316 * Load @p size() from memory into the calling class, starting at
3317 * the given address. The memory need not be aligned by 32 bytes, as opposed
3318 * to casting a float address to VectorizedArray<float>*.
3319 */
3320 DEAL_II_ALWAYS_INLINE
3321 void
3322 load(const float *ptr)
3323 {
3324 data = _mm256_loadu_ps(ptr);
3325 }
3326
3327 /**
3328 * Write the content of the calling class into memory in form of @p
3329 * size() to the given address. The memory need not be aligned by
3330 * 32 bytes, as opposed to casting a float address to
3331 * VectorizedArray<float>*.
3332 */
3333 DEAL_II_ALWAYS_INLINE
3334 void
3335 store(float *ptr) const
3336 {
3337 _mm256_storeu_ps(ptr, data);
3338 }
3339
3340 /**
3341 * @copydoc VectorizedArray<Number>::streaming_store()
3342 * @note Memory must be aligned by 32 bytes.
3343 */
3344 DEAL_II_ALWAYS_INLINE
3345 void
3346 streaming_store(float *ptr) const
3347 {
3348 Assert(reinterpret_cast<std::size_t>(ptr) % 32 == 0,
3349 ExcMessage("Memory not aligned"));
3350 _mm256_stream_ps(ptr, data);
3351 }
3352
3353 /**
3354 * Load @p size() from memory into the calling class, starting at
3355 * the given address and with given offsets, each entry from the offset
3356 * providing one element of the vectorized array.
3357 *
3358 * This operation corresponds to the following code (but uses a more
3359 * efficient implementation in case the hardware allows for that):
3360 * @code
3361 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
3362 * this->operator[](v) = base_ptr[offsets[v]];
3363 * @endcode
3364 */
3365 DEAL_II_ALWAYS_INLINE
3366 void
3367 gather(const float *base_ptr, const unsigned int *offsets)
3368 {
3369 # if defined(__AVX2__) && defined(DEAL_II_USE_VECTORIZATION_GATHER)
3370 // unfortunately, there does not appear to be a 256 bit integer load, so
3371 // do it by some reinterpret casts here. this is allowed because the Intel
3372 // API allows aliasing between different vector types.
3373 const __m256 index_val =
3374 _mm256_loadu_ps(reinterpret_cast<const float *>(offsets));
3375 const __m256i index = *reinterpret_cast<const __m256i *>(&index_val);
3376
3377 // work around a warning with gcc-12 about an uninitialized initial state
3378 // for gather by starting with a zero guess, even though all lanes will be
3379 // overwritten
3380 __m256 zero = _mm256_setzero_ps();
3381 __m256 mask = _mm256_cmp_ps(zero, zero, _CMP_EQ_OQ);
3382
3383 data = _mm256_mask_i32gather_ps(zero, base_ptr, index, mask, 4);
3384 # else
3385 for (unsigned int i = 0; i < 8; ++i)
3386 *(reinterpret_cast<float *>(&data) + i) = base_ptr[offsets[i]];
3387 # endif
3388 }
3389
3390 /**
3391 * Write the content of the calling class into memory in form of @p
3392 * size() to the given address and the given offsets, filling the
3393 * elements of the vectorized array into each offset.
3394 *
3395 * This operation corresponds to the following code (but uses a more
3396 * efficient implementation in case the hardware allows for that):
3397 * @code
3398 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
3399 * base_ptr[offsets[v]] = this->operator[](v);
3400 * @endcode
3401 */
3402 DEAL_II_ALWAYS_INLINE
3403 void
3404 scatter(const unsigned int *offsets, float *base_ptr) const
3405 {
3406 // no scatter operation in AVX/AVX2
3407 for (unsigned int i = 0; i < 8; ++i)
3408 base_ptr[offsets[i]] = *(reinterpret_cast<const float *>(&data) + i);
3409 }
3410
3411 /**
3412 * Returns sum over entries of the data field, $\sum_{i=1}^{\text{size}()}
3413 * this->data[i]$.
3414 */
3415 float
3416 sum() const
3417 {
3418 VectorizedArray<float, 4> t1;
3419 t1.data = _mm_add_ps(this->get_lower(), this->get_upper());
3420 return t1.sum();
3421 }
3422
3423 /**
3424 * Actual data field. To be consistent with the standard layout type and to
3425 * enable interaction with external SIMD functionality, this member is
3426 * declared public.
3427 */
3428 __m256 data;
3429
3430 private:
3431 /**
3432 * Extract lower half of data field.
3433 */
3434 DEAL_II_ALWAYS_INLINE
3435 __m128
3436 get_lower() const
3437 {
3438 return _mm256_castps256_ps128(data);
3439 }
3440
3441 /**
3442 * Extract upper half of data field.
3443 */
3444 DEAL_II_ALWAYS_INLINE
3445 __m128
3446 get_upper() const
3447 {
3448 return _mm256_extractf128_ps(data, 1);
3449 }
3450
3451 /**
3452 * Return the square root of this field. Not for use in user code. Use
3453 * sqrt(x) instead.
3454 */
3455 DEAL_II_ALWAYS_INLINE
3456 VectorizedArray
3457 get_sqrt() const
3458 {
3459 VectorizedArray res;
3460 res.data = _mm256_sqrt_ps(data);
3461 return res;
3462 }
3463
3464 /**
3465 * Return the absolute value of this field. Not for use in user code. Use
3466 * abs(x) instead.
3467 */
3468 DEAL_II_ALWAYS_INLINE
3469 VectorizedArray
3470 get_abs() const
3471 {
3472 // to compute the absolute value, perform bitwise andnot with -0. This
3473 // will leave all value and exponent bits unchanged but force the sign
3474 // value to +.
3475 __m256 mask = _mm256_set1_ps(-0.f);
3476 VectorizedArray res;
3477 res.data = _mm256_andnot_ps(mask, data);
3478 return res;
3479 }
3480
3481 /**
3482 * Return the component-wise maximum of this field and another one. Not for
3483 * use in user code. Use max(x,y) instead.
3484 */
3485 DEAL_II_ALWAYS_INLINE
3486 VectorizedArray
3487 get_max(const VectorizedArray &other) const
3488 {
3489 VectorizedArray res;
3490 res.data = _mm256_max_ps(data, other.data);
3491 return res;
3492 }
3493
3494 /**
3495 * Return the component-wise minimum of this field and another one. Not for
3496 * use in user code. Use min(x,y) instead.
3497 */
3498 DEAL_II_ALWAYS_INLINE
3499 VectorizedArray
3500 get_min(const VectorizedArray &other) const
3501 {
3502 VectorizedArray res;
3503 res.data = _mm256_min_ps(data, other.data);
3504 return res;
3505 }
3506
3507 // Make a few functions friends.
3508 template <typename Number2, std::size_t width2>
3509 friend VectorizedArray<Number2, width2>
3510 std::sqrt(const VectorizedArray<Number2, width2> &);
3511 template <typename Number2, std::size_t width2>
3512 friend VectorizedArray<Number2, width2>
3513 std::abs(const VectorizedArray<Number2, width2> &);
3514 template <typename Number2, std::size_t width2>
3515 friend VectorizedArray<Number2, width2>
3516 std::max(const VectorizedArray<Number2, width2> &,
3517 const VectorizedArray<Number2, width2> &);
3518 template <typename Number2, std::size_t width2>
3519 friend VectorizedArray<Number2, width2>
3520 std::min(const VectorizedArray<Number2, width2> &,
3521 const VectorizedArray<Number2, width2> &);
3522 };
3523
3524
3525
3526 /**
3527 * Specialization for float and AVX.
3528 */
3529 template <>
3530 inline DEAL_II_ALWAYS_INLINE void
3531 vectorized_load_and_transpose(const unsigned int n_entries,
3532 const float *in,
3533 const unsigned int *offsets,
3534 VectorizedArray<float, 8> *out)
3535 {
3536 const unsigned int n_chunks = n_entries / 4;
3537 for (unsigned int i = 0; i < n_chunks; ++i)
3538 {
3539 // To avoid warnings about uninitialized variables, need to initialize
3540 // one variable with zero before using it.
3541 __m256 t0, t1, t2, t3 = {};
3542 t0 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in + 4 * i + offsets[0]), 0);
3543 t0 = _mm256_insertf128_ps(t0, _mm_loadu_ps(in + 4 * i + offsets[4]), 1);
3544 t1 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in + 4 * i + offsets[1]), 0);
3545 t1 = _mm256_insertf128_ps(t1, _mm_loadu_ps(in + 4 * i + offsets[5]), 1);
3546 t2 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in + 4 * i + offsets[2]), 0);
3547 t2 = _mm256_insertf128_ps(t2, _mm_loadu_ps(in + 4 * i + offsets[6]), 1);
3548 t3 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in + 4 * i + offsets[3]), 0);
3549 t3 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in + 4 * i + offsets[7]), 1);
3550
3551 __m256 v0 = _mm256_shuffle_ps(t0, t1, 0x44);
3552 __m256 v1 = _mm256_shuffle_ps(t0, t1, 0xee);
3553 __m256 v2 = _mm256_shuffle_ps(t2, t3, 0x44);
3554 __m256 v3 = _mm256_shuffle_ps(t2, t3, 0xee);
3555 out[4 * i + 0].data = _mm256_shuffle_ps(v0, v2, 0x88);
3556 out[4 * i + 1].data = _mm256_shuffle_ps(v0, v2, 0xdd);
3557 out[4 * i + 2].data = _mm256_shuffle_ps(v1, v3, 0x88);
3558 out[4 * i + 3].data = _mm256_shuffle_ps(v1, v3, 0xdd);
3559 }
3560
3561 // remainder loop of work that does not divide by 4
3562 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3563 out[i].gather(in + i, offsets);
3564 }
3565
3566
3567
3568 /**
3569 * Specialization for float and AVX.
3570 */
3571 template <>
3572 inline DEAL_II_ALWAYS_INLINE void
3573 vectorized_load_and_transpose(const unsigned int n_entries,
3574 const std::array<float *, 8> &in,
3575 VectorizedArray<float, 8> *out)
3576 {
3577 // see the comments in the vectorized_load_and_transpose above
3578
3579 const unsigned int n_chunks = n_entries / 4;
3580 for (unsigned int i = 0; i < n_chunks; ++i)
3581 {
3582 __m256 t0, t1, t2, t3 = {};
3583 t0 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in[0] + 4 * i), 0);
3584 t0 = _mm256_insertf128_ps(t0, _mm_loadu_ps(in[4] + 4 * i), 1);
3585 t1 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in[1] + 4 * i), 0);
3586 t1 = _mm256_insertf128_ps(t1, _mm_loadu_ps(in[5] + 4 * i), 1);
3587 t2 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in[2] + 4 * i), 0);
3588 t2 = _mm256_insertf128_ps(t2, _mm_loadu_ps(in[6] + 4 * i), 1);
3589 t3 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in[3] + 4 * i), 0);
3590 t3 = _mm256_insertf128_ps(t3, _mm_loadu_ps(in[7] + 4 * i), 1);
3591
3592 __m256 v0 = _mm256_shuffle_ps(t0, t1, 0x44);
3593 __m256 v1 = _mm256_shuffle_ps(t0, t1, 0xee);
3594 __m256 v2 = _mm256_shuffle_ps(t2, t3, 0x44);
3595 __m256 v3 = _mm256_shuffle_ps(t2, t3, 0xee);
3596 out[4 * i + 0].data = _mm256_shuffle_ps(v0, v2, 0x88);
3597 out[4 * i + 1].data = _mm256_shuffle_ps(v0, v2, 0xdd);
3598 out[4 * i + 2].data = _mm256_shuffle_ps(v1, v3, 0x88);
3599 out[4 * i + 3].data = _mm256_shuffle_ps(v1, v3, 0xdd);
3600 }
3601
3602 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3603 gather(out[i], in, i);
3604 }
3605
3606
3607
3608 /**
3609 * Specialization for float and AVX.
3610 */
3611 template <>
3612 inline DEAL_II_ALWAYS_INLINE void
3613 vectorized_transpose_and_store(const bool add_into,
3614 const unsigned int n_entries,
3615 const VectorizedArray<float, 8> *in,
3616 const unsigned int *offsets,
3617 float *out)
3618 {
3619 const unsigned int n_chunks = n_entries / 4;
3620 for (unsigned int i = 0; i < n_chunks; ++i)
3621 {
3622 __m256 u0 = in[4 * i + 0].data;
3623 __m256 u1 = in[4 * i + 1].data;
3624 __m256 u2 = in[4 * i + 2].data;
3625 __m256 u3 = in[4 * i + 3].data;
3626 __m256 t0 = _mm256_shuffle_ps(u0, u1, 0x44);
3627 __m256 t1 = _mm256_shuffle_ps(u0, u1, 0xee);
3628 __m256 t2 = _mm256_shuffle_ps(u2, u3, 0x44);
3629 __m256 t3 = _mm256_shuffle_ps(u2, u3, 0xee);
3630 u0 = _mm256_shuffle_ps(t0, t2, 0x88);
3631 u1 = _mm256_shuffle_ps(t0, t2, 0xdd);
3632 u2 = _mm256_shuffle_ps(t1, t3, 0x88);
3633 u3 = _mm256_shuffle_ps(t1, t3, 0xdd);
3634 __m128 res0 = _mm256_extractf128_ps(u0, 0);
3635 __m128 res4 = _mm256_extractf128_ps(u0, 1);
3636 __m128 res1 = _mm256_extractf128_ps(u1, 0);
3637 __m128 res5 = _mm256_extractf128_ps(u1, 1);
3638 __m128 res2 = _mm256_extractf128_ps(u2, 0);
3639 __m128 res6 = _mm256_extractf128_ps(u2, 1);
3640 __m128 res3 = _mm256_extractf128_ps(u3, 0);
3641 __m128 res7 = _mm256_extractf128_ps(u3, 1);
3642
3643 // Cannot use the same store instructions in both paths of the 'if'
3644 // because the compiler cannot know that there is no aliasing between
3645 // pointers
3646 if (add_into)
3647 {
3648 res0 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[0]), res0);
3649 _mm_storeu_ps(out + 4 * i + offsets[0], res0);
3650 res1 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[1]), res1);
3651 _mm_storeu_ps(out + 4 * i + offsets[1], res1);
3652 res2 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[2]), res2);
3653 _mm_storeu_ps(out + 4 * i + offsets[2], res2);
3654 res3 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[3]), res3);
3655 _mm_storeu_ps(out + 4 * i + offsets[3], res3);
3656 res4 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[4]), res4);
3657 _mm_storeu_ps(out + 4 * i + offsets[4], res4);
3658 res5 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[5]), res5);
3659 _mm_storeu_ps(out + 4 * i + offsets[5], res5);
3660 res6 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[6]), res6);
3661 _mm_storeu_ps(out + 4 * i + offsets[6], res6);
3662 res7 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[7]), res7);
3663 _mm_storeu_ps(out + 4 * i + offsets[7], res7);
3664 }
3665 else
3666 {
3667 _mm_storeu_ps(out + 4 * i + offsets[0], res0);
3668 _mm_storeu_ps(out + 4 * i + offsets[1], res1);
3669 _mm_storeu_ps(out + 4 * i + offsets[2], res2);
3670 _mm_storeu_ps(out + 4 * i + offsets[3], res3);
3671 _mm_storeu_ps(out + 4 * i + offsets[4], res4);
3672 _mm_storeu_ps(out + 4 * i + offsets[5], res5);
3673 _mm_storeu_ps(out + 4 * i + offsets[6], res6);
3674 _mm_storeu_ps(out + 4 * i + offsets[7], res7);
3675 }
3676 }
3677
3678 // remainder loop of work that does not divide by 4
3679 if (add_into)
3680 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3681 for (unsigned int v = 0; v < 8; ++v)
3682 out[offsets[v] + i] += in[i][v];
3683 else
3684 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3685 for (unsigned int v = 0; v < 8; ++v)
3686 out[offsets[v] + i] = in[i][v];
3687 }
3688
3689
3690
3691 /**
3692 * Specialization for float and AVX.
3693 */
3694 template <>
3695 inline DEAL_II_ALWAYS_INLINE void
3696 vectorized_transpose_and_store(const bool add_into,
3697 const unsigned int n_entries,
3698 const VectorizedArray<float, 8> *in,
3699 std::array<float *, 8> &out)
3700 {
3701 // see the comments in the vectorized_transpose_and_store above
3702
3703 const unsigned int n_chunks = n_entries / 4;
3704 for (unsigned int i = 0; i < n_chunks; ++i)
3705 {
3706 __m256 u0 = in[4 * i + 0].data;
3707 __m256 u1 = in[4 * i + 1].data;
3708 __m256 u2 = in[4 * i + 2].data;
3709 __m256 u3 = in[4 * i + 3].data;
3710 __m256 t0 = _mm256_shuffle_ps(u0, u1, 0x44);
3711 __m256 t1 = _mm256_shuffle_ps(u0, u1, 0xee);
3712 __m256 t2 = _mm256_shuffle_ps(u2, u3, 0x44);
3713 __m256 t3 = _mm256_shuffle_ps(u2, u3, 0xee);
3714 u0 = _mm256_shuffle_ps(t0, t2, 0x88);
3715 u1 = _mm256_shuffle_ps(t0, t2, 0xdd);
3716 u2 = _mm256_shuffle_ps(t1, t3, 0x88);
3717 u3 = _mm256_shuffle_ps(t1, t3, 0xdd);
3718 __m128 res0 = _mm256_extractf128_ps(u0, 0);
3719 __m128 res4 = _mm256_extractf128_ps(u0, 1);
3720 __m128 res1 = _mm256_extractf128_ps(u1, 0);
3721 __m128 res5 = _mm256_extractf128_ps(u1, 1);
3722 __m128 res2 = _mm256_extractf128_ps(u2, 0);
3723 __m128 res6 = _mm256_extractf128_ps(u2, 1);
3724 __m128 res3 = _mm256_extractf128_ps(u3, 0);
3725 __m128 res7 = _mm256_extractf128_ps(u3, 1);
3726
3727 if (add_into)
3728 {
3729 res0 = _mm_add_ps(_mm_loadu_ps(out[0] + 4 * i), res0);
3730 _mm_storeu_ps(out[0] + 4 * i, res0);
3731 res1 = _mm_add_ps(_mm_loadu_ps(out[1] + 4 * i), res1);
3732 _mm_storeu_ps(out[1] + 4 * i, res1);
3733 res2 = _mm_add_ps(_mm_loadu_ps(out[2] + 4 * i), res2);
3734 _mm_storeu_ps(out[2] + 4 * i, res2);
3735 res3 = _mm_add_ps(_mm_loadu_ps(out[3] + 4 * i), res3);
3736 _mm_storeu_ps(out[3] + 4 * i, res3);
3737 res4 = _mm_add_ps(_mm_loadu_ps(out[4] + 4 * i), res4);
3738 _mm_storeu_ps(out[4] + 4 * i, res4);
3739 res5 = _mm_add_ps(_mm_loadu_ps(out[5] + 4 * i), res5);
3740 _mm_storeu_ps(out[5] + 4 * i, res5);
3741 res6 = _mm_add_ps(_mm_loadu_ps(out[6] + 4 * i), res6);
3742 _mm_storeu_ps(out[6] + 4 * i, res6);
3743 res7 = _mm_add_ps(_mm_loadu_ps(out[7] + 4 * i), res7);
3744 _mm_storeu_ps(out[7] + 4 * i, res7);
3745 }
3746 else
3747 {
3748 _mm_storeu_ps(out[0] + 4 * i, res0);
3749 _mm_storeu_ps(out[1] + 4 * i, res1);
3750 _mm_storeu_ps(out[2] + 4 * i, res2);
3751 _mm_storeu_ps(out[3] + 4 * i, res3);
3752 _mm_storeu_ps(out[4] + 4 * i, res4);
3753 _mm_storeu_ps(out[5] + 4 * i, res5);
3754 _mm_storeu_ps(out[6] + 4 * i, res6);
3755 _mm_storeu_ps(out[7] + 4 * i, res7);
3756 }
3757 }
3758
3759 if (add_into)
3760 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3761 for (unsigned int v = 0; v < 8; ++v)
3762 out[v][i] += in[i][v];
3763 else
3764 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
3765 for (unsigned int v = 0; v < 8; ++v)
3766 out[v][i] = in[i][v];
3767 }
3768
3769 # endif
3770
3771 // for safety, also check that __AVX512F__ is defined in case the user manually
3772 // set some conflicting compile flags which prevent compilation
3773
3774 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 512 && defined(__AVX512F__)
3775
3776 /**
3777 * Specialization of VectorizedArray class for double and AVX-512.
3778 */
3779 template <>
3780 class VectorizedArray<double, 8>
3781 : public VectorizedArrayBase<VectorizedArray<double, 8>, 8>
3782 {
3783 public:
3784 /**
3785 * This gives the type of the array elements.
3786 */
3787 using value_type = double;
3788
3789 /**
3790 * Record the fact that the given specialization of VectorizedArray is
3791 * indeed implemented.
3792 */
3793 static constexpr bool is_implemented = true;
3794
3795 /**
3796 * Default empty constructor, leaving the data in an uninitialized state
3797 * similar to float/double.
3798 */
3799 VectorizedArray() = default;
3800
3801 /**
3802 * Construct an array with the given scalar broadcast to all lanes.
3803 */
3804 VectorizedArray(const double scalar)
3805 {
3806 this->operator=(scalar);
3807 }
3808
3809 /**
3810 * Construct an array with the given initializer list.
3811 */
3812 template <typename U>
3813 VectorizedArray(const std::initializer_list<U> &list)
3814 : VectorizedArrayBase<VectorizedArray<double, 8>, 8>(list)
3815 {}
3816
3817 /**
3818 * This function can be used to set all data fields to a given scalar.
3819 */
3820 DEAL_II_ALWAYS_INLINE
3821 VectorizedArray &
3822 operator=(const double x) &
3823 {
3824 data = _mm512_set1_pd(x);
3825 return *this;
3826 }
3827
3828
3829 /**
3830 * Assign a scalar to the current object. This overload is used for
3831 * rvalue references; because it does not make sense to assign
3832 * something to a temporary, the function is deleted.
3833 */
3834 VectorizedArray &
3835 operator=(const double scalar) && = delete;
3836
3837 /**
3838 * Access operator.
3839 */
3840 DEAL_II_ALWAYS_INLINE
3841 double &
3842 operator[](const unsigned int comp)
3843 {
3844 AssertIndexRange(comp, 8);
3845 return *(reinterpret_cast<double *>(&data) + comp);
3846 }
3847
3848 /**
3849 * Constant access operator.
3850 */
3851 DEAL_II_ALWAYS_INLINE
3852 const double &
3853 operator[](const unsigned int comp) const
3854 {
3855 AssertIndexRange(comp, 8);
3856 return *(reinterpret_cast<const double *>(&data) + comp);
3857 }
3858
3859 /**
3860 * Element-wise addition of two arrays of numbers.
3861 */
3862 DEAL_II_ALWAYS_INLINE
3863 VectorizedArray &
3864 operator+=(const VectorizedArray &vec)
3865 {
3866 // if the compiler supports vector arithmetic, we can simply use +=
3867 // operator on the given data type. this allows the compiler to combine
3868 // additions with multiplication (fused multiply-add) if those
3869 // instructions are available. Otherwise, we need to use the built-in
3870 // intrinsic command for __m512d
3871 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
3872 data += vec.data;
3873 # else
3874 data = _mm512_add_pd(data, vec.data);
3875 # endif
3876 return *this;
3877 }
3878
3879 /**
3880 * Element-wise subtraction of two arrays of numbers.
3881 */
3882 DEAL_II_ALWAYS_INLINE
3883 VectorizedArray &
3884 operator-=(const VectorizedArray &vec)
3885 {
3886 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
3887 data -= vec.data;
3888 # else
3889 data = _mm512_sub_pd(data, vec.data);
3890 # endif
3891 return *this;
3892 }
3893 /**
3894 * Element-wise multiplication of two arrays of numbers.
3895 */
3896 DEAL_II_ALWAYS_INLINE
3897 VectorizedArray &
3898 operator*=(const VectorizedArray &vec)
3899 {
3900 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
3901 data *= vec.data;
3902 # else
3903 data = _mm512_mul_pd(data, vec.data);
3904 # endif
3905 return *this;
3906 }
3907
3908 /**
3909 * Element-wise division of two arrays of numbers.
3910 */
3911 DEAL_II_ALWAYS_INLINE
3912 VectorizedArray &
3913 operator/=(const VectorizedArray &vec)
3914 {
3915 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
3916 data /= vec.data;
3917 # else
3918 data = _mm512_div_pd(data, vec.data);
3919 # endif
3920 return *this;
3921 }
3922
3923 /**
3924 * Load size() data items from memory into the calling class, starting at
3925 * the given address. The memory need not be aligned by 64 bytes, as opposed
3926 * to casting a double address to VectorizedArray<double>*.
3927 */
3928 DEAL_II_ALWAYS_INLINE
3929 void
3930 load(const double *ptr)
3931 {
3932 data = _mm512_loadu_pd(ptr);
3933 }
3934
3935 DEAL_II_ALWAYS_INLINE
3936 void
3937 load(const float *ptr)
3938 {
3939 data = _mm512_cvtps_pd(_mm256_loadu_ps(ptr));
3940 }
3941
3942 /**
3943 * Write the content of the calling class into memory in form of @p
3944 * size() to the given address. The memory need not be aligned by
3945 * 64 bytes, as opposed to casting a double address to
3946 * VectorizedArray<double>*.
3947 */
3948 DEAL_II_ALWAYS_INLINE
3949 void
3950 store(double *ptr) const
3951 {
3952 _mm512_storeu_pd(ptr, data);
3953 }
3954
3955 DEAL_II_ALWAYS_INLINE
3956 void
3957 store(float *ptr) const
3958 {
3959 _mm256_storeu_ps(ptr, _mm512_cvtpd_ps(data));
3960 }
3961
3962 /**
3963 * @copydoc VectorizedArray<Number>::streaming_store()
3964 * @note Memory must be aligned by 64 bytes.
3965 */
3966 DEAL_II_ALWAYS_INLINE
3967 void
3968 streaming_store(double *ptr) const
3969 {
3970 Assert(reinterpret_cast<std::size_t>(ptr) % 64 == 0,
3971 ExcMessage("Memory not aligned"));
3972 _mm512_stream_pd(ptr, data);
3973 }
3974
3975 /**
3976 * Load @p size() from memory into the calling class, starting at
3977 * the given address and with given offsets, each entry from the offset
3978 * providing one element of the vectorized array.
3979 *
3980 * This operation corresponds to the following code (but uses a more
3981 * efficient implementation in case the hardware allows for that):
3982 * @code
3983 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
3984 * this->operator[](v) = base_ptr[offsets[v]];
3985 * @endcode
3986 */
3987 DEAL_II_ALWAYS_INLINE
3988 void
3989 gather(const double *base_ptr, const unsigned int *offsets)
3990 {
3991 # ifdef DEAL_II_USE_VECTORIZATION_GATHER
3992 // unfortunately, there does not appear to be a 256 bit integer load, so
3993 // do it by some reinterpret casts here. this is allowed because the Intel
3994 // API allows aliasing between different vector types.
3995 const __m256 index_val =
3996 _mm256_loadu_ps(reinterpret_cast<const float *>(offsets));
3997 const __m256i index = *reinterpret_cast<const __m256i *>(&index_val);
3998
3999 // work around a warning with gcc-12 about an uninitialized initial state
4000 // for gather by starting with a zero guess, even though all lanes will be
4001 // overwritten
4002 __m512d zero = {};
4003 __mmask8 mask = 0xFF;
4004
4005 data = _mm512_mask_i32gather_pd(zero, mask, index, base_ptr, 8);
4006 # else
4007 for (unsigned int i = 0; i < 8; ++i)
4008 *(reinterpret_cast<double *>(&data) + i) = base_ptr[offsets[i]];
4009 # endif
4010 }
4011
4012 /**
4013 * Write the content of the calling class into memory in form of @p
4014 * size() to the given address and the given offsets, filling the
4015 * elements of the vectorized array into each offset.
4016 *
4017 * This operation corresponds to the following code (but uses a more
4018 * efficient implementation in case the hardware allows for that):
4019 * @code
4020 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
4021 * base_ptr[offsets[v]] = this->operator[](v);
4022 * @endcode
4023 */
4024 DEAL_II_ALWAYS_INLINE
4025 void
4026 scatter(const unsigned int *offsets, double *base_ptr) const
4027 {
4028 # ifdef DEAL_II_USE_VECTORIZATION_GATHER
4029 for (unsigned int i = 0; i < 8; ++i)
4030 for (unsigned int j = i + 1; j < 8; ++j)
4031 Assert(offsets[i] != offsets[j],
4032 ExcMessage("Result of scatter undefined if two offset elements"
4033 " point to the same position"));
4034
4035 // unfortunately, there does not appear to be a 256 bit integer load, so
4036 // do it by some reinterpret casts here. this is allowed because the Intel
4037 // API allows aliasing between different vector types.
4038 const __m256 index_val =
4039 _mm256_loadu_ps(reinterpret_cast<const float *>(offsets));
4040 const __m256i index = *reinterpret_cast<const __m256i *>(&index_val);
4041 _mm512_i32scatter_pd(base_ptr, index, data, 8);
4042 # else
4043 for (unsigned int i = 0; i < 8; ++i)
4044 base_ptr[offsets[i]] = *(reinterpret_cast<const double *>(&data) + i);
4045 # endif
4046 }
4047
4048 /**
4049 * Returns sum over entries of the data field, $\sum_{i=1}^{\text{size}()}
4050 * this->data[i]$.
4051 */
4052 double
4053 sum() const
4054 {
4055 VectorizedArray<double, 4> t1;
4056 t1.data = _mm256_add_pd(this->get_lower(), this->get_upper());
4057 return t1.sum();
4058 }
4059
4060 /**
4061 * Actual data field. To be consistent with the standard layout type and to
4062 * enable interaction with external SIMD functionality, this member is
4063 * declared public.
4064 */
4065 __m512d data;
4066
4067 private:
4068 /**
4069 * Extract lower half of data field.
4070 */
4071 DEAL_II_ALWAYS_INLINE
4072 __m256d
4073 get_lower() const
4074 {
4075 return _mm512_castpd512_pd256(data);
4076 }
4077
4078 /**
4079 * Extract upper half of data field.
4080 */
4081 DEAL_II_ALWAYS_INLINE
4082 __m256d
4083 get_upper() const
4084 {
4085 return _mm512_extractf64x4_pd(data, 1);
4086 }
4087
4088 /**
4089 * Return the square root of this field. Not for use in user code. Use
4090 * sqrt(x) instead.
4091 */
4092 DEAL_II_ALWAYS_INLINE
4093 VectorizedArray
4094 get_sqrt() const
4095 {
4096 VectorizedArray res;
4097 res.data = _mm512_sqrt_pd(data);
4098 return res;
4099 }
4100
4101 /**
4102 * Return the absolute value of this field. Not for use in user code. Use
4103 * abs(x) instead.
4104 */
4105 DEAL_II_ALWAYS_INLINE
4106 VectorizedArray
4107 get_abs() const
4108 {
4109 // to compute the absolute value, perform bitwise andnot with -0. This
4110 // will leave all value and exponent bits unchanged but force the sign
4111 // value to +. Since there is no andnot for AVX512, we interpret the data
4112 // as 64 bit integers and do the andnot on those types (note that andnot
4113 // is a bitwise operation so the data type does not matter)
4114 __m512d mask = _mm512_set1_pd(-0.);
4115 VectorizedArray res;
4116 res.data = reinterpret_cast<__m512d>(
4117 _mm512_andnot_epi64(reinterpret_cast<__m512i>(mask),
4118 reinterpret_cast<__m512i>(data)));
4119 return res;
4120 }
4121
4122 /**
4123 * Return the component-wise maximum of this field and another one. Not for
4124 * use in user code. Use max(x,y) instead.
4125 */
4126 DEAL_II_ALWAYS_INLINE
4127 VectorizedArray
4128 get_max(const VectorizedArray &other) const
4129 {
4130 VectorizedArray res;
4131 res.data = _mm512_max_pd(data, other.data);
4132 return res;
4133 }
4134
4135 /**
4136 * Return the component-wise minimum of this field and another one. Not for
4137 * use in user code. Use min(x,y) instead.
4138 */
4139 DEAL_II_ALWAYS_INLINE
4140 VectorizedArray
4141 get_min(const VectorizedArray &other) const
4142 {
4143 VectorizedArray res;
4144 res.data = _mm512_min_pd(data, other.data);
4145 return res;
4146 }
4147
4148 // Make a few functions friends.
4149 template <typename Number2, std::size_t width2>
4150 friend VectorizedArray<Number2, width2>
4151 std::sqrt(const VectorizedArray<Number2, width2> &);
4152 template <typename Number2, std::size_t width2>
4153 friend VectorizedArray<Number2, width2>
4154 std::abs(const VectorizedArray<Number2, width2> &);
4155 template <typename Number2, std::size_t width2>
4156 friend VectorizedArray<Number2, width2>
4157 std::max(const VectorizedArray<Number2, width2> &,
4158 const VectorizedArray<Number2, width2> &);
4159 template <typename Number2, std::size_t width2>
4160 friend VectorizedArray<Number2, width2>
4161 std::min(const VectorizedArray<Number2, width2> &,
4162 const VectorizedArray<Number2, width2> &);
4163 };
4164
4165
4166
4167 /**
4168 * Specialization for double and AVX-512.
4169 */
4170 template <>
4171 inline DEAL_II_ALWAYS_INLINE void
4172 vectorized_load_and_transpose(const unsigned int n_entries,
4173 const double *in,
4174 const unsigned int *offsets,
4175 VectorizedArray<double, 8> *out)
4176 {
4177 // do not do full transpose because the code is long and will most
4178 // likely not pay off because many processors have two load units
4179 // (for the top 8 instructions) but only 1 permute unit (for the 8
4180 // shuffle/unpack instructions). rather start the transposition on the
4181 // vectorized array of half the size with 256 bits
4182 const unsigned int n_chunks = n_entries / 4;
4183 for (unsigned int i = 0; i < n_chunks; ++i)
4184 {
4185 __m512d t0, t1, t2, t3 = {};
4186
4187 t0 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in + offsets[0] + 4 * i), 0);
4188 t0 = _mm512_insertf64x4(t0, _mm256_loadu_pd(in + offsets[2] + 4 * i), 1);
4189 t1 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in + offsets[1] + 4 * i), 0);
4190 t1 = _mm512_insertf64x4(t1, _mm256_loadu_pd(in + offsets[3] + 4 * i), 1);
4191 t2 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in + offsets[4] + 4 * i), 0);
4192 t2 = _mm512_insertf64x4(t2, _mm256_loadu_pd(in + offsets[6] + 4 * i), 1);
4193 t3 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in + offsets[5] + 4 * i), 0);
4194 t3 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in + offsets[7] + 4 * i), 1);
4195
4196 __m512d v0 = _mm512_shuffle_f64x2(t0, t2, 0x88);
4197 __m512d v1 = _mm512_shuffle_f64x2(t0, t2, 0xdd);
4198 __m512d v2 = _mm512_shuffle_f64x2(t1, t3, 0x88);
4199 __m512d v3 = _mm512_shuffle_f64x2(t1, t3, 0xdd);
4200 out[4 * i + 0].data = _mm512_unpacklo_pd(v0, v2);
4201 out[4 * i + 1].data = _mm512_unpackhi_pd(v0, v2);
4202 out[4 * i + 2].data = _mm512_unpacklo_pd(v1, v3);
4203 out[4 * i + 3].data = _mm512_unpackhi_pd(v1, v3);
4204 }
4205 // remainder loop of work that does not divide by 4
4206 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
4207 out[i].gather(in + i, offsets);
4208 }
4209
4210
4211
4212 /**
4213 * Specialization for double and AVX-512.
4214 */
4215 template <>
4216 inline DEAL_II_ALWAYS_INLINE void
4217 vectorized_load_and_transpose(const unsigned int n_entries,
4218 const std::array<double *, 8> &in,
4219 VectorizedArray<double, 8> *out)
4220 {
4221 const unsigned int n_chunks = n_entries / 4;
4222 for (unsigned int i = 0; i < n_chunks; ++i)
4223 {
4224 __m512d t0, t1, t2, t3 = {};
4225
4226 t0 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in[0] + 4 * i), 0);
4227 t0 = _mm512_insertf64x4(t0, _mm256_loadu_pd(in[2] + 4 * i), 1);
4228 t1 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in[1] + 4 * i), 0);
4229 t1 = _mm512_insertf64x4(t1, _mm256_loadu_pd(in[3] + 4 * i), 1);
4230 t2 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in[4] + 4 * i), 0);
4231 t2 = _mm512_insertf64x4(t2, _mm256_loadu_pd(in[6] + 4 * i), 1);
4232 t3 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in[5] + 4 * i), 0);
4233 t3 = _mm512_insertf64x4(t3, _mm256_loadu_pd(in[7] + 4 * i), 1);
4234
4235 __m512d v0 = _mm512_shuffle_f64x2(t0, t2, 0x88);
4236 __m512d v1 = _mm512_shuffle_f64x2(t0, t2, 0xdd);
4237 __m512d v2 = _mm512_shuffle_f64x2(t1, t3, 0x88);
4238 __m512d v3 = _mm512_shuffle_f64x2(t1, t3, 0xdd);
4239 out[4 * i + 0].data = _mm512_unpacklo_pd(v0, v2);
4240 out[4 * i + 1].data = _mm512_unpackhi_pd(v0, v2);
4241 out[4 * i + 2].data = _mm512_unpacklo_pd(v1, v3);
4242 out[4 * i + 3].data = _mm512_unpackhi_pd(v1, v3);
4243 }
4244
4245 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
4246 gather(out[i], in, i);
4247 }
4248
4249
4250
4251 /**
4252 * Specialization for double and AVX-512.
4253 */
4254 template <>
4255 inline DEAL_II_ALWAYS_INLINE void
4256 vectorized_transpose_and_store(const bool add_into,
4257 const unsigned int n_entries,
4258 const VectorizedArray<double, 8> *in,
4259 const unsigned int *offsets,
4260 double *out)
4261 {
4262 // as for the load, we split the store operations into 256 bit units to
4263 // better balance between code size, shuffle instructions, and stores
4264 const unsigned int n_chunks = n_entries / 4;
4265 __m512i mask1 = _mm512_set_epi64(0xd, 0xc, 0x5, 0x4, 0x9, 0x8, 0x1, 0x0);
4266 __m512i mask2 = _mm512_set_epi64(0xf, 0xe, 0x7, 0x6, 0xb, 0xa, 0x3, 0x2);
4267 for (unsigned int i = 0; i < n_chunks; ++i)
4268 {
4269 __m512d t0 = _mm512_unpacklo_pd(in[i * 4].data, in[i * 4 + 1].data);
4270 __m512d t1 = _mm512_unpackhi_pd(in[i * 4].data, in[i * 4 + 1].data);
4271 __m512d t2 = _mm512_unpacklo_pd(in[i * 4 + 2].data, in[i * 4 + 3].data);
4272 __m512d t3 = _mm512_unpackhi_pd(in[i * 4 + 2].data, in[i * 4 + 3].data);
4273 __m512d v0 = _mm512_permutex2var_pd(t0, mask1, t2);
4274 __m512d v1 = _mm512_permutex2var_pd(t0, mask2, t2);
4275 __m512d v2 = _mm512_permutex2var_pd(t1, mask1, t3);
4276 __m512d v3 = _mm512_permutex2var_pd(t1, mask2, t3);
4277 __m256d res0 = _mm512_extractf64x4_pd(v0, 0);
4278 __m256d res4 = _mm512_extractf64x4_pd(v0, 1);
4279 __m256d res1 = _mm512_extractf64x4_pd(v2, 0);
4280 __m256d res5 = _mm512_extractf64x4_pd(v2, 1);
4281 __m256d res2 = _mm512_extractf64x4_pd(v1, 0);
4282 __m256d res6 = _mm512_extractf64x4_pd(v1, 1);
4283 __m256d res3 = _mm512_extractf64x4_pd(v3, 0);
4284 __m256d res7 = _mm512_extractf64x4_pd(v3, 1);
4285
4286 // Cannot use the same store instructions in both paths of the 'if'
4287 // because the compiler cannot know that there is no aliasing
4288 // between pointers
4289 if (add_into)
4290 {
4291 res0 = _mm256_add_pd(_mm256_loadu_pd(out + 4 * i + offsets[0]), res0);
4292 _mm256_storeu_pd(out + 4 * i + offsets[0], res0);
4293 res1 = _mm256_add_pd(_mm256_loadu_pd(out + 4 * i + offsets[1]), res1);
4294 _mm256_storeu_pd(out + 4 * i + offsets[1], res1);
4295 res2 = _mm256_add_pd(_mm256_loadu_pd(out + 4 * i + offsets[2]), res2);
4296 _mm256_storeu_pd(out + 4 * i + offsets[2], res2);
4297 res3 = _mm256_add_pd(_mm256_loadu_pd(out + 4 * i + offsets[3]), res3);
4298 _mm256_storeu_pd(out + 4 * i + offsets[3], res3);
4299 res4 = _mm256_add_pd(_mm256_loadu_pd(out + 4 * i + offsets[4]), res4);
4300 _mm256_storeu_pd(out + 4 * i + offsets[4], res4);
4301 res5 = _mm256_add_pd(_mm256_loadu_pd(out + 4 * i + offsets[5]), res5);
4302 _mm256_storeu_pd(out + 4 * i + offsets[5], res5);
4303 res6 = _mm256_add_pd(_mm256_loadu_pd(out + 4 * i + offsets[6]), res6);
4304 _mm256_storeu_pd(out + 4 * i + offsets[6], res6);
4305 res7 = _mm256_add_pd(_mm256_loadu_pd(out + 4 * i + offsets[7]), res7);
4306 _mm256_storeu_pd(out + 4 * i + offsets[7], res7);
4307 }
4308 else
4309 {
4310 _mm256_storeu_pd(out + 4 * i + offsets[0], res0);
4311 _mm256_storeu_pd(out + 4 * i + offsets[1], res1);
4312 _mm256_storeu_pd(out + 4 * i + offsets[2], res2);
4313 _mm256_storeu_pd(out + 4 * i + offsets[3], res3);
4314 _mm256_storeu_pd(out + 4 * i + offsets[4], res4);
4315 _mm256_storeu_pd(out + 4 * i + offsets[5], res5);
4316 _mm256_storeu_pd(out + 4 * i + offsets[6], res6);
4317 _mm256_storeu_pd(out + 4 * i + offsets[7], res7);
4318 }
4319 }
4320
4321 // remainder loop of work that does not divide by 4
4322 if (add_into)
4323 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
4324 for (unsigned int v = 0; v < 8; ++v)
4325 out[offsets[v] + i] += in[i][v];
4326 else
4327 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
4328 for (unsigned int v = 0; v < 8; ++v)
4329 out[offsets[v] + i] = in[i][v];
4330 }
4331
4332
4333
4334 /**
4335 * Specialization for double and AVX-512.
4336 */
4337 template <>
4338 inline DEAL_II_ALWAYS_INLINE void
4339 vectorized_transpose_and_store(const bool add_into,
4340 const unsigned int n_entries,
4341 const VectorizedArray<double, 8> *in,
4342 std::array<double *, 8> &out)
4343 {
4344 // see the comments in the vectorized_transpose_and_store above
4345
4346 const unsigned int n_chunks = n_entries / 4;
4347 __m512i mask1 = _mm512_set_epi64(0xd, 0xc, 0x5, 0x4, 0x9, 0x8, 0x1, 0x0);
4348 __m512i mask2 = _mm512_set_epi64(0xf, 0xe, 0x7, 0x6, 0xb, 0xa, 0x3, 0x2);
4349 for (unsigned int i = 0; i < n_chunks; ++i)
4350 {
4351 __m512d t0 = _mm512_unpacklo_pd(in[i * 4].data, in[i * 4 + 1].data);
4352 __m512d t1 = _mm512_unpackhi_pd(in[i * 4].data, in[i * 4 + 1].data);
4353 __m512d t2 = _mm512_unpacklo_pd(in[i * 4 + 2].data, in[i * 4 + 3].data);
4354 __m512d t3 = _mm512_unpackhi_pd(in[i * 4 + 2].data, in[i * 4 + 3].data);
4355 __m512d v0 = _mm512_permutex2var_pd(t0, mask1, t2);
4356 __m512d v1 = _mm512_permutex2var_pd(t0, mask2, t2);
4357 __m512d v2 = _mm512_permutex2var_pd(t1, mask1, t3);
4358 __m512d v3 = _mm512_permutex2var_pd(t1, mask2, t3);
4359 __m256d res0 = _mm512_extractf64x4_pd(v0, 0);
4360 __m256d res4 = _mm512_extractf64x4_pd(v0, 1);
4361 __m256d res1 = _mm512_extractf64x4_pd(v2, 0);
4362 __m256d res5 = _mm512_extractf64x4_pd(v2, 1);
4363 __m256d res2 = _mm512_extractf64x4_pd(v1, 0);
4364 __m256d res6 = _mm512_extractf64x4_pd(v1, 1);
4365 __m256d res3 = _mm512_extractf64x4_pd(v3, 0);
4366 __m256d res7 = _mm512_extractf64x4_pd(v3, 1);
4367
4368 if (add_into)
4369 {
4370 res0 = _mm256_add_pd(_mm256_loadu_pd(out[0] + 4 * i), res0);
4371 _mm256_storeu_pd(out[0] + 4 * i, res0);
4372 res1 = _mm256_add_pd(_mm256_loadu_pd(out[1] + 4 * i), res1);
4373 _mm256_storeu_pd(out[1] + 4 * i, res1);
4374 res2 = _mm256_add_pd(_mm256_loadu_pd(out[2] + 4 * i), res2);
4375 _mm256_storeu_pd(out[2] + 4 * i, res2);
4376 res3 = _mm256_add_pd(_mm256_loadu_pd(out[3] + 4 * i), res3);
4377 _mm256_storeu_pd(out[3] + 4 * i, res3);
4378 res4 = _mm256_add_pd(_mm256_loadu_pd(out[4] + 4 * i), res4);
4379 _mm256_storeu_pd(out[4] + 4 * i, res4);
4380 res5 = _mm256_add_pd(_mm256_loadu_pd(out[5] + 4 * i), res5);
4381 _mm256_storeu_pd(out[5] + 4 * i, res5);
4382 res6 = _mm256_add_pd(_mm256_loadu_pd(out[6] + 4 * i), res6);
4383 _mm256_storeu_pd(out[6] + 4 * i, res6);
4384 res7 = _mm256_add_pd(_mm256_loadu_pd(out[7] + 4 * i), res7);
4385 _mm256_storeu_pd(out[7] + 4 * i, res7);
4386 }
4387 else
4388 {
4389 _mm256_storeu_pd(out[0] + 4 * i, res0);
4390 _mm256_storeu_pd(out[1] + 4 * i, res1);
4391 _mm256_storeu_pd(out[2] + 4 * i, res2);
4392 _mm256_storeu_pd(out[3] + 4 * i, res3);
4393 _mm256_storeu_pd(out[4] + 4 * i, res4);
4394 _mm256_storeu_pd(out[5] + 4 * i, res5);
4395 _mm256_storeu_pd(out[6] + 4 * i, res6);
4396 _mm256_storeu_pd(out[7] + 4 * i, res7);
4397 }
4398 }
4399
4400 if (add_into)
4401 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
4402 for (unsigned int v = 0; v < 8; ++v)
4403 out[v][i] += in[i][v];
4404 else
4405 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
4406 for (unsigned int v = 0; v < 8; ++v)
4407 out[v][i] = in[i][v];
4408 }
4409
4410
4411
4412 /**
4413 * Specialization for float and AVX512.
4414 */
4415 template <>
4416 class VectorizedArray<float, 16>
4417 : public VectorizedArrayBase<VectorizedArray<float, 16>, 16>
4418 {
4419 public:
4420 /**
4421 * This gives the type of the array elements.
4422 */
4423 using value_type = float;
4424
4425 /**
4426 * Record the fact that the given specialization of VectorizedArray is
4427 * indeed implemented.
4428 */
4429 static constexpr bool is_implemented = true;
4430
4431 /**
4432 * Default empty constructor, leaving the data in an uninitialized state
4433 * similar to float/double.
4434 */
4435 VectorizedArray() = default;
4436
4437 /**
4438 * Construct an array with the given scalar broadcast to all lanes.
4439 */
4440 VectorizedArray(const float scalar)
4441 {
4442 this->operator=(scalar);
4443 }
4444
4445 /**
4446 * Construct an array with the given initializer list.
4447 */
4448 template <typename U>
4449 VectorizedArray(const std::initializer_list<U> &list)
4450 : VectorizedArrayBase<VectorizedArray<float, 16>, 16>(list)
4451 {}
4452
4453 /**
4454 * This function can be used to set all data fields to a given scalar.
4455 */
4456 DEAL_II_ALWAYS_INLINE
4457 VectorizedArray &
4458 operator=(const float x) &
4459 {
4460 data = _mm512_set1_ps(x);
4461 return *this;
4462 }
4463
4464 /**
4465 * Assign a scalar to the current object. This overload is used for
4466 * rvalue references; because it does not make sense to assign
4467 * something to a temporary, the function is deleted.
4468 */
4469 VectorizedArray &
4470 operator=(const float scalar) && = delete;
4471
4472 /**
4473 * Access operator.
4474 */
4475 DEAL_II_ALWAYS_INLINE
4476 float &
4477 operator[](const unsigned int comp)
4478 {
4479 AssertIndexRange(comp, 16);
4480 return *(reinterpret_cast<float *>(&data) + comp);
4481 }
4482
4483 /**
4484 * Constant access operator.
4485 */
4486 DEAL_II_ALWAYS_INLINE
4487 const float &
4488 operator[](const unsigned int comp) const
4489 {
4490 AssertIndexRange(comp, 16);
4491 return *(reinterpret_cast<const float *>(&data) + comp);
4492 }
4493
4494 /**
4495 * Element-wise addition of two arrays of numbers.
4496 */
4497 DEAL_II_ALWAYS_INLINE
4498 VectorizedArray &
4499 operator+=(const VectorizedArray &vec)
4500 {
4501 // if the compiler supports vector arithmetic, we can simply use +=
4502 // operator on the given data type. this allows the compiler to combine
4503 // additions with multiplication (fused multiply-add) if those
4504 // instructions are available. Otherwise, we need to use the built-in
4505 // intrinsic command for __m512d
4506 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
4507 data += vec.data;
4508 # else
4509 data = _mm512_add_ps(data, vec.data);
4510 # endif
4511 return *this;
4512 }
4513
4514 /**
4515 * Element-wise subtraction of two arrays of numbers.
4516 */
4517 DEAL_II_ALWAYS_INLINE
4518 VectorizedArray &
4519 operator-=(const VectorizedArray &vec)
4520 {
4521 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
4522 data -= vec.data;
4523 # else
4524 data = _mm512_sub_ps(data, vec.data);
4525 # endif
4526 return *this;
4527 }
4528 /**
4529 * Element-wise multiplication of two arrays of numbers.
4530 */
4531 DEAL_II_ALWAYS_INLINE
4532 VectorizedArray &
4533 operator*=(const VectorizedArray &vec)
4534 {
4535 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
4536 data *= vec.data;
4537 # else
4538 data = _mm512_mul_ps(data, vec.data);
4539 # endif
4540 return *this;
4541 }
4542
4543 /**
4544 * Element-wise division of two arrays of numbers.
4545 */
4546 DEAL_II_ALWAYS_INLINE
4547 VectorizedArray &
4548 operator/=(const VectorizedArray &vec)
4549 {
4550 # ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS
4551 data /= vec.data;
4552 # else
4553 data = _mm512_div_ps(data, vec.data);
4554 # endif
4555 return *this;
4556 }
4557
4558 /**
4559 * Load @p size() from memory into the calling class, starting at
4560 * the given address. The memory need not be aligned by 64 bytes, as opposed
4561 * to casting a float address to VectorizedArray<float>*.
4562 */
4563 DEAL_II_ALWAYS_INLINE
4564 void
4565 load(const float *ptr)
4566 {
4567 data = _mm512_loadu_ps(ptr);
4568 }
4569
4570 /**
4571 * Write the content of the calling class into memory in form of @p
4572 * size() to the given address. The memory need not be aligned by
4573 * 64 bytes, as opposed to casting a float address to
4574 * VectorizedArray<float>*.
4575 */
4576 DEAL_II_ALWAYS_INLINE
4577 void
4578 store(float *ptr) const
4579 {
4580 _mm512_storeu_ps(ptr, data);
4581 }
4582
4583 /**
4584 * @copydoc VectorizedArray<Number>::streaming_store()
4585 * @note Memory must be aligned by 64 bytes.
4586 */
4587 DEAL_II_ALWAYS_INLINE
4588 void
4589 streaming_store(float *ptr) const
4590 {
4591 Assert(reinterpret_cast<std::size_t>(ptr) % 64 == 0,
4592 ExcMessage("Memory not aligned"));
4593 _mm512_stream_ps(ptr, data);
4594 }
4595
4596 /**
4597 * Load @p size() from memory into the calling class, starting at
4598 * the given address and with given offsets, each entry from the offset
4599 * providing one element of the vectorized array.
4600 *
4601 * This operation corresponds to the following code (but uses a more
4602 * efficient implementation in case the hardware allows for that):
4603 * @code
4604 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
4605 * this->operator[](v) = base_ptr[offsets[v]];
4606 * @endcode
4607 */
4608 DEAL_II_ALWAYS_INLINE
4609 void
4610 gather(const float *base_ptr, const unsigned int *offsets)
4611 {
4612 # ifdef DEAL_II_USE_VECTORIZATION_GATHER
4613 // unfortunately, there does not appear to be a 512 bit integer load, so
4614 // do it by some reinterpret casts here. this is allowed because the Intel
4615 // API allows aliasing between different vector types.
4616 const __m512 index_val =
4617 _mm512_loadu_ps(reinterpret_cast<const float *>(offsets));
4618 const __m512i index = *reinterpret_cast<const __m512i *>(&index_val);
4619
4620 // work around a warning with gcc-12 about an uninitialized initial state
4621 // for gather by starting with a zero guess, even though all lanes will be
4622 // overwritten
4623 __m512 zero = {};
4624 __mmask16 mask = 0xFFFF;
4625
4626 data = _mm512_mask_i32gather_ps(zero, mask, index, base_ptr, 4);
4627 # else
4628 for (unsigned int i = 0; i < 16; ++i)
4629 *(reinterpret_cast<float *>(&data) + i) = base_ptr[offsets[i]];
4630 # endif
4631 }
4632
4633 /**
4634 * Write the content of the calling class into memory in form of @p
4635 * size() to the given address and the given offsets, filling the
4636 * elements of the vectorized array into each offset.
4637 *
4638 * This operation corresponds to the following code (but uses a more
4639 * efficient implementation in case the hardware allows for that):
4640 * @code
4641 * for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
4642 * base_ptr[offsets[v]] = this->operator[](v);
4643 * @endcode
4644 */
4645 DEAL_II_ALWAYS_INLINE
4646 void
4647 scatter(const unsigned int *offsets, float *base_ptr) const
4648 {
4649 # ifdef DEAL_II_USE_VECTORIZATION_GATHER
4650 for (unsigned int i = 0; i < 16; ++i)
4651 for (unsigned int j = i + 1; j < 16; ++j)
4652 Assert(offsets[i] != offsets[j],
4653 ExcMessage("Result of scatter undefined if two offset elements"
4654 " point to the same position"));
4655
4656 // unfortunately, there does not appear to be a 512 bit integer load, so
4657 // do it by some reinterpret casts here. this is allowed because the Intel
4658 // API allows aliasing between different vector types.
4659 const __m512 index_val =
4660 _mm512_loadu_ps(reinterpret_cast<const float *>(offsets));
4661 const __m512i index = *reinterpret_cast<const __m512i *>(&index_val);
4662 _mm512_i32scatter_ps(base_ptr, index, data, 4);
4663 # else
4664 for (unsigned int i = 0; i < 16; ++i)
4665 base_ptr[offsets[i]] = *(reinterpret_cast<const float *>(&data) + i);
4666 # endif
4667 }
4668
4669 /**
4670 * Returns sum over entries of the data field, $\sum_{i=1}^{\text{size}()}
4671 * this->data[i]$.
4672 */
4673 float
4674 sum() const
4675 {
4676 VectorizedArray<float, 8> t1;
4677 t1.data = _mm256_add_ps(this->get_lower(), this->get_upper());
4678 return t1.sum();
4679 }
4680
4681 /**
4682 * Actual data field. To be consistent with the standard layout type and to
4683 * enable interaction with external SIMD functionality, this member is
4684 * declared public.
4685 */
4686 __m512 data;
4687
4688 private:
4689 /**
4690 * Extract lower half of data field.
4691 */
4692 DEAL_II_ALWAYS_INLINE
4693 __m256
4694 get_lower() const
4695 {
4696 return _mm512_castps512_ps256(data);
4697 }
4698
4699 /**
4700 * Extract upper half of data field.
4701 */
4702 DEAL_II_ALWAYS_INLINE
4703 __m256
4704 get_upper() const
4705 {
4706 return _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(data), 1));
4707 }
4708
4709 /**
4710 * Return the square root of this field. Not for use in user code. Use
4711 * sqrt(x) instead.
4712 */
4713 DEAL_II_ALWAYS_INLINE
4714 VectorizedArray
4715 get_sqrt() const
4716 {
4717 VectorizedArray res;
4718 res.data = _mm512_sqrt_ps(data);
4719 return res;
4720 }
4721
4722 /**
4723 * Return the absolute value of this field. Not for use in user code. Use
4724 * abs(x) instead.
4725 */
4726 DEAL_II_ALWAYS_INLINE
4727 VectorizedArray
4728 get_abs() const
4729 {
4730 // to compute the absolute value, perform bitwise andnot with -0. This
4731 // will leave all value and exponent bits unchanged but force the sign
4732 // value to +. Since there is no andnot for AVX512, we interpret the data
4733 // as 32 bit integers and do the andnot on those types (note that andnot
4734 // is a bitwise operation so the data type does not matter)
4735 __m512 mask = _mm512_set1_ps(-0.f);
4736 VectorizedArray res;
4737 res.data = reinterpret_cast<__m512>(
4738 _mm512_andnot_epi32(reinterpret_cast<__m512i>(mask),
4739 reinterpret_cast<__m512i>(data)));
4740 return res;
4741 }
4742
4743 /**
4744 * Return the component-wise maximum of this field and another one. Not for
4745 * use in user code. Use max(x,y) instead.
4746 */
4747 DEAL_II_ALWAYS_INLINE
4748 VectorizedArray
4749 get_max(const VectorizedArray &other) const
4750 {
4751 VectorizedArray res;
4752 res.data = _mm512_max_ps(data, other.data);
4753 return res;
4754 }
4755
4756 /**
4757 * Return the component-wise minimum of this field and another one. Not for
4758 * use in user code. Use min(x,y) instead.
4759 */
4760 DEAL_II_ALWAYS_INLINE
4761 VectorizedArray
4762 get_min(const VectorizedArray &other) const
4763 {
4764 VectorizedArray res;
4765 res.data = _mm512_min_ps(data, other.data);
4766 return res;
4767 }
4768
4769 // Make a few functions friends.
4770 template <typename Number2, std::size_t width2>
4771 friend VectorizedArray<Number2, width2>
4772 std::sqrt(const VectorizedArray<Number2, width2> &);
4773 template <typename Number2, std::size_t width2>
4774 friend VectorizedArray<Number2, width2>
4775 std::abs(const VectorizedArray<Number2, width2> &);
4776 template <typename Number2, std::size_t width2>
4777 friend VectorizedArray<Number2, width2>
4778 std::max(const VectorizedArray<Number2, width2> &,
4779 const VectorizedArray<Number2, width2> &);
4780 template <typename Number2, std::size_t width2>
4781 friend VectorizedArray<Number2, width2>
4782 std::min(const VectorizedArray<Number2, width2> &,
4783 const VectorizedArray<Number2, width2> &);
4784 };
4785
4786
4787
4788 /**
4789 * Specialization for float and AVX-512.
4790 */
4791 template <>
4792 inline DEAL_II_ALWAYS_INLINE void
4793 vectorized_load_and_transpose(const unsigned int n_entries,
4794 const float *in,
4795 const unsigned int *offsets,
4796 VectorizedArray<float, 16> *out)
4797 {
4798 // Similar to the double case, we perform the work on smaller entities. In
4799 // this case, we start from 128 bit arrays and insert them into a full 512
4800 // bit index. This reduces the code size and register pressure because we do
4801 // shuffles on 4 numbers rather than 16.
4802 const unsigned int n_chunks = n_entries / 4;
4803
4804 // To avoid warnings about uninitialized variables, need to initialize one
4805 // variable to a pre-existing value in out, which will never get used in
4806 // the end. Keep the initialization outside the loop because of a bug in
4807 // gcc-9.1 which generates a "vmovapd" instruction instead of "vmovupd" in
4808 // case t3 is initialized to zero (inside/outside of loop), see
4809 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90991
4810 __m512 t0, t1, t2, t3;
4811 if (n_chunks > 0)
4812 t3 = out[0].data;
4813 for (unsigned int i = 0; i < n_chunks; ++i)
4814 {
4815 t0 = _mm512_insertf32x4(t3, _mm_loadu_ps(in + offsets[0] + 4 * i), 0);
4816 t0 = _mm512_insertf32x4(t0, _mm_loadu_ps(in + offsets[4] + 4 * i), 1);
4817 t0 = _mm512_insertf32x4(t0, _mm_loadu_ps(in + offsets[8] + 4 * i), 2);
4818 t0 = _mm512_insertf32x4(t0, _mm_loadu_ps(in + offsets[12] + 4 * i), 3);
4819 t1 = _mm512_insertf32x4(t3, _mm_loadu_ps(in + offsets[1] + 4 * i), 0);
4820 t1 = _mm512_insertf32x4(t1, _mm_loadu_ps(in + offsets[5] + 4 * i), 1);
4821 t1 = _mm512_insertf32x4(t1, _mm_loadu_ps(in + offsets[9] + 4 * i), 2);
4822 t1 = _mm512_insertf32x4(t1, _mm_loadu_ps(in + offsets[13] + 4 * i), 3);
4823 t2 = _mm512_insertf32x4(t3, _mm_loadu_ps(in + offsets[2] + 4 * i), 0);
4824 t2 = _mm512_insertf32x4(t2, _mm_loadu_ps(in + offsets[6] + 4 * i), 1);
4825 t2 = _mm512_insertf32x4(t2, _mm_loadu_ps(in + offsets[10] + 4 * i), 2);
4826 t2 = _mm512_insertf32x4(t2, _mm_loadu_ps(in + offsets[14] + 4 * i), 3);
4827 t3 = _mm512_insertf32x4(t3, _mm_loadu_ps(in + offsets[3] + 4 * i), 0);
4828 t3 = _mm512_insertf32x4(t3, _mm_loadu_ps(in + offsets[7] + 4 * i), 1);
4829 t3 = _mm512_insertf32x4(t3, _mm_loadu_ps(in + offsets[11] + 4 * i), 2);
4830 t3 = _mm512_insertf32x4(t3, _mm_loadu_ps(in + offsets[15] + 4 * i), 3);
4831
4832 __m512 v0 = _mm512_shuffle_ps(t0, t1, 0x44);
4833 __m512 v1 = _mm512_shuffle_ps(t0, t1, 0xee);
4834 __m512 v2 = _mm512_shuffle_ps(t2, t3, 0x44);
4835 __m512 v3 = _mm512_shuffle_ps(t2, t3, 0xee);
4836
4837 out[4 * i + 0].data = _mm512_shuffle_ps(v0, v2, 0x88);
4838 out[4 * i + 1].data = _mm512_shuffle_ps(v0, v2, 0xdd);
4839 out[4 * i + 2].data = _mm512_shuffle_ps(v1, v3, 0x88);
4840 out[4 * i + 3].data = _mm512_shuffle_ps(v1, v3, 0xdd);
4841 }
4842
4843 // remainder loop of work that does not divide by 4
4844 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
4845 out[i].gather(in + i, offsets);
4846 }
4847
4848
4849
4850 /**
4851 * Specialization for float and AVX-512.
4852 */
4853 template <>
4854 inline DEAL_II_ALWAYS_INLINE void
4855 vectorized_load_and_transpose(const unsigned int n_entries,
4856 const std::array<float *, 16> &in,
4857 VectorizedArray<float, 16> *out)
4858 {
4859 // see the comments in the vectorized_load_and_transpose above
4860
4861 const unsigned int n_chunks = n_entries / 4;
4862
4863 __m512 t0, t1, t2, t3;
4864 if (n_chunks > 0)
4865 t3 = out[0].data;
4866 for (unsigned int i = 0; i < n_chunks; ++i)
4867 {
4868 t0 = _mm512_insertf32x4(t3, _mm_loadu_ps(in[0] + 4 * i), 0);
4869 t0 = _mm512_insertf32x4(t0, _mm_loadu_ps(in[4] + 4 * i), 1);
4870 t0 = _mm512_insertf32x4(t0, _mm_loadu_ps(in[8] + 4 * i), 2);
4871 t0 = _mm512_insertf32x4(t0, _mm_loadu_ps(in[12] + 4 * i), 3);
4872 t1 = _mm512_insertf32x4(t3, _mm_loadu_ps(in[1] + 4 * i), 0);
4873 t1 = _mm512_insertf32x4(t1, _mm_loadu_ps(in[5] + 4 * i), 1);
4874 t1 = _mm512_insertf32x4(t1, _mm_loadu_ps(in[9] + 4 * i), 2);
4875 t1 = _mm512_insertf32x4(t1, _mm_loadu_ps(in[13] + 4 * i), 3);
4876 t2 = _mm512_insertf32x4(t3, _mm_loadu_ps(in[2] + 4 * i), 0);
4877 t2 = _mm512_insertf32x4(t2, _mm_loadu_ps(in[6] + 4 * i), 1);
4878 t2 = _mm512_insertf32x4(t2, _mm_loadu_ps(in[10] + 4 * i), 2);
4879 t2 = _mm512_insertf32x4(t2, _mm_loadu_ps(in[14] + 4 * i), 3);
4880 t3 = _mm512_insertf32x4(t3, _mm_loadu_ps(in[3] + 4 * i), 0);
4881 t3 = _mm512_insertf32x4(t3, _mm_loadu_ps(in[7] + 4 * i), 1);
4882 t3 = _mm512_insertf32x4(t3, _mm_loadu_ps(in[11] + 4 * i), 2);
4883 t3 = _mm512_insertf32x4(t3, _mm_loadu_ps(in[15] + 4 * i), 3);
4884
4885 __m512 v0 = _mm512_shuffle_ps(t0, t1, 0x44);
4886 __m512 v1 = _mm512_shuffle_ps(t0, t1, 0xee);
4887 __m512 v2 = _mm512_shuffle_ps(t2, t3, 0x44);
4888 __m512 v3 = _mm512_shuffle_ps(t2, t3, 0xee);
4889
4890 out[4 * i + 0].data = _mm512_shuffle_ps(v0, v2, 0x88);
4891 out[4 * i + 1].data = _mm512_shuffle_ps(v0, v2, 0xdd);
4892 out[4 * i + 2].data = _mm512_shuffle_ps(v1, v3, 0x88);
4893 out[4 * i + 3].data = _mm512_shuffle_ps(v1, v3, 0xdd);
4894 }
4895
4896 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
4897 gather(out[i], in, i);
4898 }
4899
4900
4901
4902 /**
4903 * Specialization for float and AVX-512.
4904 */
4905 template <>
4906 inline DEAL_II_ALWAYS_INLINE void
4907 vectorized_transpose_and_store(const bool add_into,
4908 const unsigned int n_entries,
4909 const VectorizedArray<float, 16> *in,
4910 const unsigned int *offsets,
4911 float *out)
4912 {
4913 const unsigned int n_chunks = n_entries / 4;
4914 for (unsigned int i = 0; i < n_chunks; ++i)
4915 {
4916 __m512 t0 = _mm512_shuffle_ps(in[4 * i].data, in[1 + 4 * i].data, 0x44);
4917 __m512 t1 = _mm512_shuffle_ps(in[4 * i].data, in[1 + 4 * i].data, 0xee);
4918 __m512 t2 =
4919 _mm512_shuffle_ps(in[2 + 4 * i].data, in[3 + 4 * i].data, 0x44);
4920 __m512 t3 =
4921 _mm512_shuffle_ps(in[2 + 4 * i].data, in[3 + 4 * i].data, 0xee);
4922 __m512 u0 = _mm512_shuffle_ps(t0, t2, 0x88);
4923 __m512 u1 = _mm512_shuffle_ps(t0, t2, 0xdd);
4924 __m512 u2 = _mm512_shuffle_ps(t1, t3, 0x88);
4925 __m512 u3 = _mm512_shuffle_ps(t1, t3, 0xdd);
4926
4927 __m128 res0 = _mm512_extractf32x4_ps(u0, 0);
4928 __m128 res4 = _mm512_extractf32x4_ps(u0, 1);
4929 __m128 res8 = _mm512_extractf32x4_ps(u0, 2);
4930 __m128 res12 = _mm512_extractf32x4_ps(u0, 3);
4931 __m128 res1 = _mm512_extractf32x4_ps(u1, 0);
4932 __m128 res5 = _mm512_extractf32x4_ps(u1, 1);
4933 __m128 res9 = _mm512_extractf32x4_ps(u1, 2);
4934 __m128 res13 = _mm512_extractf32x4_ps(u1, 3);
4935 __m128 res2 = _mm512_extractf32x4_ps(u2, 0);
4936 __m128 res6 = _mm512_extractf32x4_ps(u2, 1);
4937 __m128 res10 = _mm512_extractf32x4_ps(u2, 2);
4938 __m128 res14 = _mm512_extractf32x4_ps(u2, 3);
4939 __m128 res3 = _mm512_extractf32x4_ps(u3, 0);
4940 __m128 res7 = _mm512_extractf32x4_ps(u3, 1);
4941 __m128 res11 = _mm512_extractf32x4_ps(u3, 2);
4942 __m128 res15 = _mm512_extractf32x4_ps(u3, 3);
4943
4944 // Cannot use the same store instructions in both paths of the 'if'
4945 // because the compiler cannot know that there is no aliasing between
4946 // pointers
4947 if (add_into)
4948 {
4949 res0 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[0]), res0);
4950 _mm_storeu_ps(out + 4 * i + offsets[0], res0);
4951 res1 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[1]), res1);
4952 _mm_storeu_ps(out + 4 * i + offsets[1], res1);
4953 res2 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[2]), res2);
4954 _mm_storeu_ps(out + 4 * i + offsets[2], res2);
4955 res3 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[3]), res3);
4956 _mm_storeu_ps(out + 4 * i + offsets[3], res3);
4957 res4 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[4]), res4);
4958 _mm_storeu_ps(out + 4 * i + offsets[4], res4);
4959 res5 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[5]), res5);
4960 _mm_storeu_ps(out + 4 * i + offsets[5], res5);
4961 res6 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[6]), res6);
4962 _mm_storeu_ps(out + 4 * i + offsets[6], res6);
4963 res7 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[7]), res7);
4964 _mm_storeu_ps(out + 4 * i + offsets[7], res7);
4965 res8 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[8]), res8);
4966 _mm_storeu_ps(out + 4 * i + offsets[8], res8);
4967 res9 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[9]), res9);
4968 _mm_storeu_ps(out + 4 * i + offsets[9], res9);
4969 res10 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[10]), res10);
4970 _mm_storeu_ps(out + 4 * i + offsets[10], res10);
4971 res11 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[11]), res11);
4972 _mm_storeu_ps(out + 4 * i + offsets[11], res11);
4973 res12 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[12]), res12);
4974 _mm_storeu_ps(out + 4 * i + offsets[12], res12);
4975 res13 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[13]), res13);
4976 _mm_storeu_ps(out + 4 * i + offsets[13], res13);
4977 res14 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[14]), res14);
4978 _mm_storeu_ps(out + 4 * i + offsets[14], res14);
4979 res15 = _mm_add_ps(_mm_loadu_ps(out + 4 * i + offsets[15]), res15);
4980 _mm_storeu_ps(out + 4 * i + offsets[15], res15);
4981 }
4982 else
4983 {
4984 _mm_storeu_ps(out + 4 * i + offsets[0], res0);
4985 _mm_storeu_ps(out + 4 * i + offsets[1], res1);
4986 _mm_storeu_ps(out + 4 * i + offsets[2], res2);
4987 _mm_storeu_ps(out + 4 * i + offsets[3], res3);
4988 _mm_storeu_ps(out + 4 * i + offsets[4], res4);
4989 _mm_storeu_ps(out + 4 * i + offsets[5], res5);
4990 _mm_storeu_ps(out + 4 * i + offsets[6], res6);
4991 _mm_storeu_ps(out + 4 * i + offsets[7], res7);
4992 _mm_storeu_ps(out + 4 * i + offsets[8], res8);
4993 _mm_storeu_ps(out + 4 * i + offsets[9], res9);
4994 _mm_storeu_ps(out + 4 * i + offsets[10], res10);
4995 _mm_storeu_ps(out + 4 * i + offsets[11], res11);
4996 _mm_storeu_ps(out + 4 * i + offsets[12], res12);
4997 _mm_storeu_ps(out + 4 * i + offsets[13], res13);
4998 _mm_storeu_ps(out + 4 * i + offsets[14], res14);
4999 _mm_storeu_ps(out + 4 * i + offsets[15], res15);
5000 }
5001 }
5002
5003 // remainder loop of work that does not divide by 4
5004 if (add_into)
5005 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
5006 for (unsigned int v = 0; v < 16; ++v)
5007 out[offsets[v] + i] += in[i][v];
5008 else
5009 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
5010 for (unsigned int v = 0; v < 16; ++v)
5011 out[offsets[v] + i] = in[i][v];
5012 }
5013
5014
5015
5016 /**
5017 * Specialization for float and AVX-512.
5018 */
5019 template <>
5020 inline DEAL_II_ALWAYS_INLINE void
5021 vectorized_transpose_and_store(const bool add_into,
5022 const unsigned int n_entries,
5023 const VectorizedArray<float, 16> *in,
5024 std::array<float *, 16> &out)
5025 {
5026 // see the comments in the vectorized_transpose_and_store above
5027
5028 const unsigned int n_chunks = n_entries / 4;
5029 for (unsigned int i = 0; i < n_chunks; ++i)
5030 {
5031 __m512 t0 = _mm512_shuffle_ps(in[4 * i].data, in[1 + 4 * i].data, 0x44);
5032 __m512 t1 = _mm512_shuffle_ps(in[4 * i].data, in[1 + 4 * i].data, 0xee);
5033 __m512 t2 =
5034 _mm512_shuffle_ps(in[2 + 4 * i].data, in[3 + 4 * i].data, 0x44);
5035 __m512 t3 =
5036 _mm512_shuffle_ps(in[2 + 4 * i].data, in[3 + 4 * i].data, 0xee);
5037 __m512 u0 = _mm512_shuffle_ps(t0, t2, 0x88);
5038 __m512 u1 = _mm512_shuffle_ps(t0, t2, 0xdd);
5039 __m512 u2 = _mm512_shuffle_ps(t1, t3, 0x88);
5040 __m512 u3 = _mm512_shuffle_ps(t1, t3, 0xdd);
5041
5042 __m128 res0 = _mm512_extractf32x4_ps(u0, 0);
5043 __m128 res4 = _mm512_extractf32x4_ps(u0, 1);
5044 __m128 res8 = _mm512_extractf32x4_ps(u0, 2);
5045 __m128 res12 = _mm512_extractf32x4_ps(u0, 3);
5046 __m128 res1 = _mm512_extractf32x4_ps(u1, 0);
5047 __m128 res5 = _mm512_extractf32x4_ps(u1, 1);
5048 __m128 res9 = _mm512_extractf32x4_ps(u1, 2);
5049 __m128 res13 = _mm512_extractf32x4_ps(u1, 3);
5050 __m128 res2 = _mm512_extractf32x4_ps(u2, 0);
5051 __m128 res6 = _mm512_extractf32x4_ps(u2, 1);
5052 __m128 res10 = _mm512_extractf32x4_ps(u2, 2);
5053 __m128 res14 = _mm512_extractf32x4_ps(u2, 3);
5054 __m128 res3 = _mm512_extractf32x4_ps(u3, 0);
5055 __m128 res7 = _mm512_extractf32x4_ps(u3, 1);
5056 __m128 res11 = _mm512_extractf32x4_ps(u3, 2);
5057 __m128 res15 = _mm512_extractf32x4_ps(u3, 3);
5058
5059 if (add_into)
5060 {
5061 res0 = _mm_add_ps(_mm_loadu_ps(out[0] + 4 * i), res0);
5062 _mm_storeu_ps(out[0] + 4 * i, res0);
5063 res1 = _mm_add_ps(_mm_loadu_ps(out[1] + 4 * i), res1);
5064 _mm_storeu_ps(out[1] + 4 * i, res1);
5065 res2 = _mm_add_ps(_mm_loadu_ps(out[2] + 4 * i), res2);
5066 _mm_storeu_ps(out[2] + 4 * i, res2);
5067 res3 = _mm_add_ps(_mm_loadu_ps(out[3] + 4 * i), res3);
5068 _mm_storeu_ps(out[3] + 4 * i, res3);
5069 res4 = _mm_add_ps(_mm_loadu_ps(out[4] + 4 * i), res4);
5070 _mm_storeu_ps(out[4] + 4 * i, res4);
5071 res5 = _mm_add_ps(_mm_loadu_ps(out[5] + 4 * i), res5);
5072 _mm_storeu_ps(out[5] + 4 * i, res5);
5073 res6 = _mm_add_ps(_mm_loadu_ps(out[6] + 4 * i), res6);
5074 _mm_storeu_ps(out[6] + 4 * i, res6);
5075 res7 = _mm_add_ps(_mm_loadu_ps(out[7] + 4 * i), res7);
5076 _mm_storeu_ps(out[7] + 4 * i, res7);
5077 res8 = _mm_add_ps(_mm_loadu_ps(out[8] + 4 * i), res8);
5078 _mm_storeu_ps(out[8] + 4 * i, res8);
5079 res9 = _mm_add_ps(_mm_loadu_ps(out[9] + 4 * i), res9);
5080 _mm_storeu_ps(out[9] + 4 * i, res9);
5081 res10 = _mm_add_ps(_mm_loadu_ps(out[10] + 4 * i), res10);
5082 _mm_storeu_ps(out[10] + 4 * i, res10);
5083 res11 = _mm_add_ps(_mm_loadu_ps(out[11] + 4 * i), res11);
5084 _mm_storeu_ps(out[11] + 4 * i, res11);
5085 res12 = _mm_add_ps(_mm_loadu_ps(out[12] + 4 * i), res12);
5086 _mm_storeu_ps(out[12] + 4 * i, res12);
5087 res13 = _mm_add_ps(_mm_loadu_ps(out[13] + 4 * i), res13);
5088 _mm_storeu_ps(out[13] + 4 * i, res13);
5089 res14 = _mm_add_ps(_mm_loadu_ps(out[14] + 4 * i), res14);
5090 _mm_storeu_ps(out[14] + 4 * i, res14);
5091 res15 = _mm_add_ps(_mm_loadu_ps(out[15] + 4 * i), res15);
5092 _mm_storeu_ps(out[15] + 4 * i, res15);
5093 }
5094 else
5095 {
5096 _mm_storeu_ps(out[0] + 4 * i, res0);
5097 _mm_storeu_ps(out[1] + 4 * i, res1);
5098 _mm_storeu_ps(out[2] + 4 * i, res2);
5099 _mm_storeu_ps(out[3] + 4 * i, res3);
5100 _mm_storeu_ps(out[4] + 4 * i, res4);
5101 _mm_storeu_ps(out[5] + 4 * i, res5);
5102 _mm_storeu_ps(out[6] + 4 * i, res6);
5103 _mm_storeu_ps(out[7] + 4 * i, res7);
5104 _mm_storeu_ps(out[8] + 4 * i, res8);
5105 _mm_storeu_ps(out[9] + 4 * i, res9);
5106 _mm_storeu_ps(out[10] + 4 * i, res10);
5107 _mm_storeu_ps(out[11] + 4 * i, res11);
5108 _mm_storeu_ps(out[12] + 4 * i, res12);
5109 _mm_storeu_ps(out[13] + 4 * i, res13);
5110 _mm_storeu_ps(out[14] + 4 * i, res14);
5111 _mm_storeu_ps(out[15] + 4 * i, res15);
5112 }
5113 }
5114
5115 if (add_into)
5116 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
5117 for (unsigned int v = 0; v < 16; ++v)
5118 out[v][i] += in[i][v];
5119 else
5120 for (unsigned int i = 4 * n_chunks; i < n_entries; ++i)
5121 for (unsigned int v = 0; v < 16; ++v)
5122 out[v][i] = in[i][v];
5123 }
5124
5125 # endif
5126
5127 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 128 && defined(__ALTIVEC__) && \
5128 defined(__VSX__)
5129
5130 template <>
5131 class VectorizedArray<double, 2>
5132 : public VectorizedArrayBase<VectorizedArray<double, 2>, 2>
5133 {
5134 public:
5135 /**
5136 * This gives the type of the array elements.
5137 */
5138 using value_type = double;
5139
5140 /**
5141 * Record the fact that the given specialization of VectorizedArray is
5142 * indeed implemented.
5143 */
5144 static constexpr bool is_implemented = true;
5145
5146 /**
5147 * Default empty constructor, leaving the data in an uninitialized state
5148 * similar to float/double.
5149 */
5150 VectorizedArray() = default;
5151
5152 /**
5153 * Construct an array with the given scalar broadcast to all lanes.
5154 */
5155 VectorizedArray(const double scalar)
5156 {
5157 this->operator=(scalar);
5158 }
5159
5160 /**
5161 * Construct an array with the given initializer list.
5162 */
5163 template <typename U>
5164 VectorizedArray(const std::initializer_list<U> &list)
5165 : VectorizedArrayBase<VectorizedArray<double, 2>, 2>(list)
5166 {}
5167
5168 /**
5169 * This function assigns a scalar to the current object.
5170 */
5171 DEAL_II_ALWAYS_INLINE
5172 VectorizedArray &
5173 operator=(const double x) &
5174 {
5175 data = vec_splats(x);
5176
5177 // Some compilers believe that vec_splats sets 'x', but that's not true.
5178 // They then warn about setting a variable and not using it. Suppress the
5179 // warning by "using" the variable:
5180 (void)x;
5181 return *this;
5182 }
5183
5184 /**
5185 * Assign a scalar to the current object. This overload is used for
5186 * rvalue references; because it does not make sense to assign
5187 * something to a temporary, the function is deleted.
5188 */
5189 VectorizedArray &
5190 operator=(const double scalar) && = delete;
5191
5192 /**
5193 * Access operator. The component must be either 0 or 1.
5194 */
5195 DEAL_II_ALWAYS_INLINE
5196 double &
5197 operator[](const unsigned int comp)
5198 {
5199 AssertIndexRange(comp, 2);
5200 return *(reinterpret_cast<double *>(&data) + comp);
5201 }
5202
5203 /**
5204 * Constant access operator.
5205 */
5206 DEAL_II_ALWAYS_INLINE
5207 const double &
5208 operator[](const unsigned int comp) const
5209 {
5210 AssertIndexRange(comp, 2);
5211 return *(reinterpret_cast<const double *>(&data) + comp);
5212 }
5213
5214 /**
5215 * Element-wise addition of two arrays of numbers.
5216 */
5217 DEAL_II_ALWAYS_INLINE
5218 VectorizedArray &
5219 operator+=(const VectorizedArray &vec)
5220 {
5221 data = vec_add(data, vec.data);
5222 return *this;
5223 }
5224
5225 /**
5226 * Element-wise subtraction of two arrays of numbers.
5227 */
5228 DEAL_II_ALWAYS_INLINE
5229 VectorizedArray &
5230 operator-=(const VectorizedArray &vec)
5231 {
5232 data = vec_sub(data, vec.data);
5233 return *this;
5234 }
5235
5236 /**
5237 * Element-wise multiplication of two arrays of numbers.
5238 */
5239 DEAL_II_ALWAYS_INLINE
5240 VectorizedArray &
5241 operator*=(const VectorizedArray &vec)
5242 {
5243 data = vec_mul(data, vec.data);
5244 return *this;
5245 }
5246
5247 /**
5248 * Element-wise division of two arrays of numbers.
5249 */
5250 DEAL_II_ALWAYS_INLINE
5251 VectorizedArray &
5252 operator/=(const VectorizedArray &vec)
5253 {
5254 data = vec_div(data, vec.data);
5255 return *this;
5256 }
5257
5258 /**
5259 * Load @p size() from memory into the calling class, starting at
5260 * the given address.
5261 */
5262 DEAL_II_ALWAYS_INLINE
5263 void
5264 load(const double *ptr)
5265 {
5266 data = vec_vsx_ld(0, ptr);
5267 }
5268
5269 /**
5270 * Write the content of the calling class into memory in form of @p
5271 * size() to the given address.
5272 */
5273 DEAL_II_ALWAYS_INLINE
5274 void
5275 store(double *ptr) const
5276 {
5277 vec_vsx_st(data, 0, ptr);
5278 }
5279
5280 /**
5281 * @copydoc VectorizedArray<Number>::streaming_store()
5282 */
5283 DEAL_II_ALWAYS_INLINE
5284 void
5285 streaming_store(double *ptr) const
5286 {
5287 store(ptr);
5288 }
5289
5290 /**
5291 * @copydoc VectorizedArray<Number>::gather()
5292 */
5293 DEAL_II_ALWAYS_INLINE
5294 void
5295 gather(const double *base_ptr, const unsigned int *offsets)
5296 {
5297 for (unsigned int i = 0; i < 2; ++i)
5298 *(reinterpret_cast<double *>(&data) + i) = base_ptr[offsets[i]];
5299 }
5300
5301 /**
5302 * @copydoc VectorizedArray<Number>::scatter
5303 */
5304 DEAL_II_ALWAYS_INLINE
5305 void
5306 scatter(const unsigned int *offsets, double *base_ptr) const
5307 {
5308 for (unsigned int i = 0; i < 2; ++i)
5309 base_ptr[offsets[i]] = *(reinterpret_cast<const double *>(&data) + i);
5310 }
5311
5312 /**
5313 * Actual data field. To be consistent with the standard layout type and to
5314 * enable interaction with external SIMD functionality, this member is
5315 * declared public.
5316 */
5317 __vector double data;
5318
5319 private:
5320 /**
5321 * Return the square root of this field. Not for use in user code. Use
5322 * sqrt(x) instead.
5323 */
5324 DEAL_II_ALWAYS_INLINE
5325 VectorizedArray
5326 get_sqrt() const
5327 {
5328 VectorizedArray res;
5329 res.data = vec_sqrt(data);
5330 return res;
5331 }
5332
5333 /**
5334 * Return the absolute value of this field. Not for use in user code. Use
5335 * abs(x) instead.
5336 */
5337 DEAL_II_ALWAYS_INLINE
5338 VectorizedArray
5339 get_abs() const
5340 {
5341 VectorizedArray res;
5342 res.data = vec_abs(data);
5343 return res;
5344 }
5345
5346 /**
5347 * Return the component-wise maximum of this field and another one. Not for
5348 * use in user code. Use max(x,y) instead.
5349 */
5350 DEAL_II_ALWAYS_INLINE
5351 VectorizedArray
5352 get_max(const VectorizedArray &other) const
5353 {
5354 VectorizedArray res;
5355 res.data = vec_max(data, other.data);
5356 return res;
5357 }
5358
5359 /**
5360 * Return the component-wise minimum of this field and another one. Not for
5361 * use in user code. Use min(x,y) instead.
5362 */
5363 DEAL_II_ALWAYS_INLINE
5364 VectorizedArray
5365 get_min(const VectorizedArray &other) const
5366 {
5367 VectorizedArray res;
5368 res.data = vec_min(data, other.data);
5369 return res;
5370 }
5371
5372 // Make a few functions friends.
5373 template <typename Number2, std::size_t width2>
5374 friend VectorizedArray<Number2, width2>
5375 std::sqrt(const VectorizedArray<Number2, width2> &);
5376 template <typename Number2, std::size_t width2>
5377 friend VectorizedArray<Number2, width2>
5378 std::abs(const VectorizedArray<Number2, width2> &);
5379 template <typename Number2, std::size_t width2>
5380 friend VectorizedArray<Number2, width2>
5381 std::max(const VectorizedArray<Number2, width2> &,
5382 const VectorizedArray<Number2, width2> &);
5383 template <typename Number2, std::size_t width2>
5384 friend VectorizedArray<Number2, width2>
5385 std::min(const VectorizedArray<Number2, width2> &,
5386 const VectorizedArray<Number2, width2> &);
5387 };
5388
5389
5390
5391 template <>
5392 class VectorizedArray<float, 4>
5393 : public VectorizedArrayBase<VectorizedArray<float, 4>, 4>
5394 {
5395 public:
5396 /**
5397 * This gives the type of the array elements.
5398 */
5399 using value_type = float;
5400
5401 /**
5402 * Record the fact that the given specialization of VectorizedArray is
5403 * indeed implemented.
5404 */
5405 static constexpr bool is_implemented = true;
5406
5407 /**
5408 * Default empty constructor, leaving the data in an uninitialized state
5409 * similar to float/double.
5410 */
5411 VectorizedArray() = default;
5412
5413 /**
5414 * Construct an array with the given scalar broadcast to all lanes.
5415 */
5416 VectorizedArray(const float scalar)
5417 {
5418 this->operator=(scalar);
5419 }
5420
5421 /**
5422 * Construct an array with the given initializer list.
5423 */
5424 template <typename U>
5425 VectorizedArray(const std::initializer_list<U> &list)
5426 : VectorizedArrayBase<VectorizedArray<float, 4>, 4>(list)
5427 {}
5428
5429 /**
5430 * This function assigns a scalar to the current object.
5431 */
5432 DEAL_II_ALWAYS_INLINE
5433 VectorizedArray &
5434 operator=(const float x) &
5435 {
5436 data = vec_splats(x);
5437
5438 // Some compilers believe that vec_splats sets 'x', but that's not true.
5439 // They then warn about setting a variable and not using it. Suppress the
5440 // warning by "using" the variable:
5441 (void)x;
5442 return *this;
5443 }
5444
5445 /**
5446 * Assign a scalar to the current object. This overload is used for
5447 * rvalue references; because it does not make sense to assign
5448 * something to a temporary, the function is deleted.
5449 */
5450 VectorizedArray &
5451 operator=(const float scalar) && = delete;
5452
5453 /**
5454 * Access operator. The component must be between 0 and 3.
5455 */
5456 DEAL_II_ALWAYS_INLINE
5457 float &
5458 operator[](const unsigned int comp)
5459 {
5460 AssertIndexRange(comp, 4);
5461 return *(reinterpret_cast<float *>(&data) + comp);
5462 }
5463
5464 /**
5465 * Constant access operator.
5466 */
5467 DEAL_II_ALWAYS_INLINE
5468 const float &
5469 operator[](const unsigned int comp) const
5470 {
5471 AssertIndexRange(comp, 4);
5472 return *(reinterpret_cast<const float *>(&data) + comp);
5473 }
5474
5475 /**
5476 * Element-wise addition of two arrays of numbers.
5477 */
5478 DEAL_II_ALWAYS_INLINE
5479 VectorizedArray &
5480 operator+=(const VectorizedArray &vec)
5481 {
5482 data = vec_add(data, vec.data);
5483 return *this;
5484 }
5485
5486 /**
5487 * Element-wise subtraction of two arrays of numbers.
5488 */
5489 DEAL_II_ALWAYS_INLINE
5490 VectorizedArray &
5491 operator-=(const VectorizedArray &vec)
5492 {
5493 data = vec_sub(data, vec.data);
5494 return *this;
5495 }
5496
5497 /**
5498 * Element-wise multiplication of two arrays of numbers.
5499 */
5500 DEAL_II_ALWAYS_INLINE
5501 VectorizedArray &
5502 operator*=(const VectorizedArray &vec)
5503 {
5504 data = vec_mul(data, vec.data);
5505 return *this;
5506 }
5507
5508 /**
5509 * Element-wise division of two arrays of numbers.
5510 */
5511 DEAL_II_ALWAYS_INLINE
5512 VectorizedArray &
5513 operator/=(const VectorizedArray &vec)
5514 {
5515 data = vec_div(data, vec.data);
5516 return *this;
5517 }
5518
5519 /**
5520 * Load @p size() from memory into the calling class, starting at
5521 * the given address.
5522 */
5523 DEAL_II_ALWAYS_INLINE
5524 void
5525 load(const float *ptr)
5526 {
5527 data = vec_vsx_ld(0, ptr);
5528 }
5529
5530 /**
5531 * Write the content of the calling class into memory in form of @p
5532 * size() to the given address.
5533 */
5534 DEAL_II_ALWAYS_INLINE
5535 void
5536 store(float *ptr) const
5537 {
5538 vec_vsx_st(data, 0, ptr);
5539 }
5540
5541 /**
5542 * @copydoc VectorizedArray<Number>::streaming_store()
5543 */
5544 DEAL_II_ALWAYS_INLINE
5545 void
5546 streaming_store(float *ptr) const
5547 {
5548 store(ptr);
5549 }
5550
5551 /**
5552 * @copydoc VectorizedArray<Number>::gather()
5553 */
5554 DEAL_II_ALWAYS_INLINE
5555 void
5556 gather(const float *base_ptr, const unsigned int *offsets)
5557 {
5558 for (unsigned int i = 0; i < 4; ++i)
5559 *(reinterpret_cast<float *>(&data) + i) = base_ptr[offsets[i]];
5560 }
5561
5562 /**
5563 * @copydoc VectorizedArray<Number>::scatter
5564 */
5565 DEAL_II_ALWAYS_INLINE
5566 void
5567 scatter(const unsigned int *offsets, float *base_ptr) const
5568 {
5569 for (unsigned int i = 0; i < 4; ++i)
5570 base_ptr[offsets[i]] = *(reinterpret_cast<const float *>(&data) + i);
5571 }
5572
5573 /**
5574 * Actual data field. To be consistent with the standard layout type and to
5575 * enable interaction with external SIMD functionality, this member is
5576 * declared public.
5577 */
5578 __vector float data;
5579
5580 private:
5581 /**
5582 * Return the square root of this field. Not for use in user code. Use
5583 * sqrt(x) instead.
5584 */
5585 DEAL_II_ALWAYS_INLINE
5586 VectorizedArray
5587 get_sqrt() const
5588 {
5589 VectorizedArray res;
5590 res.data = vec_sqrt(data);
5591 return res;
5592 }
5593
5594 /**
5595 * Return the absolute value of this field. Not for use in user code. Use
5596 * abs(x) instead.
5597 */
5598 DEAL_II_ALWAYS_INLINE
5599 VectorizedArray
5600 get_abs() const
5601 {
5602 VectorizedArray res;
5603 res.data = vec_abs(data);
5604 return res;
5605 }
5606
5607 /**
5608 * Return the component-wise maximum of this field and another one. Not for
5609 * use in user code. Use max(x,y) instead.
5610 */
5611 DEAL_II_ALWAYS_INLINE
5612 VectorizedArray
5613 get_max(const VectorizedArray &other) const
5614 {
5615 VectorizedArray res;
5616 res.data = vec_max(data, other.data);
5617 return res;
5618 }
5619
5620 /**
5621 * Return the component-wise minimum of this field and another one. Not for
5622 * use in user code. Use min(x,y) instead.
5623 */
5624 DEAL_II_ALWAYS_INLINE
5625 VectorizedArray
5626 get_min(const VectorizedArray &other) const
5627 {
5628 VectorizedArray res;
5629 res.data = vec_min(data, other.data);
5630 return res;
5631 }
5632
5633 // Make a few functions friends.
5634 template <typename Number2, std::size_t width2>
5635 friend VectorizedArray<Number2, width2>
5636 std::sqrt(const VectorizedArray<Number2, width2> &);
5637 template <typename Number2, std::size_t width2>
5638 friend VectorizedArray<Number2, width2>
5639 std::abs(const VectorizedArray<Number2, width2> &);
5640 template <typename Number2, std::size_t width2>
5641 friend VectorizedArray<Number2, width2>
5642 std::max(const VectorizedArray<Number2, width2> &,
5643 const VectorizedArray<Number2, width2> &);
5644 template <typename Number2, std::size_t width2>
5645 friend VectorizedArray<Number2, width2>
5646 std::min(const VectorizedArray<Number2, width2> &,
5647 const VectorizedArray<Number2, width2> &);
5648 };
5649
5650 # endif // if DEAL_II_VECTORIZATION_LEVEL >=1 && defined(__ALTIVEC__) &&
5651 // defined(__VSX__)
5652
5653
5654 #endif // DOXYGEN
5655
5656
5657
5658 /**
5659 * @name Arithmetic operations with VectorizedArray
5660 * @{
5661 */
5662
5663 /**
5664 * Relational operator == for VectorizedArray
5665 *
5666 * @relatesalso VectorizedArray
5667 */
5668 template <typename Number, std::size_t width>
5669 inline DEAL_II_ALWAYS_INLINE bool
5670 operator==(const VectorizedArray<Number, width> &lhs,
5671 const VectorizedArray<Number, width> &rhs)
5672 {
5673 for (unsigned int i = 0; i < VectorizedArray<Number, width>::size(); ++i)
5674 if (lhs[i] != rhs[i])
5675 return false;
5676
5677 return true;
5678 }
5679
5680
5681 /**
5682 * Addition of two vectorized arrays with operator +.
5683 *
5684 * @relatesalso VectorizedArray
5685 */
5686 template <typename Number, std::size_t width>
5687 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5688 operator+(const VectorizedArray<Number, width> &u,
5689 const VectorizedArray<Number, width> &v)
5690 {
5691 VectorizedArray<Number, width> tmp = u;
5692 return tmp += v;
5693 }
5694
5695 /**
5696 * Subtraction of two vectorized arrays with operator -.
5697 *
5698 * @relatesalso VectorizedArray
5699 */
5700 template <typename Number, std::size_t width>
5701 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5702 operator-(const VectorizedArray<Number, width> &u,
5703 const VectorizedArray<Number, width> &v)
5704 {
5705 VectorizedArray<Number, width> tmp = u;
5706 return tmp -= v;
5707 }
5708
5709 /**
5710 * Multiplication of two vectorized arrays with operator *.
5711 *
5712 * @relatesalso VectorizedArray
5713 */
5714 template <typename Number, std::size_t width>
5715 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5716 operator*(const VectorizedArray<Number, width> &u,
5717 const VectorizedArray<Number, width> &v)
5718 {
5719 VectorizedArray<Number, width> tmp = u;
5720 return tmp *= v;
5721 }
5722
5723 /**
5724 * Division of two vectorized arrays with operator /.
5725 *
5726 * @relatesalso VectorizedArray
5727 */
5728 template <typename Number, std::size_t width>
5729 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5730 operator/(const VectorizedArray<Number, width> &u,
5731 const VectorizedArray<Number, width> &v)
5732 {
5733 VectorizedArray<Number, width> tmp = u;
5734 return tmp /= v;
5735 }
5736
5737 /**
5738 * Addition of a scalar (expanded to a vectorized array with @p
5739 * size() equal entries) and a vectorized array.
5740 *
5741 * @relatesalso VectorizedArray
5742 */
5743 template <typename Number, std::size_t width>
5744 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5745 operator+(const Number &u, const VectorizedArray<Number, width> &v)
5746 {
5747 VectorizedArray<Number, width> tmp = u;
5748 return tmp += v;
5749 }
5750
5751 /**
5752 * Addition of a scalar (expanded to a vectorized array with @p
5753 * size() equal entries) and a vectorized array in case the scalar
5754 * is a double (needed in order to be able to write simple code with constants
5755 * that are usually double numbers).
5756 *
5757 * @relatesalso VectorizedArray
5758 */
5759 template <std::size_t width>
5760 inline DEAL_II_ALWAYS_INLINE VectorizedArray<float, width>
5761 operator+(const double u, const VectorizedArray<float, width> &v)
5762 {
5763 VectorizedArray<float, width> tmp = u;
5764 return tmp += v;
5765 }
5766
5767 /**
5768 * Addition of a vectorized array and a scalar (expanded to a vectorized array
5769 * with @p size() equal entries).
5770 *
5771 * @relatesalso VectorizedArray
5772 */
5773 template <typename Number, std::size_t width>
5774 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5775 operator+(const VectorizedArray<Number, width> &v, const Number &u)
5776 {
5777 return u + v;
5778 }
5779
5780 /**
5781 * Addition of a vectorized array and a scalar (expanded to a vectorized array
5782 * with @p size() equal entries) in case the scalar is a double
5783 * (needed in order to be able to write simple code with constants that are
5784 * usually double numbers).
5785 *
5786 * @relatesalso VectorizedArray
5787 */
5788 template <std::size_t width>
5789 inline DEAL_II_ALWAYS_INLINE VectorizedArray<float, width>
5790 operator+(const VectorizedArray<float, width> &v, const double u)
5791 {
5792 return u + v;
5793 }
5794
5795 /**
5796 * Subtraction of a vectorized array from a scalar (expanded to a vectorized
5797 * array with @p size() equal entries).
5798 *
5799 * @relatesalso VectorizedArray
5800 */
5801 template <typename Number, std::size_t width>
5802 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5803 operator-(const Number &u, const VectorizedArray<Number, width> &v)
5804 {
5805 VectorizedArray<Number, width> tmp = u;
5806 return tmp -= v;
5807 }
5808
5809 /**
5810 * Subtraction of a vectorized array from a scalar (expanded to a vectorized
5811 * array with @p size() equal entries) in case the scalar is a
5812 * double (needed in order to be able to write simple code with constants that
5813 * are usually double numbers).
5814 *
5815 * @relatesalso VectorizedArray
5816 */
5817 template <std::size_t width>
5818 inline DEAL_II_ALWAYS_INLINE VectorizedArray<float, width>
5819 operator-(const double u, const VectorizedArray<float, width> &v)
5820 {
5821 VectorizedArray<float, width> tmp = static_cast<float>(u);
5822 return tmp -= v;
5823 }
5824
5825 /**
5826 * Subtraction of a scalar (expanded to a vectorized array with @p
5827 * size() equal entries) from a vectorized array.
5828 *
5829 * @relatesalso VectorizedArray
5830 */
5831 template <typename Number, std::size_t width>
5832 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5833 operator-(const VectorizedArray<Number, width> &v, const Number &u)
5834 {
5835 VectorizedArray<Number, width> tmp = u;
5836 return v - tmp;
5837 }
5838
5839 /**
5840 * Subtraction of a scalar (expanded to a vectorized array with @p
5841 * size() equal entries) from a vectorized array in case the scalar
5842 * is a double (needed in order to be able to write simple code with constants
5843 * that are usually double numbers).
5844 *
5845 * @relatesalso VectorizedArray
5846 */
5847 template <std::size_t width>
5848 inline DEAL_II_ALWAYS_INLINE VectorizedArray<float, width>
5849 operator-(const VectorizedArray<float, width> &v, const double u)
5850 {
5851 VectorizedArray<float, width> tmp = static_cast<float>(u);
5852 return v - tmp;
5853 }
5854
5855 /**
5856 * Multiplication of a scalar (expanded to a vectorized array with @p
5857 * size() equal entries) and a vectorized array.
5858 *
5859 * @relatesalso VectorizedArray
5860 */
5861 template <typename Number, std::size_t width>
5862 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5863 operator*(const Number &u, const VectorizedArray<Number, width> &v)
5864 {
5865 VectorizedArray<Number, width> tmp = u;
5866 return tmp *= v;
5867 }
5868
5869 /**
5870 * Multiplication of a scalar (expanded to a vectorized array with @p
5871 * size() equal entries) and a vectorized array in case the scalar
5872 * is a double (needed in order to be able to write simple code with constants
5873 * that are usually double numbers).
5874 *
5875 * @relatesalso VectorizedArray
5876 */
5877 template <std::size_t width>
5878 inline DEAL_II_ALWAYS_INLINE VectorizedArray<float, width>
5879 operator*(const double u, const VectorizedArray<float, width> &v)
5880 {
5881 VectorizedArray<float, width> tmp = static_cast<float>(u);
5882 return tmp *= v;
5883 }
5884
5885 /**
5886 * Multiplication of a vectorized array and a scalar (expanded to a vectorized
5887 * array with @p size() equal entries).
5888 *
5889 * @relatesalso VectorizedArray
5890 */
5891 template <typename Number, std::size_t width>
5892 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5893 operator*(const VectorizedArray<Number, width> &v, const Number &u)
5894 {
5895 return u * v;
5896 }
5897
5898 /**
5899 * Multiplication of a vectorized array and a scalar (expanded to a vectorized
5900 * array with @p size() equal entries) in case the scalar is a
5901 * double (needed in order to be able to write simple code with constants that
5902 * are usually double numbers).
5903 *
5904 * @relatesalso VectorizedArray
5905 */
5906 template <std::size_t width>
5907 inline DEAL_II_ALWAYS_INLINE VectorizedArray<float, width>
5908 operator*(const VectorizedArray<float, width> &v, const double u)
5909 {
5910 return u * v;
5911 }
5912
5913 /**
5914 * Quotient between a scalar (expanded to a vectorized array with @p
5915 * size() equal entries) and a vectorized array.
5916 *
5917 * @relatesalso VectorizedArray
5918 */
5919 template <typename Number, std::size_t width>
5920 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5921 operator/(const Number &u, const VectorizedArray<Number, width> &v)
5922 {
5923 VectorizedArray<Number, width> tmp = u;
5924 return tmp /= v;
5925 }
5926
5927 /**
5928 * Quotient between a scalar (expanded to a vectorized array with @p
5929 * size() equal entries) and a vectorized array in case the scalar
5930 * is a double (needed in order to be able to write simple code with constants
5931 * that are usually double numbers).
5932 *
5933 * @relatesalso VectorizedArray
5934 */
5935 template <std::size_t width>
5936 inline DEAL_II_ALWAYS_INLINE VectorizedArray<float, width>
5937 operator/(const double u, const VectorizedArray<float, width> &v)
5938 {
5939 VectorizedArray<float, width> tmp = static_cast<float>(u);
5940 return tmp /= v;
5941 }
5942
5943 /**
5944 * Quotient between a vectorized array and a scalar (expanded to a vectorized
5945 * array with @p size() equal entries).
5946 *
5947 * @relatesalso VectorizedArray
5948 */
5949 template <typename Number, std::size_t width>
5950 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5951 operator/(const VectorizedArray<Number, width> &v, const Number &u)
5952 {
5953 VectorizedArray<Number, width> tmp = u;
5954 return v / tmp;
5955 }
5956
5957 /**
5958 * Quotient between a vectorized array and a scalar (expanded to a vectorized
5959 * array with @p size() equal entries) in case the scalar is a
5960 * double (needed in order to be able to write simple code with constants that
5961 * are usually double numbers).
5962 *
5963 * @relatesalso VectorizedArray
5964 */
5965 template <std::size_t width>
5966 inline DEAL_II_ALWAYS_INLINE VectorizedArray<float, width>
5967 operator/(const VectorizedArray<float, width> &v, const double u)
5968 {
5969 VectorizedArray<float, width> tmp = static_cast<float>(u);
5970 return v / tmp;
5971 }
5972
5973 /**
5974 * Unary operator + on a vectorized array.
5975 *
5976 * @relatesalso VectorizedArray
5977 */
5978 template <typename Number, std::size_t width>
5979 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5980 operator+(const VectorizedArray<Number, width> &u)
5981 {
5982 return u;
5983 }
5984
5985 /**
5986 * Unary operator - on a vectorized array.
5987 *
5988 * @relatesalso VectorizedArray
5989 */
5990 template <typename Number, std::size_t width>
5991 inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
5992 operator-(const VectorizedArray<Number, width> &u)
5993 {
5994 // to get a negative sign, subtract the input from zero (could also
5995 // multiply by -1, but this one is slightly simpler)
5996 return VectorizedArray<Number, width>() - u;
5997 }
5998
5999 /**
6000 * Output operator for vectorized array.
6001 *
6002 * @relatesalso VectorizedArray
6003 */
6004 template <typename Number, std::size_t width>
6005 inline std::ostream &
6006 operator<<(std::ostream &out, const VectorizedArray<Number, width> &p)
6007 {
6008 constexpr unsigned int n = VectorizedArray<Number, width>::size();
6009 for (unsigned int i = 0; i < n - 1; ++i)
6010 out << p[i] << ' ';
6011 out << p[n - 1];
6012
6013 return out;
6014 }
6015
6016 /** @} */
6017
6018 /**
6019 * @name Ternary operations on VectorizedArray
6020 * @{
6021 */
6022
6023 /**
6024 * enum class encoding binary operations for a component-wise comparison of
6025 * VectorizedArray data types.
6026 *
6027 * @note In case of SIMD vecorization (sse, avx, av512) we select the
6028 * corresponding ordered, non-signalling (<code>OQ</code>) variants.
6029 */
6030 enum class SIMDComparison : int
6031 {
6032 #if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 256 && defined(__AVX__)
6033 equal = _CMP_EQ_OQ,
6034 not_equal = _CMP_NEQ_OQ,
6035 less_than = _CMP_LT_OQ,
6036 less_than_or_equal = _CMP_LE_OQ,
6037 greater_than = _CMP_GT_OQ,
6038 greater_than_or_equal = _CMP_GE_OQ
6039 #else
6040 equal,
6041 not_equal,
6042 less_than,
6043 less_than_or_equal,
6044 greater_than,
6045 greater_than_or_equal
6046 #endif
6047 };
6048
6049
6050 /**
6051 * Computes the vectorized equivalent of the following ternary operation:
6052 * @code
6053 * (left OP right) ? true_value : false_value
6054 * @endcode
6055 * where <code>OP</code> is a binary operator (such as <code>=</code>,
6056 * <code>!=</code>, <code><</code>, <code><=</code>, <code>></code>, and
6057 * <code>>=</code>).
6058 *
6059 * Such a computational idiom is useful as an alternative to branching
6060 * whenever the control flow itself would depend on (computed) data. For
6061 * example, in case of a scalar data type the statement
6062 * <code>(left < right) ? true_value : false_value</code>
6063 * could have been also implemented using an <code>if</code>-statement:
6064 * @code
6065 * if (left < right)
6066 * result = true_value;
6067 * else
6068 * result = false_value;
6069 * @endcode
6070 * This, however, is fundamentally impossible in case of vectorization
6071 * because different decisions will be necessary on different vector entries
6072 * (lanes) and
6073 * the first variant (based on a ternary operator) has to be used instead:
6074 * @code
6075 * result = compare_and_apply_mask<SIMDComparison::less_than>
6076 * (left, right, true_value, false_value);
6077 * @endcode
6078 * Some more illustrative examples (that are less efficient than the
6079 * dedicated <code>std::max</code> and <code>std::abs</code> overloads):
6080 * @code
6081 * VectorizedArray<double> left;
6082 * VectorizedArray<double> right;
6083 *
6084 * // std::max
6085 * const auto maximum = compare_and_apply_mask<SIMDComparison::greater_than>
6086 * (left, right, left, right);
6087 *
6088 * // std::abs
6089 * const auto absolute = compare_and_apply_mask<SIMDComparison::less_than>
6090 * (left, VectorizedArray<double>(0.), -left, left);
6091 * @endcode
6092 *
6093 * More precisely, this function first computes a (boolean) mask that is
6094 * the result of a binary operator <code>OP</code> applied to all elements
6095 * of the VectorizedArray arguments @p left and @p right. The mask is then
6096 * used to either select the corresponding component of @p true_value (if
6097 * the binary operation equates to true), or @p false_value. The binary
6098 * operator is encoded via the SIMDComparison template argument
6099 * @p predicate.
6100 *
6101 * In order to ease with generic programming approaches, the function
6102 * provides overloads for all VectorizedArray<Number> variants as well as
6103 * generic POD types such as double and float.
6104 *
6105 * @note For this function to work the binary operation has to be encoded
6106 * via a SIMDComparison template argument @p predicate. Depending on it
6107 * appropriate low-level machine instructions are generated replacing the
6108 * call to compare_and_apply_mask. This also explains why @p predicate is a
6109 * compile-time constant template parameter and not a constant function
6110 * argument. In order to be able to emit the correct low-level instruction,
6111 * the compiler has to know the comparison at compile time.
6112 */
6113 template <SIMDComparison predicate, typename Number>
6114 DEAL_II_ALWAYS_INLINE inline Number
6115 compare_and_apply_mask(const Number &left,
6116 const Number &right,
6117 const Number &true_value,
6118 const Number &false_value)
6119 {
6120 bool mask;
6121 switch (predicate)
6122 {
6123 case SIMDComparison::equal:
6124 mask = (left == right);
6125 break;
6126 case SIMDComparison::not_equal:
6127 mask = (left != right);
6128 break;
6129 case SIMDComparison::less_than:
6130 mask = (left < right);
6131 break;
6132 case SIMDComparison::less_than_or_equal:
6133 mask = (left <= right);
6134 break;
6135 case SIMDComparison::greater_than:
6136 mask = (left > right);
6137 break;
6138 case SIMDComparison::greater_than_or_equal:
6139 mask = (left >= right);
6140 break;
6141 }
6142
6143 return mask ? true_value : false_value;
6144 }
6145
6146
6147 /**
6148 * Specialization of above function for the non-vectorized
6149 * VectorizedArray<Number, 1> variant.
6150 */
6151 template <SIMDComparison predicate, typename Number>
6152 DEAL_II_ALWAYS_INLINE inline VectorizedArray<Number, 1>
6153 compare_and_apply_mask(const VectorizedArray<Number, 1> &left,
6154 const VectorizedArray<Number, 1> &right,
6155 const VectorizedArray<Number, 1> &true_value,
6156 const VectorizedArray<Number, 1> &false_value)
6157 {
6158 VectorizedArray<Number, 1> result;
6159 result.data = compare_and_apply_mask<predicate, Number>(left.data,
6160 right.data,
6161 true_value.data,
6162 false_value.data);
6163 return result;
6164 }
6165
6166 /** @} */
6167
6168 #ifndef DOXYGEN
6169 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 512 && defined(__AVX512F__)
6170
6171 template <SIMDComparison predicate>
6172 DEAL_II_ALWAYS_INLINE inline VectorizedArray<float, 16>
6173 compare_and_apply_mask(const VectorizedArray<float, 16> &left,
6174 const VectorizedArray<float, 16> &right,
6175 const VectorizedArray<float, 16> &true_values,
6176 const VectorizedArray<float, 16> &false_values)
6177 {
6178 const __mmask16 mask =
6179 _mm512_cmp_ps_mask(left.data, right.data, static_cast<int>(predicate));
6180 VectorizedArray<float, 16> result;
6181 result.data = _mm512_mask_mov_ps(false_values.data, mask, true_values.data);
6182 return result;
6183 }
6184
6185
6186
6187 template <SIMDComparison predicate>
6188 DEAL_II_ALWAYS_INLINE inline VectorizedArray<double, 8>
6189 compare_and_apply_mask(const VectorizedArray<double, 8> &left,
6190 const VectorizedArray<double, 8> &right,
6191 const VectorizedArray<double, 8> &true_values,
6192 const VectorizedArray<double, 8> &false_values)
6193 {
6194 const __mmask16 mask =
6195 _mm512_cmp_pd_mask(left.data, right.data, static_cast<int>(predicate));
6196 VectorizedArray<double, 8> result;
6197 result.data = _mm512_mask_mov_pd(false_values.data, mask, true_values.data);
6198 return result;
6199 }
6200
6201 # endif
6202
6203 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 256 && defined(__AVX__)
6204
6205 template <SIMDComparison predicate>
6206 DEAL_II_ALWAYS_INLINE inline VectorizedArray<float, 8>
6207 compare_and_apply_mask(const VectorizedArray<float, 8> &left,
6208 const VectorizedArray<float, 8> &right,
6209 const VectorizedArray<float, 8> &true_values,
6210 const VectorizedArray<float, 8> &false_values)
6211 {
6212 const auto mask =
6213 _mm256_cmp_ps(left.data, right.data, static_cast<int>(predicate));
6214
6215 VectorizedArray<float, 8> result;
6216 result.data = _mm256_blendv_ps(false_values.data, true_values.data, mask);
6217 return result;
6218 }
6219
6220
6221 template <SIMDComparison predicate>
6222 DEAL_II_ALWAYS_INLINE inline VectorizedArray<double, 4>
6223 compare_and_apply_mask(const VectorizedArray<double, 4> &left,
6224 const VectorizedArray<double, 4> &right,
6225 const VectorizedArray<double, 4> &true_values,
6226 const VectorizedArray<double, 4> &false_values)
6227 {
6228 const auto mask =
6229 _mm256_cmp_pd(left.data, right.data, static_cast<int>(predicate));
6230
6231 VectorizedArray<double, 4> result;
6232 result.data = _mm256_blendv_pd(false_values.data, true_values.data, mask);
6233 return result;
6234 }
6235
6236 # endif
6237
6238 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 128 && defined(__SSE2__)
6239
6240 template <SIMDComparison predicate>
6241 DEAL_II_ALWAYS_INLINE inline VectorizedArray<float, 4>
6242 compare_and_apply_mask(const VectorizedArray<float, 4> &left,
6243 const VectorizedArray<float, 4> &right,
6244 const VectorizedArray<float, 4> &true_values,
6245 const VectorizedArray<float, 4> &false_values)
6246 {
6247 __m128 mask;
6248 switch (predicate)
6249 {
6250 case SIMDComparison::equal:
6251 mask = _mm_cmpeq_ps(left.data, right.data);
6252 break;
6253 case SIMDComparison::not_equal:
6254 mask = _mm_cmpneq_ps(left.data, right.data);
6255 break;
6256 case SIMDComparison::less_than:
6257 mask = _mm_cmplt_ps(left.data, right.data);
6258 break;
6259 case SIMDComparison::less_than_or_equal:
6260 mask = _mm_cmple_ps(left.data, right.data);
6261 break;
6262 case SIMDComparison::greater_than:
6263 mask = _mm_cmpgt_ps(left.data, right.data);
6264 break;
6265 case SIMDComparison::greater_than_or_equal:
6266 mask = _mm_cmpge_ps(left.data, right.data);
6267 break;
6268 }
6269
6270 VectorizedArray<float, 4> result;
6271 result.data = _mm_or_ps(_mm_and_ps(mask, true_values.data),
6272 _mm_andnot_ps(mask, false_values.data));
6273
6274 return result;
6275 }
6276
6277
6278 template <SIMDComparison predicate>
6279 DEAL_II_ALWAYS_INLINE inline VectorizedArray<double, 2>
6280 compare_and_apply_mask(const VectorizedArray<double, 2> &left,
6281 const VectorizedArray<double, 2> &right,
6282 const VectorizedArray<double, 2> &true_values,
6283 const VectorizedArray<double, 2> &false_values)
6284 {
6285 __m128d mask;
6286 switch (predicate)
6287 {
6288 case SIMDComparison::equal:
6289 mask = _mm_cmpeq_pd(left.data, right.data);
6290 break;
6291 case SIMDComparison::not_equal:
6292 mask = _mm_cmpneq_pd(left.data, right.data);
6293 break;
6294 case SIMDComparison::less_than:
6295 mask = _mm_cmplt_pd(left.data, right.data);
6296 break;
6297 case SIMDComparison::less_than_or_equal:
6298 mask = _mm_cmple_pd(left.data, right.data);
6299 break;
6300 case SIMDComparison::greater_than:
6301 mask = _mm_cmpgt_pd(left.data, right.data);
6302 break;
6303 case SIMDComparison::greater_than_or_equal:
6304 mask = _mm_cmpge_pd(left.data, right.data);
6305 break;
6306 }
6307
6308 VectorizedArray<double, 2> result;
6309 result.data = _mm_or_pd(_mm_and_pd(mask, true_values.data),
6310 _mm_andnot_pd(mask, false_values.data));
6311
6312 return result;
6313 }
6314
6315 # endif
6316
6317 # if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 128 && defined(__ARM_NEON)
6318
6319 template <SIMDComparison predicate>
6320 DEAL_II_ALWAYS_INLINE inline VectorizedArray<float, 4>
6321 compare_and_apply_mask(const VectorizedArray<float, 4> &left,
6322 const VectorizedArray<float, 4> &right,
6323 const VectorizedArray<float, 4> &true_values,
6324 const VectorizedArray<float, 4> &false_values)
6325 {
6326 uint32x4_t mask;
6327 switch (predicate)
6328 {
6329 case SIMDComparison::equal:
6330 mask = vceqq_f32(left.data, right.data);
6331 break;
6332 case SIMDComparison::not_equal:
6333 mask = vmvnq_u32(vceqq_f32(left.data, right.data));
6334 break;
6335 case SIMDComparison::less_than:
6336 mask = vcltq_f32(left.data, right.data);
6337 break;
6338 case SIMDComparison::less_than_or_equal:
6339 mask = vcleq_f32(left.data, right.data);
6340 break;
6341 case SIMDComparison::greater_than:
6342 mask = vcgtq_f32(left.data, right.data);
6343 break;
6344 case SIMDComparison::greater_than_or_equal:
6345 mask = vcgeq_f32(left.data, right.data);
6346 break;
6347 }
6348
6349 VectorizedArray<float, 4> result;
6350 result.data = vreinterpretq_f32_u32(vorrq_u32(
6351 vandq_u32(mask, vreinterpretq_u32_f32(true_values.data)),
6352 vandq_u32(vmvnq_u32(mask), vreinterpretq_u32_f32(false_values.data))));
6353
6354 return result;
6355 }
6356
6357
6358 template <SIMDComparison predicate>
6359 DEAL_II_ALWAYS_INLINE inline VectorizedArray<double, 2>
6360 compare_and_apply_mask(const VectorizedArray<double, 2> &left,
6361 const VectorizedArray<double, 2> &right,
6362 const VectorizedArray<double, 2> &true_values,
6363 const VectorizedArray<double, 2> &false_values)
6364 {
6365 uint64x2_t mask;
6366 switch (predicate)
6367 {
6368 case SIMDComparison::equal:
6369 mask = vceqq_f64(left.data, right.data);
6370 break;
6371 case SIMDComparison::not_equal:
6372 mask = vreinterpretq_u64_u32(
6373 vmvnq_u32(vreinterpretq_u32_u64(vceqq_f64(left.data, right.data))));
6374 break;
6375 case SIMDComparison::less_than:
6376 mask = vcltq_f64(left.data, right.data);
6377 break;
6378 case SIMDComparison::less_than_or_equal:
6379 mask = vcleq_f64(left.data, right.data);
6380 break;
6381 case SIMDComparison::greater_than:
6382 mask = vcgtq_f64(left.data, right.data);
6383 break;
6384 case SIMDComparison::greater_than_or_equal:
6385 mask = vcgeq_f64(left.data, right.data);
6386 break;
6387 }
6388
6389 VectorizedArray<double, 2> result;
6390 result.data = vreinterpretq_f64_u64(vorrq_u64(
6391 vandq_u64(mask, vreinterpretq_u64_f64(true_values.data)),
6392 vandq_u64(vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(mask))),
6393 vreinterpretq_u64_f64(false_values.data))));
6394
6395 return result;
6396 }
6397
6398 # endif
6399 #endif // DOXYGEN
6400
6401
6402 namespace internal
6403 {
6404 template <typename T>
6405 struct VectorizedArrayTrait
6406 {
6407 /**
6408 * Define scalar value type.
6409 */
6410 using value_type = T;
6411
6412 /**
6413 * Return the width of template type.
6414 */
6415 static constexpr std::size_t
6416 width()
6417 {
6418 return 1;
6419 }
6420
6421 /**
6422 * Define vectorized value type for internal vectorization.
6423 */
6424 using vectorized_value_type = VectorizedArray<T>;
6425
6426 /**
6427 * Return a stride which defines how often the template type T fits into
6428 * the vectorized_value_type. This is useful to write vectorized templated
6429 * code where the internal computation is vectorized and the user
6430 * interface is optionally scalar or also vectorized.
6431 */
6432 static constexpr std::size_t
6433 stride()
6434 {
6435 return vectorized_value_type::size();
6436 }
6437
6438 /**
6439 * Get a reference to scalar value (on lane 0).
6440 */
6441 static value_type &
6442 get(value_type &value, unsigned int c)
6443 {
6444 AssertIndexRange(c, 1);
6445 (void)c;
6446
6447 return value;
6448 }
6449
6450 /**
6451 * Get a read-only reference to scalar value (on lane 0).
6452 */
6453 static const value_type &
6454 get(const value_type &value, unsigned int c)
6455 {
6456 AssertIndexRange(c, 1);
6457 (void)c;
6458
6459 return value;
6460 }
6461
6462 /**
6463 * Get a reference to scalar value on lane c from a vectorized values field.
6464 */
6465 static value_type &
6466 get_from_vectorized(vectorized_value_type &values, unsigned int c)
6467 {
6468 AssertIndexRange(c, stride());
6469
6470 return values[c];
6471 }
6472
6473 /**
6474 * Get a read-only reference to scalar value on lane c from a vectorized
6475 * values field.
6476 */
6477 static const value_type &
6478 get_from_vectorized(const vectorized_value_type &values, unsigned int c)
6479 {
6480 AssertIndexRange(c, stride());
6481
6482 return values[c];
6483 }
6484 };
6485
6486 template <typename T, std::size_t width_>
6487 struct VectorizedArrayTrait<VectorizedArray<T, width_>>
6488 {
6489 /**
6490 * Define scalar value type.
6491 */
6492 using value_type = T;
6493
6494 /**
6495 * Return the width of template type.
6496 */
6497 static constexpr std::size_t
6498 width()
6499 {
6500 return width_;
6501 }
6502
6503 /**
6504 * Define vectorized value type for internal vectorization.
6505 */
6506 using vectorized_value_type = VectorizedArray<T, width_>;
6507
6508 /**
6509 * Return a stride which defines how often the template type
6510 * VectorizedArray<T, width_> fits into the vectorized value type. This is
6511 * useful to write vectorized templated code where the internal computation
6512 * is vectorized and the user interface is optionally scalar or also
6513 * vectorized.
6514 */
6515 static constexpr std::size_t
6516 stride()
6517 {
6518 return 1;
6519 }
6520
6521 /**
6522 * Get a reference to scalar value on lane c.
6523 */
6524 static value_type &
6525 get(vectorized_value_type &values, unsigned int c)
6526 {
6527 AssertIndexRange(c, width_);
6528
6529 return values[c];
6530 }
6531
6532 /**
6533 * Get a read-only reference to scalar value on lane c.
6534 */
6535 static const value_type &
6536 get(const vectorized_value_type &values, unsigned int c)
6537 {
6538 AssertIndexRange(c, width_);
6539
6540 return values[c];
6541 }
6542
6543 /**
6544 * Get a reference to vectorized values from a vectorized values field.
6545 */
6546 static vectorized_value_type &
6547 get_from_vectorized(vectorized_value_type &values, unsigned int c)
6548 {
6549 (void)c;
6550 AssertIndexRange(c, stride());
6551
6552 return values;
6553 }
6554
6555 /**
6556 * Get a read-only reference to vectorized values from a vectorized values
6557 * field.
6558 */
6559 static const vectorized_value_type &
6560 get_from_vectorized(const vectorized_value_type &values, unsigned int c)
6561 {
6562 (void)c;
6563 AssertIndexRange(c, stride());
6564
6565 return values;
6566 }
6567 };
6568 } // namespace internal
6569
6570
6571 DEAL_II_NAMESPACE_CLOSE
6572
6573 /**
6574 * Implementation of functions from cmath on VectorizedArray. These functions
6575 * do not reside in the dealii namespace in order to ensure a similar
6576 * interface as for the respective functions in cmath. Instead, call them
6577 * using std::sin.
6578 */
6579 namespace std
6580 {
6581 /**
6582 * Compute the sine of a vectorized data field. The result is returned as
6583 * vectorized array in the form <tt>{sin(x[0]), sin(x[1]), ...,
6584 * sin(x[VectorizedArray::size()-1])}</tt>.
6585 *
6586 * @relatesalso VectorizedArray
6587 */
6588 template <typename Number, std::size_t width>
6589 inline ::dealii::VectorizedArray<Number, width>
6590 sin(const ::dealii::VectorizedArray<Number, width> &x)
6591 {
6592 ::dealii::VectorizedArray<Number, width> out;
6593 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6594 ++i)
6595 out[i] = std::sin(x[i]);
6596 return out;
6597 }
6598
6599
6600
6601 /**
6602 * Compute the cosine of a vectorized data field. The result is returned as
6603 * vectorized array in the form <tt>{cos(x[0]), cos(x[1]), ...,
6604 * cos(x[size()-1])}</tt>.
6605 *
6606 * @relatesalso VectorizedArray
6607 */
6608 template <typename Number, std::size_t width>
6609 inline ::dealii::VectorizedArray<Number, width>
6610 cos(const ::dealii::VectorizedArray<Number, width> &x)
6611 {
6612 ::dealii::VectorizedArray<Number, width> out;
6613 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6614 ++i)
6615 out[i] = std::cos(x[i]);
6616 return out;
6617 }
6618
6619
6620
6621 /**
6622 * Compute the tangent of a vectorized data field. The result is returned
6623 * as vectorized array in the form <tt>{tan(x[0]), tan(x[1]), ...,
6624 * tan(x[size()-1])}</tt>.
6625 *
6626 * @relatesalso VectorizedArray
6627 */
6628 template <typename Number, std::size_t width>
6629 inline ::dealii::VectorizedArray<Number, width>
6630 tan(const ::dealii::VectorizedArray<Number, width> &x)
6631 {
6632 ::dealii::VectorizedArray<Number, width> out;
6633 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6634 ++i)
6635 out[i] = std::tan(x[i]);
6636 return out;
6637 }
6638
6639
6640
6641 /**
6642 * Compute the arc cosine of a vectorized data field. The result is
6643 * returned as vectorized array in the form <tt>{acos(x[0]), acos(x[1]), ...,
6644 * acos(x[size()-1])}</tt>.
6645 *
6646 * @relatesalso VectorizedArray
6647 */
6648 template <typename Number, std::size_t width>
6649 inline ::dealii::VectorizedArray<Number, width>
6650 acos(const ::dealii::VectorizedArray<Number, width> &x)
6651 {
6652 ::dealii::VectorizedArray<Number, width> out;
6653 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6654 ++i)
6655 out[i] = std::acos(x[i]);
6656 return out;
6657 }
6658
6659
6660
6661 /**
6662 * Compute the arc sine of a vectorized data field. The result is
6663 * returned as vectorized array in the form <tt>{asin(x[0]), asin(x[1]), ...,
6664 * asin(x[size()-1])}</tt>.
6665 *
6666 * @relatesalso VectorizedArray
6667 */
6668 template <typename Number, std::size_t width>
6669 inline ::dealii::VectorizedArray<Number, width>
6670 asin(const ::dealii::VectorizedArray<Number, width> &x)
6671 {
6672 ::dealii::VectorizedArray<Number, width> out;
6673 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6674 ++i)
6675 out[i] = std::asin(x[i]);
6676 return out;
6677 }
6678
6679
6680
6681 /**
6682 * Compute the arc tangent of a vectorized data field. The result is
6683 * returned as vectorized array in the form <tt>{atan(x[0]), atan(x[1]), ...,
6684 * atan(x[size()-1])}</tt>.
6685 *
6686 * @relatesalso VectorizedArray
6687 */
6688 template <typename Number, std::size_t width>
6689 inline ::dealii::VectorizedArray<Number, width>
6690 atan(const ::dealii::VectorizedArray<Number, width> &x)
6691 {
6692 ::dealii::VectorizedArray<Number, width> out;
6693 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6694 ++i)
6695 out[i] = std::atan(x[i]);
6696 return out;
6697 }
6698
6699
6700
6701 /**
6702 * Compute the hyperbolic cosine of a vectorized data field. The result is
6703 * returned as vectorized array in the form <tt>{cosh(x[0]), cosh(x[1]), ...,
6704 * cosh(x[size()-1])}</tt>.
6705 *
6706 * @relatesalso VectorizedArray
6707 */
6708 template <typename Number, std::size_t width>
6709 inline ::dealii::VectorizedArray<Number, width>
6710 cosh(const ::dealii::VectorizedArray<Number, width> &x)
6711 {
6712 ::dealii::VectorizedArray<Number, width> out;
6713 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6714 ++i)
6715 out[i] = std::cosh(x[i]);
6716 return out;
6717 }
6718
6719
6720
6721 /**
6722 * Compute the hyperbolic sine of a vectorized data field. The result is
6723 * returned as vectorized array in the form <tt>{sinh(x[0]), sinh(x[1]), ...,
6724 * sinh(x[size()-1])}</tt>.
6725 *
6726 * @relatesalso VectorizedArray
6727 */
6728 template <typename Number, std::size_t width>
6729 inline ::dealii::VectorizedArray<Number, width>
6730 sinh(const ::dealii::VectorizedArray<Number, width> &x)
6731 {
6732 ::dealii::VectorizedArray<Number, width> out;
6733 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6734 ++i)
6735 out[i] = std::sinh(x[i]);
6736 return out;
6737 }
6738
6739
6740
6741 /**
6742 * Compute the hyperbolic tangent of a vectorized data field. The result is
6743 * returned as vectorized array in the form <tt>{tanh(x[0]), tanh(x[1]), ...,
6744 * tanh(x[size()-1])}</tt>.
6745 *
6746 * @relatesalso VectorizedArray
6747 */
6748 template <typename Number, std::size_t width>
6749 inline ::dealii::VectorizedArray<Number, width>
6750 tanh(const ::dealii::VectorizedArray<Number, width> &x)
6751 {
6752 ::dealii::VectorizedArray<Number, width> out;
6753 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6754 ++i)
6755 out[i] = std::tanh(x[i]);
6756 return out;
6757 }
6758
6759
6760
6761 /**
6762 * Compute the area hyperbolic cosine of a vectorized data field. The result
6763 * is returned as vectorized array in the form <tt>{acosh(x[0]), acosh(x[1]),
6764 * ..., acosh(x[size()-1])}</tt>.
6765 *
6766 * @relatesalso VectorizedArray
6767 */
6768 template <typename Number, std::size_t width>
6769 inline ::dealii::VectorizedArray<Number, width>
6770 acosh(const ::dealii::VectorizedArray<Number, width> &x)
6771 {
6772 ::dealii::VectorizedArray<Number, width> out;
6773 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6774 ++i)
6775 out[i] = std::acosh(x[i]);
6776 return out;
6777 }
6778
6779
6780
6781 /**
6782 * Compute the area hyperbolic sine of a vectorized data field. The result is
6783 * returned as vectorized array in the form <tt>{asinh(x[0]), asinh(x[1]),
6784 * ..., asinh(x[size()-1])}</tt>.
6785 *
6786 * @relatesalso VectorizedArray
6787 */
6788 template <typename Number, std::size_t width>
6789 inline ::dealii::VectorizedArray<Number, width>
6790 asinh(const ::dealii::VectorizedArray<Number, width> &x)
6791 {
6792 ::dealii::VectorizedArray<Number, width> out;
6793 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6794 ++i)
6795 out[i] = std::asinh(x[i]);
6796 return out;
6797 }
6798
6799
6800
6801 /**
6802 * Compute the area hyperbolic tangent of a vectorized data field. The result
6803 * is returned as vectorized array in the form <tt>{atanh(x[0]), atanh(x[1]),
6804 * ..., atanh(x[size()-1])}</tt>.
6805 *
6806 * @relatesalso VectorizedArray
6807 */
6808 template <typename Number, std::size_t width>
6809 inline ::dealii::VectorizedArray<Number, width>
6810 atanh(const ::dealii::VectorizedArray<Number, width> &x)
6811 {
6812 ::dealii::VectorizedArray<Number, width> out;
6813 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6814 ++i)
6815 out[i] = std::atanh(x[i]);
6816 return out;
6817 }
6818
6819
6820
6821 /**
6822 * Compute the exponential of a vectorized data field. The result is
6823 * returned as vectorized array in the form <tt>{exp(x[0]), exp(x[1]), ...,
6824 * exp(x[size()-1])}</tt>.
6825 *
6826 * @relatesalso VectorizedArray
6827 */
6828 template <typename Number, std::size_t width>
6829 inline ::dealii::VectorizedArray<Number, width>
6830 exp(const ::dealii::VectorizedArray<Number, width> &x)
6831 {
6832 ::dealii::VectorizedArray<Number, width> out;
6833 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6834 ++i)
6835 out[i] = std::exp(x[i]);
6836 return out;
6837 }
6838
6839
6840
6841 /**
6842 * Compute the natural logarithm of a vectorized data field. The result is
6843 * returned as vectorized array in the form <tt>{log(x[0]), log(x[1]), ...,
6844 * log(x[size()-1])}</tt>.
6845 *
6846 * @relatesalso VectorizedArray
6847 */
6848 template <typename Number, std::size_t width>
6849 inline ::dealii::VectorizedArray<Number, width>
6850 log(const ::dealii::VectorizedArray<Number, width> &x)
6851 {
6852 ::dealii::VectorizedArray<Number, width> out;
6853 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6854 ++i)
6855 out[i] = std::log(x[i]);
6856 return out;
6857 }
6858
6859
6860
6861 /**
6862 * Compute the square root of a vectorized data field. The result is
6863 * returned as vectorized array in the form <tt>{sqrt(x[0]), sqrt(x[1]),
6864 * ..., sqrt(x[size()-1])}</tt>.
6865 *
6866 * @relatesalso VectorizedArray
6867 */
6868 template <typename Number, std::size_t width>
6869 inline ::dealii::VectorizedArray<Number, width>
6870 sqrt(const ::dealii::VectorizedArray<Number, width> &x)
6871 {
6872 return x.get_sqrt();
6873 }
6874
6875
6876
6877 /**
6878 * Raises the given number @p x to the power @p p for a vectorized data
6879 * field. The result is returned as vectorized array in the form
6880 * <tt>{pow(x[0],p), pow(x[1],p), ..., pow(x[size()-1],p)}</tt>.
6881 *
6882 * @relatesalso VectorizedArray
6883 */
6884 template <typename Number, std::size_t width>
6885 inline ::dealii::VectorizedArray<Number, width>
6886 pow(const ::dealii::VectorizedArray<Number, width> &x, const Number p)
6887 {
6888 ::dealii::VectorizedArray<Number, width> out;
6889 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6890 ++i)
6891 out[i] = std::pow(x[i], p);
6892 return out;
6893 }
6894
6895
6896
6897 /**
6898 * Raises the given number @p x to the power @p p for a vectorized data
6899 * field. The result is returned as vectorized array in the form
6900 * <tt>{pow(x[0],p[0]), pow(x[1],p[1]), ...,
6901 * pow(x[size()-1],p[size()-1])}</tt>.
6902 *
6903 * @relatesalso VectorizedArray
6904 */
6905 template <typename Number, std::size_t width>
6906 inline ::dealii::VectorizedArray<Number, width>
6907 pow(const ::dealii::VectorizedArray<Number, width> &x,
6908 const ::dealii::VectorizedArray<Number, width> &p)
6909 {
6910 ::dealii::VectorizedArray<Number, width> out;
6911 for (unsigned int i = 0; i < dealii::VectorizedArray<Number, width>::size();
6912 ++i)
6913 out[i] = std::pow(x[i], p[i]);
6914 return out;
6915 }
6916
6917
6918
6919 /**
6920 * Compute the absolute value (modulus) of a vectorized data field. The
6921 * result is returned as vectorized array in the form <tt>{abs(x[0]),
6922 * abs(x[1]), ..., abs(x[size()-1])}</tt>.
6923 *
6924 * @relatesalso VectorizedArray
6925 */
6926 template <typename Number, std::size_t width>
6927 inline ::dealii::VectorizedArray<Number, width>
6928 abs(const ::dealii::VectorizedArray<Number, width> &x)
6929 {
6930 return x.get_abs();
6931 }
6932
6933
6934
6935 /**
6936 * Compute the componentwise maximum of two vectorized data fields. The
6937 * result is returned as vectorized array in the form <tt>{max(x[0],y[0]),
6938 * max(x[1],y[1]), ...}</tt>.
6939 *
6940 * @relatesalso VectorizedArray
6941 */
6942 template <typename Number, std::size_t width>
6943 inline ::dealii::VectorizedArray<Number, width>
6944 max(const ::dealii::VectorizedArray<Number, width> &x,
6945 const ::dealii::VectorizedArray<Number, width> &y)
6946 {
6947 return x.get_max(y);
6948 }
6949
6950
6951
6952 /**
6953 * Compute the componentwise minimum of two vectorized data fields. The
6954 * result is returned as vectorized array in the form <tt>{min(x[0],y[0]),
6955 * min(x[1],y[1]), ...}</tt>.
6956 *
6957 * @relatesalso VectorizedArray
6958 */
6959 template <typename Number, std::size_t width>
6960 inline ::dealii::VectorizedArray<Number, width>
6961 min(const ::dealii::VectorizedArray<Number, width> &x,
6962 const ::dealii::VectorizedArray<Number, width> &y)
6963 {
6964 return x.get_min(y);
6965 }
6966
6967
6968
6969 /**
6970 * Iterator traits for VectorizedArrayIterator.
6971 */
6972 template <class T>
6973 struct iterator_traits<dealii::VectorizedArrayIterator<T>>
6974 {
6975 #ifdef DEAL_II_HAVE_CXX20
6976 using iterator_category = contiguous_iterator_tag;
6977 #else
6978 using iterator_category = random_access_iterator_tag;
6979 #endif
6980 using value_type = typename T::value_type;
6981 using difference_type = std::ptrdiff_t;
6982 };
6983
6984 } // namespace std
6985
6986 #endif