Debian Codegraph @

include/deal.II/base/aligned_vector.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2014 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15
16 #ifndef dealii_aligned_vector_h
17 #define dealii_aligned_vector_h
18
19 #include <deal.II/base/config.h>
20
21 #include <deal.II/base/exceptions.h>
22 #include <deal.II/base/memory_consumption.h>
23 #include <deal.II/base/mpi.h>
24 #include <deal.II/base/parallel.h>
25 #include <deal.II/base/utilities.h>
26
27 // boost::serialization::make_array used to be in array.hpp, but was
28 // moved to a different file in BOOST 1.64
29 #include <boost/version.hpp>
30 #if BOOST_VERSION >= 106400
31 # include <boost/serialization/array_wrapper.hpp>
32 #else
33 # include <boost/serialization/array.hpp>
34 #endif
35 #include <boost/serialization/split_member.hpp>
36
37 #include <cstring>
38 #include <memory>
39 #include <type_traits>
40
41
42
43 DEAL_II_NAMESPACE_OPEN
44
45
46 /**
47 * This is a replacement class for std::vector to be used in combination with
48 * VectorizedArray and derived data types. It allocates memory aligned to
49 * addresses of a vectorized data type (in order to avoid segmentation faults
50 * when a variable of type VectorizedArray which the compiler assumes to be
51 * aligned to certain memory addresses does not actually follow these rules).
52 * This could also be achieved by proving std::vector with a user-defined
53 * allocator. On the other hand, writing an own small vector class lets us
54 * implement parallel copy and move operations with TBB, insert deal.II-style
55 * assertions, and cut some unnecessary functionality. Note that this vector
56 * is a bit more memory-consuming than std::vector because of alignment, so it
57 * is recommended to only use this vector on long vectors.
58 */
59 template <class T>
60 class AlignedVector
61 {
62 public:
63 /**
64 * Declare standard types used in all containers. These types parallel those
65 * in the <tt>C++</tt> standard libraries <tt>vector<...></tt> class.
66 */
67 using value_type = T;
68 using pointer = value_type *;
69 using const_pointer = const value_type *;
70 using iterator = value_type *;
71 using const_iterator = const value_type *;
72 using reference = value_type &;
73 using const_reference = const value_type &;
74 using size_type = std::size_t;
75
76 /**
77 * Empty constructor. Sets the vector size to zero.
78 */
79 AlignedVector();
80
81 /**
82 * Range constructor.
83 *
84 * @note Unlike std::vector, this constructor uses random-access iterators so
85 * that the copy may be parallelized.
86 *
87 * @dealiiOperationIsMultithreaded
88 */
89 template <
90 typename RandomAccessIterator,
91 typename = std::enable_if_t<std::is_convertible_v<
92 typename std::iterator_traits<RandomAccessIterator>::iterator_category,
93 std::random_access_iterator_tag>>>
94 AlignedVector(RandomAccessIterator begin, RandomAccessIterator end);
95
96 /**
97 * Set the vector size to the given size and initializes all elements with
98 * T().
99 *
100 * @dealiiOperationIsMultithreaded
101 */
102 explicit AlignedVector(const size_type size, const T &init = T());
103
104 /**
105 * Destructor.
106 */
107 ~AlignedVector() = default;
108
109 /**
110 * Copy constructor.
111 *
112 * @dealiiOperationIsMultithreaded
113 */
114 AlignedVector(const AlignedVector<T> &vec);
115
116 /**
117 * Move constructor. Create a new aligned vector by stealing the contents of
118 * @p vec.
119 */
120 AlignedVector(AlignedVector<T> &&vec) noexcept;
121
122 /**
123 * Assignment to the input vector @p vec.
124 *
125 * @dealiiOperationIsMultithreaded
126 */
127 AlignedVector &
128 operator=(const AlignedVector<T> &vec);
129
130 /**
131 * Move assignment operator.
132 */
133 AlignedVector &
134 operator=(AlignedVector<T> &&vec) noexcept;
135
136 /**
137 * Change the size of the vector. If the new size is larger than the previous
138 * size, then new elements will be added to the end of the vector; these
139 * elements will remain uninitialized (i.e., left in an undefined state) if
140 * `std::is_trivially_default_constructible_v<T>` is `true`, and will be
141 * default initialized if that type trait is `false`. See
142 * [here](https://en.cppreference.com/w/cpp/types/is_default_constructible)
143 * for a precise definition of `std::is_trivially_default_constructible`.
144 *
145 * If the new size is less than the previous size, then the `new_size`th and
146 * all subsequent elements will be destroyed.
147 *
148 * As a consequence of the outline above, the "_fast" suffix of this function
149 * refers to the fact that for trivially default constructible types `T`, this
150 * function omits the initialization of new elements.
151 *
152 * @note This method can only be invoked for classes @p T that define a
153 * default constructor, @p T(). Otherwise, compilation will fail.
154 */
155 void
156 resize_fast(const size_type new_size);
157
158 /**
159 * Change the size of the vector. It keeps old elements previously
160 * available, and initializes each newly added element to a
161 * default-constructed object of type @p T.
162 *
163 * If the new vector size is shorter than the old one, the memory is
164 * not immediately released unless the new size is zero; however,
165 * the size of the current object is of course set to the requested
166 * value. The destructors of released elements are also called.
167 *
168 * @dealiiOperationIsMultithreaded
169 */
170 void
171 resize(const size_type new_size);
172
173 /**
174 * Change the size of the vector. It keeps old elements previously
175 * available, and initializes each newly added element with the
176 * provided initializer.
177 *
178 * If the new vector size is shorter than the old one, the memory is
179 * not immediately released unless the new size is zero; however,
180 * the size of the current object is of course set to the requested
181 * value.
182 *
183 * @note This method can only be invoked for classes that define the copy
184 * assignment operator. Otherwise, compilation will fail.
185 *
186 * @dealiiOperationIsMultithreaded
187 */
188 void
189 resize(const size_type new_size, const T &init);
190
191 /**
192 * Reserve memory space for @p new_allocated_size elements.
193 *
194 * If the argument @p new_allocated_size is less than the current number of stored
195 * elements (as indicated by calling size()), then this function does not
196 * do anything at all. Except if the argument @p new_allocated_size is set
197 * to zero, then all previously allocated memory is released (this operation
198 * then being equivalent to directly calling the clear() function).
199 *
200 * In order to avoid too frequent reallocation (which involves copy of the
201 * data), this function doubles the amount of memory occupied when the given
202 * size is larger than the previously allocated size.
203 *
204 * Note that this function only changes the amount of elements the object
205 * *can* store, but not the number of elements it *actually* stores. As
206 * a consequence, no constructors or destructors of newly created objects
207 * are run, though the existing elements may be moved to a new location (which
208 * involves running the move constructor at the new location and the
209 * destructor at the old location).
210 */
211 void
212 reserve(const size_type new_allocated_size);
213
214 /**
215 * Releases the memory allocated but not used.
216 */
217 void
218 shrink_to_fit();
219
220 /**
221 * Releases all previously allocated memory and leaves the vector in a state
222 * equivalent to the state after the default constructor has been called.
223 */
224 void
225 clear();
226
227 /**
228 * Inserts an element at the end of the vector, increasing the vector size
229 * by one. Note that the allocated size will double whenever the previous
230 * space is not enough to hold the new element.
231 */
232 void
233 push_back(const T in_data);
234
235 /**
236 * Return the last element of the vector (read and write access).
237 */
238 reference
239 back();
240
241 /**
242 * Return the last element of the vector (read-only access).
243 */
244 const_reference
245 back() const;
246
247 /**
248 * Inserts several elements at the end of the vector given by a range of
249 * elements.
250 */
251 template <typename ForwardIterator>
252 void
253 insert_back(ForwardIterator begin, ForwardIterator end);
254
255 /**
256 * Insert the range specified by @p begin and @p end after the element @p position.
257 *
258 * @note Unlike std::vector, this function uses random-access iterators so
259 * that the copy may be parallelized.
260 *
261 * @dealiiOperationIsMultithreaded
262 */
263 template <
264 typename RandomAccessIterator,
265 typename = std::enable_if_t<std::is_convertible_v<
266 typename std::iterator_traits<RandomAccessIterator>::iterator_category,
267 std::random_access_iterator_tag>>>
268 iterator
269 insert(const_iterator position,
270 RandomAccessIterator begin,
271 RandomAccessIterator end);
272
273 /**
274 * Fills the vector with size() copies of a default constructed object.
275 *
276 * @note Unlike the other fill() function, this method can also be
277 * invoked for classes that do not define a copy assignment
278 * operator.
279 *
280 * @dealiiOperationIsMultithreaded
281 */
282 void
283 fill();
284
285 /**
286 * Fills the vector with size() copies of the given input.
287 *
288 * @note This method can only be invoked for classes that define the copy
289 * assignment operator. Otherwise, compilation will fail.
290 *
291 * @dealiiOperationIsMultithreaded
292 */
293 void
294 fill(const T &element);
295
296 /**
297 * This function replicates the state found on the process indicated by
298 * @p root_process across all processes of the MPI communicator. The current
299 * state found on any of the processes other than @p root_process is lost
300 * in this process. One can imagine this operation to act like a call to
301 * Utilities::MPI::broadcast() from the root process to all other processes,
302 * though in practice the function may try to move the data into shared
303 * memory regions on each of the machines that host MPI processes and
304 * let all MPI processes on this machine then access this shared memory
305 * region instead of keeping their own copy.
306 *
307 * The intent of this function is to quickly exchange large arrays from
308 * one process to others, rather than having to compute or create it on
309 * all processes. This is specifically the case for data loaded from
310 * disk -- say, large data tables -- that are more easily dealt with by
311 * reading once and then distributing across all processes in an MPI
312 * universe, than letting each process read the data from disk itself.
313 * Specifically, the use of shared memory regions allows for replicating
314 * the data only once per multicore machine in the MPI universe, rather
315 * than replicating data once for each MPI process. This results in
316 * large memory savings if the data is large on today's machines that
317 * can easily house several dozen MPI processes per shared memory
318 * space. This use case is outlined in the TableBase class documentation
319 * as the current function is called from
320 * TableBase::replicate_across_communicator(). Indeed, the primary rationale
321 * for this function is to enable sharing data tables based on TableBase
322 * across MPI processes.
323 *
324 * This function does not imply a model of keeping data on different processes
325 * in sync, as LinearAlgebra::distributed::Vector and other vector classes do
326 * where there exists a notion of certain elements of the vector owned by each
327 * process and possibly ghost elements that are mirrored from its owning
328 * process to other processes. Rather, the elements of the current object are
329 * simply copied to the other processes, and it is useful to think of this
330 * operation as creating a set of `const` AlignedVector objects on all
331 * processes that should not be changed any more after the replication
332 * operation, as this is the only way to ensure that the vectors remain the
333 * same on all processes. This is particularly true because of the use of
334 * shared memory regions where any modification of a vector element on one MPI
335 * process may also result in a modification of elements visible on other
336 * processes, assuming they are located within one shared memory node.
337 *
338 * @note The use of shared memory between MPI processes requires
339 * that the detected MPI installation supports the necessary operations.
340 * This is the case for MPI 3.0 and higher.
341 *
342 * @note This function is not cheap. It needs to create sub-communicators
343 * of the provided @p communicator object, which is generally an expensive
344 * operation. Likewise, the generation of shared memory spaces is not
345 * a cheap operation. As a consequence, this function primarily makes
346 * sense when the goal is to share large read-only data tables among
347 * processes; examples are data tables that are loaded at start-up
348 * time and then used over the course of the run time of the program.
349 * In such cases, the start-up cost of running this function can be
350 * amortized over time, and the potential memory savings from not having to
351 * store the table on each process may be substantial on machines with
352 * large core counts on which many MPI processes run on the same machine.
353 *
354 * @note This function only makes sense if the data type `T` is
355 * "self-contained", i.e., all if its information is stored in its
356 * member variables, and if none of the member variables are pointers
357 * to other parts of the memory. This is because if a type `T` does
358 * have pointers to other parts of memory, then moving `T` into
359 * a shared memory space does not result in the other processes having
360 * access to data that the object points to with its member variable
361 * pointers: These continue to live only on one process, and are
362 * typically in memory areas not accessible to the other processes.
363 * As a consequence, the usual use case for this function is to share
364 * arrays of simple objects such as `double`s or `int`s.
365 *
366 * @note After calling this function, objects on different MPI processes
367 * share a common state. That means that certain operations become
368 * "collective", i.e., they must be called on all participating
369 * processors at the same time. In particular, you can no longer call
370 * resize(), reserve(), or clear() on one MPI process -- you have to do
371 * so on all processes at the same time, because they have to communicate
372 * for these operations. If you do not do so, you will likely get
373 * a deadlock that may be difficult to debug. By extension, this rule of
374 * only collectively resizing extends to this function itself: You can
375 * not call it twice in a row because that implies that first all but the
376 * `root_process` throw away their data, which is not a collective
377 * operation. Generally, these restrictions on what can and can not be
378 * done hint at the correctness of the comments above: You should treat
379 * an AlignedVector on which the current function has been called as
380 * `const`, on which no further operations can be performed until
381 * the destructor is called.
382 */
383 void
384 replicate_across_communicator(const MPI_Comm communicator,
385 const unsigned int root_process);
386
387 /**
388 * Swaps the given vector with the calling vector.
389 */
390 void
391 swap(AlignedVector<T> &vec) noexcept;
392
393 /**
394 * Return whether the vector is empty, i.e., its size is zero.
395 */
396 bool
397 empty() const;
398
399 /**
400 * Return the size of the vector.
401 */
402 size_type
403 size() const;
404
405 /**
406 * Return the capacity of the vector, i.e., the size this vector can hold
407 * without reallocation. Note that capacity() >= size().
408 */
409 size_type
410 capacity() const;
411
412 /**
413 * Read-write access to entry @p index in the vector.
414 */
415 reference
416 operator[](const size_type index);
417
418 /**
419 * Read-only access to entry @p index in the vector.
420 */
421 const_reference
422 operator[](const size_type index) const;
423
424 /**
425 * Return a pointer to the underlying data buffer.
426 */
427 pointer
428 data();
429
430 /**
431 * Return a const pointer to the underlying data buffer.
432 */
433 const_pointer
434 data() const;
435
436 /**
437 * Return a read and write pointer to the beginning of the data array.
438 */
439 iterator
440 begin();
441
442 /**
443 * Return a read and write pointer to the end of the data array.
444 */
445 iterator
446 end();
447
448 /**
449 * Return a read-only pointer to the beginning of the data array.
450 */
451 const_iterator
452 begin() const;
453
454 /**
455 * Return a read-only pointer to the end of the data array.
456 */
457 const_iterator
458 end() const;
459
460 /**
461 * Return the memory consumption of the allocated memory in this class. If
462 * the underlying type @p T allocates memory by itself, this memory is not
463 * counted.
464 */
465 size_type
466 memory_consumption() const;
467
468 /**
469 * Write the data of this object to a stream for the purpose of
470 * serialization using the [BOOST serialization
471 * library](https://www.boost.org/doc/libs/1_74_0/libs/serialization/doc/index.html).
472 */
473 template <class Archive>
474 void
475 save(Archive &ar, const unsigned int version) const;
476
477 /**
478 * Read the data of this object from a stream for the purpose of
479 * serialization using the [BOOST serialization
480 * library](https://www.boost.org/doc/libs/1_74_0/libs/serialization/doc/index.html).
481 */
482 template <class Archive>
483 void
484 load(Archive &ar, const unsigned int version);
485
486 #ifdef DOXYGEN
487 /**
488 * Write and read the data of this object from a stream for the purpose
489 * of serialization using the [BOOST serialization
490 * library](https://www.boost.org/doc/libs/1_74_0/libs/serialization/doc/index.html).
491 */
492 template <class Archive>
493 void
494 serialize(Archive &archive, const unsigned int version);
495 #else
496 // This macro defines the serialize() method that is compatible with
497 // the templated save() and load() method that have been implemented.
498 BOOST_SERIALIZATION_SPLIT_MEMBER()
499 #endif
500
501 /**
502 * Exception message for changing the vector after a call to
503 * replicate_across_communicator().
504 *
505 * @ingroup Exceptions
506 */
507 DeclExceptionMsg(ExcAlignedVectorChangeAfterReplication,
508 "Changing the vector after a call to "
509 "replicate_across_communicator() is not allowed.");
510
511 private:
512 /**
513 * Make a new allocation, move the data from the old memory region to the new
514 * region, and release the old memory.
515 */
516 void
517 allocate_and_move(const std::size_t old_size,
518 const std::size_t new_size,
519 const std::size_t new_allocated_size);
520
521 /**
522 * A class that is used as the "deleter" for a `std::unique_ptr` object that
523 * AlignedVector uses to store the memory used for the elements.
524 *
525 * There are two ways the AlignedVector class can handle memory:
526 * - Allocation via `new[]` in reserve() where we call `posix_memalign()`
527 * to obtain a chunk of memory and then do placement-`new` to initialize
528 * memory. If this is what we have done, then we need to call the
529 * destructors of the currently active elements by hand, and then
530 * call `std::free()` to return memory. In order to call the destructors
531 * of currently used elements, the deleter object needs to have access
532 * to the owning `AlignedVector` object to know which of the allocated
533 * elements are currently actually used.
534 * - We have called `replicate_across_communicator()`, in which case the
535 * elements have been moved into a memory "window" managed by MPI.
536 * In that case, one process (the root process of an MPI communicator
537 * that ties together all processes on one node) needs to call the
538 * destructor of all elements in this shared memory space, and then
539 * all processes on that communicator need to first destroy the
540 * MPI window object, and then the MPI communicator for the shared
541 * memory node's processes. In this approach, we need to store the
542 * following data: A pointer to the owning AlignedVector object to know
543 * which elements need to be destroyed, copies of the MPI window
544 * and communicator objects, and a couple of ancillary pieces of data.
545 *
546 * A common idiom towards using `std::unique_ptr` with complex de-allocation
547 * semantics is to use `std::unique_ptr<T, std::function<void (T*)>`
548 * and then use a lambda function to initialize the `std::function`
549 * deleter object. This approach is used in numerous other places in
550 * deal.II, but this has two downsides:
551 * - `std::function` is relatively memory-hungry. It takes 24 bytes by
552 * itself, but then also needs to allocate memory dynamically, for
553 * example to store the capture object of a lambda function. This ends
554 * up to be quite a lot of memory given that we frequently use small
555 * Vector objects (which build on AlignedVector).
556 * - More importantly, this breaks move operations. In a move constructor
557 * or move assignment of AlignedVector, we want to steal the memory pointed
558 * to, but we then need to install a new deleter object because the deleter
559 * needs to know about the owning object to determine which elements to
560 * call the destructor for. The problem is that we can't use the old
561 * deleter (it is a lambda function that still points to the previous
562 * owner that we are moving *from*) but we also don't know what deleter
563 * to install otherwise -- we don't know the innards of the lambda function
564 * that was previously installed; in fact, we don't even know whether it
565 * was for regular or MPI shared-memory data management.
566 *
567 * The way out of this is to write a custom deleter. It stores a pointer to
568 * the owning AlignedVector object. It then also stores a `std::unique_ptr`
569 * to an object of a class derived from a base class that implements the
570 * concrete action necessary to facilitate the "deleter" action. Based on
571 * the arguments given to the constructor of the Deleter class, the
572 * constructor then either allocates a "regular" or an MPI-based action
573 * object, and the action is facilitated by an overloaded `virtual` function.
574 *
575 * In the end, this solves both of our problems:
576 * - The deleter object only requires 16 bytes (two pointers) plus whatever
577 * dynamic memory is necessary to store the "action" objects.
578 * - In move operations, the only thing that needs to be done is to tell
579 * the deleter about the change of owning AlignedVector object, which we
580 * can achieve via the reset_owning_object() function.
581 *
582 * This scheme can be further optimized in the following way: In the most
583 * common case, memory is allocated via `posix_memalign()` and needs to
584 * destroyed via `std::free()`. Rather than derive an action class for
585 * this common case, dynamically allocate an object for this case, and
586 * call it, we can just special case this situation: If the pointer to the
587 * action object is `nullptr`, then we just execute the default action.
588 * This means that for the most common case, we do not need any dynamic
589 * memory allocation at all, and in that case the deleter object truly
590 * only uses 16 bytes: One pointer to the owning AlignedVector object,
591 * and a pointer to the action object that happens to be a `nullptr`.
592 * Only in the case of the MPI shared memory data management does the
593 * action pointer point somewhere, but this case is expensive anyway and so
594 * the extra dynamic memory allocation does little harm.
595 *
596 * (One could in that case go even further: There is no really only one
597 * possible non-default action left, namely the MPI-based destruction of
598 * the shared-memory window. Instead of having the Deleter::DeleterActionBase
599 * class below, and the derived Deleter::MPISharedMemDeleterAction class, both
600 * with `virtual` functions, we could just get rid of the base class and make
601 * the member functions of the derived class non-`virtual`. We would then
602 * either store a pointer to an MPISharedMemDeleterAction object if the
603 * non-default action is requested, or a `nullptr` for the default action.
604 * Avoiding `virtual` functions would make these objects smaller by a few
605 * bytes, and the call to the `delete_array()` function marginally faster.
606 * That said, given that the Deleter::MPISharedMemDeleterAction object already
607 * stores all sorts of stuff and its execution is not cheap, these additional
608 * optimizations are probably not worth it. Instead, we kept the class
609 * hierarchy so that one could define other non-standard actions in the future
610 * through other derived classes.)
611 */
612 class Deleter
613 {
614 public:
615 /**
616 * Constructor. When this constructor is called, it installs an
617 * action that corresponds to "regular" memory allocation that
618 * needs to be handled by using `std::free()`.
619 */
620 Deleter(AlignedVector<T> *owning_object);
621
622 #ifdef DEAL_II_WITH_MPI
623 /**
624 * Constructor. When this constructor is called, it installs an
625 * action that corresponds to MPI-based shared memory allocation that
626 * needs to be handled by letting MPI de-allocate the shared memory
627 * window, plus destroying the MPI communicator, and doing other
628 * clean-up work.
629 */
630 Deleter(AlignedVector<T> *owning_object,
631 const bool is_shmem_root,
632 T *aligned_shmem_pointer,
633 MPI_Comm shmem_group_communicator,
634 MPI_Win shmem_window);
635 #endif
636
637 /**
638 * The operator called by `std::unique_ptr` to destroy the data it
639 * is storing. This function dispatches to the different actions that
640 * this class implements.
641 */
642 void
643 operator()(T *ptr);
644
645 /**
646 * Reset the pointer to the owning AlignedVector object. This function
647 * is used in move operations when the pointer to the data is transferred
648 * from one AlignedVector object -- i.e., the pointer itself remains
649 * unchanged, but the deleter object needs to be updated to know who
650 * the new owner now is.
651 */
652 void
653 reset_owning_object(const AlignedVector<T> *new_aligned_vector_ptr);
654
655 private:
656 /**
657 * Base class for the action necessary to de-allocate memory.
658 */
659 class DeleterActionBase
660 {
661 public:
662 /**
663 * Destructor, made `virtual` to allow for derived classes.
664 */
665 virtual ~DeleterActionBase() = default;
666
667 /**
668 * The function that implements the action of de-allocating memory.
669 * It receives as arguments a pointer to the owning AlignedVector object
670 * as well as a pointer to the memory being de-allocated.
671 */
672 virtual void
673 delete_array(const AlignedVector<T> *owning_aligned_vector, T *ptr) = 0;
674 };
675
676 #ifdef DEAL_II_WITH_MPI
677
678 /**
679 * A class that implements the deleter action for MPI shared-memory
680 * allocated data.
681 */
682 class MPISharedMemDeleterAction : public DeleterActionBase
683 {
684 public:
685 /**
686 * Constructor. Store the various pieces of information necessary to
687 * identify the MPI window in which the data resides.
688 */
689 MPISharedMemDeleterAction(const bool is_shmem_root,
690 T *aligned_shmem_pointer,
691 MPI_Comm shmem_group_communicator,
692 MPI_Win shmem_window);
693
694 /**
695 * The function that implements the action of de-allocating memory.
696 * It receives as arguments a pointer to the owning AlignedVector object
697 * as well as a pointer to the memory being de-allocated.
698 */
699 virtual void
700 delete_array(const AlignedVector<T> *aligned_vector, T *ptr) override;
701
702 private:
703 /**
704 * Variables necessary to identify the MPI shared-memory window plus
705 * all ancillary information to destroy this window.
706 */
707 const bool is_shmem_root;
708 T *aligned_shmem_pointer;
709 MPI_Comm shmem_group_communicator;
710 MPI_Win shmem_window;
711 };
712 #endif
713
714 /**
715 * A pointer to the object that facilitates the actual action of
716 * destroying the memory.
717 */
718 std::unique_ptr<DeleterActionBase> deleter_action_object;
719
720 /**
721 * A (non-owned) pointer to the surrounding AlignedVector object that owns
722 * the memory this deleter object is responsible for deleting.
723 */
724 const AlignedVector<T> *owning_aligned_vector;
725 };
726
727 /**
728 * Pointer to actual data array, using the custom deleter class above.
729 */
730 std::unique_ptr<T[], Deleter> elements;
731
732 /**
733 * Pointer to one past the last valid value.
734 */
735 T *used_elements_end;
736
737 /**
738 * Pointer to the end of the allocated memory.
739 */
740 T *allocated_elements_end;
741
742 /**
743 * Flag indicating if replicate_across_communicator() has been called.
744 */
745 bool replicated_across_communicator;
746 };
747
748
749 // ------------------------------- inline functions --------------------------
750
751 /**
752 * This namespace defines the copy and set functions used in AlignedVector.
753 * These functions operate in parallel when there are enough elements in the
754 * vector.
755 */
756 namespace internal
757 {
758 /**
759 * A class that given a range of memory locations calls the placement-new
760 * operator on these memory locations and copy-constructs objects of type
761 * `T` there.
762 *
763 * This class is based on the specialized for loop base class
764 * ParallelForLoop in parallel.h whose purpose is the following: When
765 * calling a parallel for loop on AlignedVector with apply_to_subranges, it
766 * generates different code for every different argument we might choose (as
767 * it is templated). This gives a lot of code (e.g. it triples the memory
768 * required for compiling the file matrix_free.cc and the final object size
769 * is several times larger) which is completely useless. Therefore, this
770 * class channels all copy commands through one call to apply_to_subrange
771 * for all possible types, which makes the copy operation much cleaner
772 * (thanks to a virtual function, whose cost is negligible in this context).
773 *
774 * @relatesalso AlignedVector
775 */
776 template <typename RandomAccessIterator, typename T>
777 class AlignedVectorCopyConstruct
778 : private dealii::parallel::ParallelForInteger
779 {
780 static const std::size_t minimum_parallel_grain_size =
781 160000 / sizeof(T) + 1;
782
783 public:
784 /**
785 * Constructor. Issues a parallel call if there are sufficiently many
786 * elements, otherwise works in serial. Copies the data from the half-open
787 * interval between @p source_begin and @p source_end to array starting at
788 * @p destination (by calling the copy constructor with placement new).
789 *
790 * The elements from the source array are simply copied via the placement
791 * new copy constructor.
792 */
793 AlignedVectorCopyConstruct(RandomAccessIterator source_begin,
794 RandomAccessIterator source_end,
795 T *const destination)
796 : source_(source_begin)
797 , destination_(destination)
798 {
799 Assert(source_end >= source_begin, ExcInternalError());
800 Assert(source_end == source_begin || destination != nullptr,
801 ExcInternalError());
802 const std::size_t size = source_end - source_begin;
803 if (size < minimum_parallel_grain_size)
804 AlignedVectorCopyConstruct::apply_to_subrange(0, size);
805 else
806 apply_parallel(0, size, minimum_parallel_grain_size);
807 }
808
809 /**
810 * This method moves elements from the source to the destination given in
811 * the constructor on a subrange given by two integers.
812 */
813 virtual void
814 apply_to_subrange(const std::size_t begin,
815 const std::size_t end) const override
816 {
817 if (end == begin)
818 return;
819
820 // We can use memcpy() with trivially copyable objects.
821 if constexpr (std::is_trivially_copyable_v<T> == true &&
822 (std::is_same_v<T *, RandomAccessIterator> ||
823 std::is_same_v<const T *, RandomAccessIterator>) == true)
824 std::memcpy(destination_ + begin,
825 source_ + begin,
826 (end - begin) * sizeof(T));
827 else
828 for (std::size_t i = begin; i < end; ++i)
829 new (&destination_[i]) T(*(source_ + i));
830 }
831
832 private:
833 RandomAccessIterator source_;
834 T *const destination_;
835 };
836
837
838 /**
839 * Like AlignedVectorCopyConstruct, but use the move-constructor of `T`
840 * to create new objects.
841 *
842 * @relatesalso AlignedVector
843 */
844 template <typename RandomAccessIterator, typename T>
845 class AlignedVectorMoveConstruct
846 : private dealii::parallel::ParallelForInteger
847 {
848 static const std::size_t minimum_parallel_grain_size =
849 160000 / sizeof(T) + 1;
850
851 public:
852 /**
853 * Constructor. Issues a parallel call if there are sufficiently many
854 * elements, otherwise works in serial. Moves the data from the half-open
855 * interval between @p source_begin and @p source_end to array starting at
856 * @p destination (by calling the move constructor with placement new).
857 *
858 * The data is moved between the two arrays by invoking the destructor on
859 * the source range (preparing for a subsequent call to free).
860 */
861 AlignedVectorMoveConstruct(RandomAccessIterator source_begin,
862 RandomAccessIterator source_end,
863 T *const destination)
864 : source_(source_begin)
865 , destination_(destination)
866 {
867 Assert(source_end >= source_begin, ExcInternalError());
868 Assert(source_end == source_begin || destination != nullptr,
869 ExcInternalError());
870 const std::size_t size = source_end - source_begin;
871 if (size < minimum_parallel_grain_size)
872 AlignedVectorMoveConstruct::apply_to_subrange(0, size);
873 else
874 apply_parallel(0, size, minimum_parallel_grain_size);
875 }
876
877 /**
878 * This method moves elements from the source to the destination given in
879 * the constructor on a subrange given by two integers.
880 */
881 virtual void
882 apply_to_subrange(const std::size_t begin,
883 const std::size_t end) const override
884 {
885 if (end == begin)
886 return;
887
888 // We can use memcpy() with trivially copyable objects.
889 if constexpr (std::is_trivially_copyable_v<T> == true &&
890 (std::is_same_v<T *, RandomAccessIterator> ||
891 std::is_same_v<const T *, RandomAccessIterator>) == true)
892 std::memcpy(destination_ + begin,
893 source_ + begin,
894 (end - begin) * sizeof(T));
895 else
896 // For everything else just use the move constructor. The original
897 // object remains alive and will be destroyed elsewhere.
898 for (std::size_t i = begin; i < end; ++i)
899 new (&destination_[i]) T(std::move(*(source_ + i)));
900 }
901
902 private:
903 RandomAccessIterator source_;
904 T *const destination_;
905 };
906
907
908 /**
909 * A class that given a range of memory locations either calls
910 * the placement-new operator on these memory locations (if
911 * `initialize_memory==true`) or just copies the given initializer
912 * into this memory location (if `initialize_memory==false`). The
913 * latter is appropriate for classes that have only trivial constructors,
914 * such as the built-in types `double`, `int`, etc., and structures
915 * composed of such types.
916 *
917 * @tparam initialize_memory Determines whether the set command should
918 * initialize memory (with a call to the copy constructor) or rather use the
919 * copy assignment operator. A template is necessary to select the
920 * appropriate operation since some classes might define only one of those
921 * two operations.
922 *
923 * @relatesalso AlignedVector
924 */
925 template <typename T, bool initialize_memory>
926 class AlignedVectorInitialize : private dealii::parallel::ParallelForInteger
927 {
928 static const std::size_t minimum_parallel_grain_size =
929 160000 / sizeof(T) + 1;
930
931 public:
932 /**
933 * Constructor. Issues a parallel call if there are sufficiently many
934 * elements, otherwise work in serial.
935 */
936 AlignedVectorInitialize(const std::size_t size,
937 const T &element,
938 T *const destination)
939 : element_(element)
940 , destination_(destination)
941 , trivial_element(false)
942 {
943 if (size == 0)
944 return;
945 Assert(destination != nullptr, ExcInternalError());
946
947 // do not use memcmp() for long double because on some systems it does not
948 // completely fill its memory and may lead to false positives in e.g.
949 // valgrind
950 if constexpr (std::is_trivially_default_constructible_v<T> == true &&
951 std::is_same_v<T, long double> == false)
952 {
953 const unsigned char zero[sizeof(T)] = {};
954 if (std::memcmp(zero, &element, sizeof(T)) == 0)
955 trivial_element = true;
956 }
957 if (size < minimum_parallel_grain_size)
958 AlignedVectorInitialize::apply_to_subrange(0, size);
959 else
960 apply_parallel(0, size, minimum_parallel_grain_size);
961 }
962
963 /**
964 * This sets elements on a subrange given by two integers.
965 */
966 virtual void
967 apply_to_subrange(const std::size_t begin,
968 const std::size_t end) const override
969 {
970 // Only use memset() with types whose default constructors don't do
971 // anything.
972 if constexpr (std::is_trivially_default_constructible_v<T> == true)
973 if (trivial_element)
974 {
975 std::memset(destination_ + begin, 0, (end - begin) * sizeof(T));
976 return;
977 }
978
979 copy_construct_or_assign(begin,
980 end,
981 std::bool_constant<initialize_memory>());
982 }
983
984 private:
985 const T &element_;
986 mutable T *destination_;
987 bool trivial_element;
988
989 // copy assignment operation
990 void
991 copy_construct_or_assign(const std::size_t begin,
992 const std::size_t end,
993 std::bool_constant<false>) const
994 {
995 for (std::size_t i = begin; i < end; ++i)
996 destination_[i] = element_;
997 }
998
999 // copy constructor (memory initialization)
1000 void
1001 copy_construct_or_assign(const std::size_t begin,
1002 const std::size_t end,
1003 std::bool_constant<true>) const
1004 {
1005 for (std::size_t i = begin; i < end; ++i)
1006 new (&destination_[i]) T(element_);
1007 }
1008 };
1009
1010
1011
1012 /**
1013 * Like AlignedVectorInitialize, but use default-constructed objects
1014 * as initializers.
1015 *
1016 * @tparam initialize_memory Sets whether the set command should
1017 * initialize memory (with a call to the copy constructor) or rather use the
1018 * copy assignment operator. A template is necessary to select the
1019 * appropriate operation since some classes might define only one of those
1020 * two operations.
1021 *
1022 * @relatesalso AlignedVector
1023 */
1024 template <typename T, bool initialize_memory>
1025 class AlignedVectorDefaultInitialize
1026 : private dealii::parallel::ParallelForInteger
1027 {
1028 static const std::size_t minimum_parallel_grain_size =
1029 160000 / sizeof(T) + 1;
1030
1031 public:
1032 /**
1033 * Constructor. Issues a parallel call if there are sufficiently many
1034 * elements, otherwise work in serial.
1035 */
1036 AlignedVectorDefaultInitialize(const std::size_t size, T *const destination)
1037 : destination_(destination)
1038 {
1039 if (size == 0)
1040 return;
1041 Assert(destination != nullptr, ExcInternalError());
1042
1043 if (size < minimum_parallel_grain_size)
1044 AlignedVectorDefaultInitialize::apply_to_subrange(0, size);
1045 else
1046 apply_parallel(0, size, minimum_parallel_grain_size);
1047 }
1048
1049 /**
1050 * This initializes elements on a subrange given by two integers.
1051 */
1052 virtual void
1053 apply_to_subrange(const std::size_t begin,
1054 const std::size_t end) const override
1055 {
1056 // Only use memset() with types whose default constructors don't do
1057 // anything.
1058 if constexpr (std::is_trivially_default_constructible_v<T> == true)
1059 std::memset(destination_ + begin, 0, (end - begin) * sizeof(T));
1060 else
1061 default_construct_or_assign(begin,
1062 end,
1063 std::bool_constant<initialize_memory>());
1064 }
1065
1066 private:
1067 mutable T *destination_;
1068
1069 // copy assignment operation
1070 void
1071 default_construct_or_assign(const std::size_t begin,
1072 const std::size_t end,
1073 std::bool_constant<false>) const
1074 {
1075 for (std::size_t i = begin; i < end; ++i)
1076 destination_[i] = std::move(T());
1077 }
1078
1079 // copy constructor (memory initialization)
1080 void
1081 default_construct_or_assign(const std::size_t begin,
1082 const std::size_t end,
1083 std::bool_constant<true>) const
1084 {
1085 for (std::size_t i = begin; i < end; ++i)
1086 new (&destination_[i]) T;
1087 }
1088 };
1089
1090 } // end of namespace internal
1091
1092
1093 #ifndef DOXYGEN
1094
1095
1096
1097 template <typename T>
1098 inline AlignedVector<T>::Deleter::Deleter(AlignedVector<T> *owning_object)
1099 : deleter_action_object(nullptr) // encode default action by using a nullptr
1100 , owning_aligned_vector(owning_object)
1101 {}
1102
1103
1104 # ifdef DEAL_II_WITH_MPI
1105
1106 template <typename T>
1107 inline AlignedVector<T>::Deleter::Deleter(AlignedVector<T> *owning_object,
1108 const bool is_shmem_root,
1109 T *aligned_shmem_pointer,
1110 MPI_Comm shmem_group_communicator,
1111 MPI_Win shmem_window)
1112 : deleter_action_object(
1113 std::make_unique<MPISharedMemDeleterAction>(is_shmem_root,
1114 aligned_shmem_pointer,
1115 shmem_group_communicator,
1116 shmem_window))
1117 , owning_aligned_vector(owning_object)
1118 {}
1119 # endif
1120
1121
1122 template <typename T>
1123 inline void
1124 AlignedVector<T>::Deleter::operator()(T *ptr)
1125 {
1126 // If no special action has been registered (i.e., if the action pointer is
1127 // nullptr), then just perform the default action right here.
1128 if (deleter_action_object == nullptr)
1129 {
1130 if (ptr != nullptr)
1131 {
1132 Assert(owning_aligned_vector->used_elements_end != nullptr,
1133 ExcInternalError());
1134
1135 if (std::is_trivially_destructible_v<T> == false)
1136 for (T *p = owning_aligned_vector->used_elements_end - 1; p >= ptr;
1137 --p)
1138 p->~T();
1139
1140 std::free(ptr);
1141 }
1142 }
1143 else
1144 // Otherwise, let the action object do what is necessary
1145 deleter_action_object->delete_array(owning_aligned_vector, ptr);
1146 }
1147
1148
1149
1150 template <typename T>
1151 inline void
1152 AlignedVector<T>::Deleter::reset_owning_object(
1153 const AlignedVector<T> *new_aligned_vector_ptr)
1154 {
1155 owning_aligned_vector = new_aligned_vector_ptr;
1156 }
1157
1158
1159 # ifdef DEAL_II_WITH_MPI
1160
1161 template <typename T>
1162 inline AlignedVector<T>::Deleter::MPISharedMemDeleterAction::
1163 MPISharedMemDeleterAction(const bool is_shmem_root,
1164 T *aligned_shmem_pointer,
1165 MPI_Comm shmem_group_communicator,
1166 MPI_Win shmem_window)
1167 : is_shmem_root(is_shmem_root)
1168 , aligned_shmem_pointer(aligned_shmem_pointer)
1169 , shmem_group_communicator(shmem_group_communicator)
1170 , shmem_window(shmem_window)
1171 {}
1172
1173
1174
1175 template <typename T>
1176 inline void
1177 AlignedVector<T>::Deleter::MPISharedMemDeleterAction::delete_array(
1178 const AlignedVector<T> *aligned_vector,
1179 T *ptr)
1180 {
1181 (void)ptr;
1182 // It would be nice to assert that aligned_vector->elements.get() equals ptr,
1183 // but it is not guaranteed to work: clang, for example, sets elements.get()
1184 // to nullptr and then calls the deleter on a previously made copy. Hence we
1185 // must assume here that elements.get() (which is managed by the unique_ptr)
1186 // may be nullptr at this point.
1187 //
1188 // used_elements_end is a member variable of AlignedVector (i.e., we control
1189 // it, not unique_ptr) so it is still set to its correct value.
1190
1191 if (is_shmem_root)
1192 if (std::is_trivially_destructible_v<T> == false)
1193 for (T *p = aligned_vector->used_elements_end - 1; p >= ptr; --p)
1194 p->~T();
1195
1196 int ierr;
1197 ierr = MPI_Win_free(&shmem_window);
1198 AssertThrowMPI(ierr);
1199
1200 Utilities::MPI::free_communicator(shmem_group_communicator);
1201 }
1202
1203 # endif
1204
1205
1206 template <class T>
1207 inline AlignedVector<T>::AlignedVector()
1208 : elements(nullptr, Deleter(this))
1209 , used_elements_end(nullptr)
1210 , allocated_elements_end(nullptr)
1211 , replicated_across_communicator(false)
1212 {}
1213
1214
1215
1216 template <class T>
1217 template <typename RandomAccessIterator, typename>
1218 inline AlignedVector<T>::AlignedVector(RandomAccessIterator begin,
1219 RandomAccessIterator end)
1220 : elements(nullptr, Deleter(this))
1221 , used_elements_end(nullptr)
1222 , allocated_elements_end(nullptr)
1223 , replicated_across_communicator(false)
1224 {
1225 allocate_and_move(0u, end - begin, end - begin);
1226 used_elements_end = allocated_elements_end;
1227 dealii::internal::AlignedVectorCopyConstruct<RandomAccessIterator, T>(begin,
1228 end,
1229 data());
1230 }
1231
1232
1233 template <class T>
1234 inline AlignedVector<T>::AlignedVector(const size_type size, const T &init)
1235 : elements(nullptr, Deleter(this))
1236 , used_elements_end(nullptr)
1237 , allocated_elements_end(nullptr)
1238 , replicated_across_communicator(false)
1239 {
1240 if (size > 0)
1241 resize(size, init);
1242 }
1243
1244
1245
1246 template <class T>
1247 inline AlignedVector<T>::AlignedVector(const AlignedVector<T> &vec)
1248 : elements(nullptr, Deleter(this))
1249 , used_elements_end(nullptr)
1250 , allocated_elements_end(nullptr)
1251 , replicated_across_communicator(false)
1252 {
1253 // copy the data from vec
1254 reserve(vec.size());
1255 used_elements_end = allocated_elements_end;
1256 internal::AlignedVectorCopyConstruct<T *, T>(vec.elements.get(),
1257 vec.used_elements_end,
1258 elements.get());
1259 }
1260
1261
1262
1263 template <class T>
1264 inline AlignedVector<T>::AlignedVector(AlignedVector<T> &&vec) noexcept
1265 : AlignedVector<T>()
1266 {
1267 // forward to the move operator
1268 *this = std::move(vec);
1269 }
1270
1271
1272
1273 template <class T>
1274 inline AlignedVector<T> &
1275 AlignedVector<T>::operator=(const AlignedVector<T> &vec)
1276 {
1277 const size_type new_size = vec.used_elements_end - vec.elements.get();
1278
1279 // First throw away everything and re-allocate memory but leave that
1280 // memory uninitialized for now:
1281 resize(0);
1282 reserve(new_size);
1283
1284 // Then copy the elements over by using the copy constructor on these
1285 // elements:
1286 internal::AlignedVectorCopyConstruct<T *, T>(vec.elements.get(),
1287 vec.used_elements_end,
1288 elements.get());
1289
1290 // Finally adjust the pointer to the end of the elements that are used:
1291 used_elements_end = elements.get() + new_size;
1292
1293 return *this;
1294 }
1295
1296
1297
1298 template <class T>
1299 inline AlignedVector<T> &
1300 AlignedVector<T>::operator=(AlignedVector<T> &&vec) noexcept
1301 {
1302 clear();
1303
1304 // Move the actual data in the 'elements' object. One problem is that this
1305 // also moves the deleter object, but the deleter object
1306 // references 'this' (i.e., the 'this' pointer of the *moved-from*
1307 // object). The way this is implemented is that we have to move the
1308 // deleter as well, and then reset the pointer inside the deleter
1309 // that references the outer object.
1310 elements = std::move(vec.elements);
1311 elements.get_deleter().reset_owning_object(this);
1312
1313 // Then also steal the other pointers and clear them in the original object:
1314 used_elements_end = vec.used_elements_end;
1315 allocated_elements_end = vec.allocated_elements_end;
1316
1317 vec.used_elements_end = nullptr;
1318 vec.allocated_elements_end = nullptr;
1319
1320 return *this;
1321 }
1322
1323
1324
1325 template <class T>
1326 inline void
1327 AlignedVector<T>::resize_fast(const size_type new_size)
1328 {
1329 const size_type old_size = size();
1330
1331 if (new_size == 0)
1332 clear();
1333 else if (new_size == old_size)
1334 {
1335 } // nothing to do here
1336 else if (new_size < old_size)
1337 {
1338 // call destructor on fields that are released, if the type requires it.
1339 // doing it backward releases the elements in reverse order as compared to
1340 // how they were created
1341 if (std::is_trivially_destructible_v<T> == false)
1342 for (T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1343 p->~T();
1344 used_elements_end = elements.get() + new_size;
1345 }
1346 else // new_size > old_size
1347 {
1348 // Allocate more space, and claim that space as used
1349 reserve(new_size);
1350 used_elements_end = elements.get() + new_size;
1351
1352 // Leave the new array entries as-is (with undefined values) unless T's
1353 // default constructor is nontrivial (i.e., it is not a no-op)
1354 if (std::is_trivially_default_constructible_v<T> == false)
1355 dealii::internal::AlignedVectorDefaultInitialize<T, true>(
1356 new_size - old_size, elements.get() + old_size);
1357 }
1358 }
1359
1360
1361
1362 template <class T>
1363 inline void
1364 AlignedVector<T>::resize(const size_type new_size)
1365 {
1366 const size_type old_size = size();
1367
1368 if (new_size == 0)
1369 clear();
1370 else if (new_size == old_size)
1371 {
1372 } // nothing to do here
1373 else if (new_size < old_size)
1374 {
1375 // call destructor on fields that are released, if the type requires it.
1376 // doing it backward releases the elements in reverse order as compared to
1377 // how they were created
1378 if (std::is_trivially_destructible_v<T> == false)
1379 for (T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1380 p->~T();
1381 used_elements_end = elements.get() + new_size;
1382 }
1383 else // new_size > old_size
1384 {
1385 // Allocate more space, and claim that space as used
1386 reserve(new_size);
1387 used_elements_end = elements.get() + new_size;
1388
1389 // finally set the values to the default initializer
1390 dealii::internal::AlignedVectorDefaultInitialize<T, true>(
1391 new_size - old_size, elements.get() + old_size);
1392 }
1393 }
1394
1395
1396
1397 template <class T>
1398 inline void
1399 AlignedVector<T>::resize(const size_type new_size, const T &init)
1400 {
1401 const size_type old_size = size();
1402
1403 if (new_size == 0)
1404 clear();
1405 else if (new_size == old_size)
1406 {
1407 } // nothing to do here
1408 else if (new_size < old_size)
1409 {
1410 // call destructor on fields that are released, if the type requires it.
1411 // doing it backward releases the elements in reverse order as compared to
1412 // how they were created
1413 if (std::is_trivially_destructible_v<T> == false)
1414 for (T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1415 p->~T();
1416 used_elements_end = elements.get() + new_size;
1417 }
1418 else // new_size > old_size
1419 {
1420 // Allocate more space, and claim that space as used
1421 reserve(new_size);
1422 used_elements_end = elements.get() + new_size;
1423
1424 // finally set the desired init values
1425 dealii::internal::AlignedVectorInitialize<T, true>(
1426 new_size - old_size, init, elements.get() + old_size);
1427 }
1428 }
1429
1430
1431
1432 template <class T>
1433 inline void
1434 AlignedVector<T>::allocate_and_move(const std::size_t old_size,
1435 const std::size_t new_size,
1436 const std::size_t new_allocated_size)
1437 {
1438 // allocate and align along 64-byte boundaries (this is enough for all
1439 // levels of vectorization currently supported by deal.II)
1440 T *new_data_ptr;
1441 Utilities::System::posix_memalign(reinterpret_cast<void **>(&new_data_ptr),
1442 64,
1443 new_size * sizeof(T));
1444
1445 // Now create a deleter that encodes what should happen when the object is
1446 // released: We need to destroy the objects that are currently alive (in
1447 // reverse order, and then release the memory. Note that we catch the
1448 // 'this' pointer because the number of elements currently alive might
1449 // change over time.
1450 Deleter deleter(this);
1451
1452 // copy whatever elements we need to retain
1453 if (new_allocated_size > 0)
1454 dealii::internal::AlignedVectorMoveConstruct<T *, T>(
1455 elements.get(), elements.get() + old_size, new_data_ptr);
1456
1457 // Now reset all the member variables of the current object
1458 // based on the allocation above. Assigning to a std::unique_ptr
1459 // object also releases the previously pointed to memory.
1460 //
1461 // Note that at the time of releasing the old memory, 'used_elements_end'
1462 // still points to its previous value, and this is important for the
1463 // deleter object of the previously allocated array (see how it loops over
1464 // the to-be-destroyed elements at the Deleter::DefaultDeleterAction
1465 // class).
1466 elements = decltype(elements)(new_data_ptr, std::move(deleter));
1467 used_elements_end = elements.get() + old_size;
1468 allocated_elements_end = elements.get() + new_size;
1469 }
1470
1471
1472
1473 template <class T>
1474 inline void
1475 AlignedVector<T>::reserve(const size_type new_allocated_size)
1476 {
1477 const size_type old_size = used_elements_end - elements.get();
1478 const size_type old_allocated_size = allocated_elements_end - elements.get();
1479 if (new_allocated_size > old_allocated_size)
1480 {
1481 // if we continuously increase the size of the vector, we might be
1482 // reallocating a lot of times. therefore, try to increase the size more
1483 // aggressively
1484 const size_type new_size =
1485 std::max(new_allocated_size, 2 * old_allocated_size);
1486
1487 allocate_and_move(old_size, new_size, new_allocated_size);
1488 }
1489 else if (new_allocated_size == 0)
1490 clear();
1491 else // size_alloc < allocated_size
1492 {
1493 } // nothing to do here
1494 }
1495
1496
1497
1498 template <class T>
1499 inline void
1500 AlignedVector<T>::shrink_to_fit()
1501 {
1502 if constexpr (running_in_debug_mode())
1503 {
1504 Assert(replicated_across_communicator == false,
1505 ExcAlignedVectorChangeAfterReplication());
1506 }
1507 const size_type used_size = used_elements_end - elements.get();
1508 const size_type allocated_size = allocated_elements_end - elements.get();
1509 if (allocated_size > used_size)
1510 allocate_and_move(used_size, used_size, used_size);
1511 }
1512
1513
1514
1515 template <class T>
1516 inline void
1517 AlignedVector<T>::clear()
1518 {
1519 // Just release the memory (which also calls the destructor of the elements),
1520 // and then set the auxiliary pointers to invalid values.
1521 //
1522 // Note that at the time of releasing the old memory, 'used_elements_end'
1523 // still points to its previous value, and this is important for the
1524 // deleter object of the previously allocated array (see how it loops over
1525 // the to-be-destroyed elements a few lines above).
1526 elements.reset();
1527 used_elements_end = nullptr;
1528 allocated_elements_end = nullptr;
1529 }
1530
1531
1532
1533 template <class T>
1534 inline void
1535 AlignedVector<T>::push_back(const T in_data)
1536 {
1537 Assert(used_elements_end <= allocated_elements_end, ExcInternalError());
1538 if (used_elements_end == allocated_elements_end)
1539 reserve(std::max(2 * capacity(), static_cast<size_type>(16)));
1540 new (used_elements_end++) T(in_data);
1541 }
1542
1543
1544
1545 template <class T>
1546 inline typename AlignedVector<T>::reference
1547 AlignedVector<T>::back()
1548 {
1549 AssertIndexRange(0, size());
1550 T *field = used_elements_end - 1;
1551 return *field;
1552 }
1553
1554
1555
1556 template <class T>
1557 inline typename AlignedVector<T>::const_reference
1558 AlignedVector<T>::back() const
1559 {
1560 AssertIndexRange(0, size());
1561 const T *field = used_elements_end - 1;
1562 return *field;
1563 }
1564
1565
1566
1567 template <class T>
1568 template <typename ForwardIterator>
1569 inline void
1570 AlignedVector<T>::insert_back(ForwardIterator begin, ForwardIterator end)
1571 {
1572 const size_type old_size = size();
1573 reserve(old_size + (end - begin));
1574 for (; begin != end; ++begin, ++used_elements_end)
1575 new (used_elements_end) T(*begin);
1576 }
1577
1578
1579
1580 template <class T>
1581 template <typename RandomAccessIterator, typename>
1582 inline typename AlignedVector<T>::iterator
1583 AlignedVector<T>::insert(const_iterator position,
1584 RandomAccessIterator begin,
1585 RandomAccessIterator end)
1586 {
1587 Assert(replicated_across_communicator == false,
1588 ExcAlignedVectorChangeAfterReplication());
1589 Assert(this->begin() <= position && position <= this->end(),
1590 ExcMessage("The position iterator is not valid."));
1591 const auto offset = position - this->begin();
1592
1593 const size_type old_size = size();
1594 const size_type range_size = end - begin;
1595 const size_type new_size = old_size + range_size;
1596 if (range_size != 0)
1597 {
1598 // This is similar to allocate_and_move(), except that we need to move
1599 // whatever was before position and whatever is after it into two
1600 // different places
1601 T *new_data_ptr = nullptr;
1602 Utilities::System::posix_memalign(
1603 reinterpret_cast<void **>(&new_data_ptr), 64, new_size * sizeof(T));
1604
1605 // Correctly handle the case where the range is inside the present array
1606 // by creating a temporary.
1607 AlignedVector<T> temporary(begin, end);
1608 dealii::internal::AlignedVectorMoveConstruct<T *, T>(
1609 elements.get(), elements.get() + offset, new_data_ptr);
1610 dealii::internal::AlignedVectorMoveConstruct<T *, T>(
1611 temporary.begin(), temporary.end(), new_data_ptr + offset);
1612 dealii::internal::AlignedVectorMoveConstruct<T *, T>(
1613 elements.get() + offset,
1614 elements.get() + old_size,
1615 new_data_ptr + offset + range_size);
1616
1617 Deleter deleter(this);
1618 elements = decltype(elements)(new_data_ptr, std::move(deleter));
1619 used_elements_end = elements.get() + new_size;
1620 allocated_elements_end = elements.get() + new_size;
1621 }
1622 return this->begin() + offset;
1623 }
1624
1625
1626
1627 template <class T>
1628 inline void
1629 AlignedVector<T>::fill()
1630 {
1631 dealii::internal::AlignedVectorDefaultInitialize<T, false>(size(),
1632 elements.get());
1633 }
1634
1635
1636
1637 template <class T>
1638 inline void
1639 AlignedVector<T>::fill(const T &value)
1640 {
1641 dealii::internal::AlignedVectorInitialize<T, false>(size(),
1642 value,
1643 elements.get());
1644 }
1645
1646
1647
1648 template <class T>
1649 inline void
1650 AlignedVector<T>::replicate_across_communicator(const MPI_Comm communicator,
1651 const unsigned int root_process)
1652 {
1653 # ifdef DEAL_II_WITH_MPI
1654
1655 // Let the root process broadcast its size. If it is zero, then all
1656 // processes just clear() their memory and reset themselves to a non-shared
1657 // empty object -- there is no point to run through complicated MPI
1658 // calls if the end result is an empty array. Otherwise, we continue on.
1659 const size_type new_size =
1660 Utilities::MPI::broadcast(communicator, size(), root_process);
1661 if (new_size == 0)
1662 {
1663 clear();
1664 return;
1665 }
1666
1667
1668 // **** Step 0 ****
1669 // All but the root process no longer need their data, so release the memory
1670 // used to store the previous elements.
1671 if (Utilities::MPI::this_mpi_process(communicator) != root_process)
1672 {
1673 elements.reset();
1674 used_elements_end = nullptr;
1675 allocated_elements_end = nullptr;
1676 }
1677
1678 // **** Step 1 ****
1679 // Create communicators for each group of processes that can use
1680 // shared memory areas. Within each of these groups, we don't care about
1681 // which rank each of the old processes gets except that we would like to
1682 // make sure that the (global) root process will have rank=0 within
1683 // its own sub-communicator. We can do that through the third argument of
1684 // MPI_Comm_split_type (the "key") which is an integer meant to indicate the
1685 // order of processes within the split communicators, and we should set it to
1686 // zero for the root processes and one for all others -- which means that
1687 // for all of these other processes, MPI can choose whatever order it
1688 // wants because they have the same key (MPI then documents that these ties
1689 // will be broken according to these processes' rank in the old group).
1690 //
1691 // At least that's the theory. In practice, the MPI implementation where
1692 // this function was developed on does not seem to do that. (Bug report
1693 // is here: https://github.com/open-mpi/ompi/issues/8854)
1694 // We work around this by letting MPI_Comm_split_type choose whatever
1695 // rank it wants, and then reshuffle with MPI_Comm_split in a second
1696 // step -- not elegant, nor efficient, but seems to work:
1697 MPI_Comm shmem_group_communicator;
1698 {
1699 MPI_Comm shmem_group_communicator_temp;
1700 int ierr = MPI_Comm_split_type(communicator,
1701 MPI_COMM_TYPE_SHARED,
1702 /* key */ 0,
1703 MPI_INFO_NULL,
1704 &shmem_group_communicator_temp);
1705 AssertThrowMPI(ierr);
1706
1707 const int key =
1708 (Utilities::MPI::this_mpi_process(communicator) == root_process ? 0 : 1);
1709 ierr = MPI_Comm_split(shmem_group_communicator_temp,
1710 /* color */ 0,
1711 key,
1712 &shmem_group_communicator);
1713 AssertThrowMPI(ierr);
1714
1715 // Verify the explanation from above
1716 if (Utilities::MPI::this_mpi_process(communicator) == root_process)
1717 Assert(Utilities::MPI::this_mpi_process(shmem_group_communicator) == 0,
1718 ExcInternalError());
1719
1720 // And get rid of the temporary communicator
1721 Utilities::MPI::free_communicator(shmem_group_communicator_temp);
1722 }
1723 const bool is_shmem_root =
1724 Utilities::MPI::this_mpi_process(shmem_group_communicator) == 0;
1725
1726 // **** Step 2 ****
1727 // We then have to send the state of the current object from the
1728 // root process to one exemplar in each shmem group. To this end,
1729 // we create another subcommunicator that includes the ranks zero
1730 // of all shmem groups, and because of the trick above, we know
1731 // that this also includes the original root process.
1732 //
1733 // There are different ways of creating a "shmem_roots_communicator".
1734 // The conceptually easiest way is to create an MPI_Group that only
1735 // includes the shmem roots and then create a communicator from this
1736 // via MPI_Comm_create or MPI_Comm_create_group. The problem
1737 // with this is that we would have to exchange among all processes
1738 // which ones are shmem roots and which are not. This is awkward.
1739 //
1740 // A simpler way is to use MPI_Comm_split that uses "colors" to
1741 // indicate which sub-communicator each process wants to be in.
1742 // We use color=0 to indicate the group of shmem roots, and color=1
1743 // for all other processes -- the latter will simply not ever do
1744 // anything among themselves with the communicator so created.
1745 //
1746 // Using MPI_Comm_split has the additional benefit that, just as above,
1747 // we can choose where each rank will end up in shmem_roots_communicator.
1748 // We again set key=0 for the original root_process, and key=1 for all other
1749 // ranks; then, the global root becomes rank=0 on the
1750 // shmem_roots_communicator. We don't care how the other processes are
1751 // ordered.
1752 MPI_Comm shmem_roots_communicator;
1753 {
1754 const int key =
1755 (Utilities::MPI::this_mpi_process(communicator) == root_process ? 0 : 1);
1756
1757 const int ierr = MPI_Comm_split(communicator,
1758 /*color=*/
1759 (is_shmem_root ? 0 : 1),
1760 key,
1761 &shmem_roots_communicator);
1762 AssertThrowMPI(ierr);
1763
1764 // Again verify the explanation from above
1765 if (Utilities::MPI::this_mpi_process(communicator) == root_process)
1766 Assert(Utilities::MPI::this_mpi_process(shmem_roots_communicator) == 0,
1767 ExcInternalError());
1768 }
1769
1770 const unsigned int shmem_roots_root_rank = 0;
1771 const bool is_shmem_roots_root =
1772 (is_shmem_root && (Utilities::MPI::this_mpi_process(
1773 shmem_roots_communicator) == shmem_roots_root_rank));
1774
1775 // Now let the original root_process broadcast the current object to all
1776 // shmem roots. We know that the last rank is the original root process that
1777 // has all of the data.
1778 if (is_shmem_root)
1779 {
1780 if (std::is_trivially_copyable_v<T> == true)
1781 {
1782 // The data is trivially copyable, i.e., we can copy things directly
1783 // without having to go through the serialization/deserialization
1784 // machinery of Utilities::MPI::broadcast.
1785 //
1786 // In that case, first tell all of the other shmem roots how many
1787 // elements we will have to deal with, and let them resize their
1788 // (non-shared) arrays.
1789 const size_type new_size =
1790 Utilities::MPI::broadcast(shmem_roots_communicator,
1791 size(),
1792 shmem_roots_root_rank);
1793 if (is_shmem_roots_root == false)
1794 resize(new_size);
1795
1796 // Then directly copy from the root process into these buffers
1797 int ierr = MPI_Bcast(elements.get(),
1798 sizeof(T) * new_size,
1799 MPI_CHAR,
1800 shmem_roots_root_rank,
1801 shmem_roots_communicator);
1802 AssertThrowMPI(ierr);
1803 }
1804 else
1805 {
1806 // The objects to be sent around are not "trivial", and so we have
1807 // to go through the serialization/deserialization machinery. On all
1808 // but the sending process, overwrite the current state with the
1809 // vector just broadcast.
1810 //
1811 // On the root rank, this would lead to resetting the 'entries'
1812 // pointer, which would trigger the deleter which would lead to a
1813 // deadlock. So we just send the result of the broadcast() call to
1814 // nirvana on the root process and keep our current state.
1815 if (Utilities::MPI::this_mpi_process(shmem_roots_communicator) == 0)
1816 std::ignore = Utilities::MPI::broadcast(shmem_roots_communicator,
1817 *this,
1818 shmem_roots_root_rank);
1819 else
1820 *this = Utilities::MPI::broadcast(shmem_roots_communicator,
1821 *this,
1822 shmem_roots_root_rank);
1823 }
1824 }
1825
1826 // We no longer need the shmem roots communicator, so get rid of it
1827 Utilities::MPI::free_communicator(shmem_roots_communicator);
1828
1829
1830 // **** Step 3 ****
1831 // At this point, all shmem groups have one shmem root process that has
1832 // a copy of the data. This is the point where each shmem group should
1833 // establish a shmem area to put the data into. As mentioned above,
1834 // we know that the shmem roots are the last rank in their respective
1835 // shmem_group_communicator.
1836 //
1837 // The process for all of this works as follows: While all processes in
1838 // the shmem group participate in the generation of the shmem memory window,
1839 // only the shmem root actually allocates any memory -- the rest just
1840 // allocate zero bytes of their own. We allocate space for exactly
1841 // size() elements (computed on the shmem_root that already has the data)
1842 // and add however many bytes are necessary so that we know that we can align
1843 // things to 64-byte boundaries. The worst case happens if the memory system
1844 // gives us a pointer to an address one byte past a desired alignment
1845 // boundary, and in that case aligning the memory will require us to waste the
1846 // first (align_by-1) bytes. So we have to ask for
1847 // size() * sizeof(T) + (align_by - 1)
1848 // bytes.
1849 //
1850 // Before MPI 4.0, there was no way to specify that we want memory aligned to
1851 // a certain number of bytes. This is going to come back to bite us further
1852 // down below when we try to get a properly aligned pointer to our memory
1853 // region, see the commentary there. Starting with MPI 4.0, one can set a
1854 // flag in an MPI_Info structure that requests a desired alignment, so we do
1855 // this for forward compatibility; MPI implementations ignore flags they don't
1856 // know anything about, and so setting this flag is backward compatible also
1857 // to older MPI versions.
1858 MPI_Win shmem_window;
1859 void *base_ptr;
1860 const MPI_Aint align_by = 64;
1861 const MPI_Aint alloc_size =
1862 Utilities::MPI::broadcast(shmem_group_communicator,
1863 (size() * sizeof(T) + (align_by - 1)),
1864 0);
1865
1866 {
1867 int ierr;
1868
1869 MPI_Info mpi_info;
1870 ierr = MPI_Info_create(&mpi_info);
1871 AssertThrowMPI(ierr);
1872 ierr = MPI_Info_set(mpi_info,
1873 "mpi_minimum_memory_alignment",
1874 std::to_string(align_by).c_str());
1875 AssertThrowMPI(ierr);
1876 ierr = MPI_Win_allocate_shared((is_shmem_root ? alloc_size : 0),
1877 /* disp_unit = */ 1,
1878 mpi_info,
1879 shmem_group_communicator,
1880 &base_ptr,
1881 &shmem_window);
1882 AssertThrowMPI(ierr);
1883
1884 ierr = MPI_Info_free(&mpi_info);
1885 AssertThrowMPI(ierr);
1886 }
1887
1888
1889 // **** Step 4 ****
1890 // The next step is to teach all non-shmem root processes what the pointer to
1891 // the array is that the shmem-root created. MPI has a nifty way for this
1892 // given that only a single process actually allocated memory in the window:
1893 // When calling MPI_Win_shared_query, the MPI documentation says that
1894 // "When rank is MPI_PROC_NULL, the pointer, disp_unit, and size returned are
1895 // the pointer, disp_unit, and size of the memory segment belonging the lowest
1896 // rank that specified size > 0. If all processes in the group attached to the
1897 // window specified size = 0, then the call returns size = 0 and a baseptr as
1898 // if MPI_ALLOC_MEM was called with size = 0."
1899 //
1900 // This will allow us to obtain the pointer to the shmem root's memory area,
1901 // which is the only one we care about. (None of the other processes have
1902 // even allocated any memory.)
1903 //
1904 // We don't need to do this on the shmem root process: This process has
1905 // already gotten its base_ptr correctly set above, and we can determine the
1906 // array size by just calling size().
1907 if (is_shmem_root == false)
1908 {
1909 int disp_unit;
1910 MPI_Aint alloc_size; // not actually used
1911 const int ierr = MPI_Win_shared_query(
1912 shmem_window, MPI_PROC_NULL, &alloc_size, &disp_unit, &base_ptr);
1913 AssertThrowMPI(ierr);
1914
1915 // Make sure we actually got a pointer, and check that the disp_unit is
1916 // equal to 1 (as set above)
1917 Assert(base_ptr != nullptr, ExcInternalError());
1918 Assert(disp_unit == 1, ExcInternalError());
1919 }
1920
1921
1922 // **** Step 5 ****
1923 // Now that all processes know the address of the space that is visible to
1924 // everyone, we need to figure out whether it is properly aligned and if not,
1925 // find the next aligned address.
1926 //
1927 // std::align does that, but it also modifies its last two arguments. The
1928 // documentation of that function at
1929 // https://en.cppreference.com/w/cpp/memory/align is not entirely clear, but I
1930 // *think* that the following should do given that we do not use base_ptr and
1931 // available_space any further after the call to std::align.
1932 std::size_t available_space = alloc_size;
1933 void *base_ptr_backup = base_ptr;
1934 T *aligned_shmem_pointer = static_cast<T *>(
1935 std::align(align_by, new_size * sizeof(T), base_ptr, available_space));
1936 Assert(aligned_shmem_pointer != nullptr, ExcInternalError());
1937
1938 // There is one step to guard against. It is *conceivable* that the base_ptr
1939 // we have previously obtained from MPI_Win_shared_query is mapped so
1940 // awkwardly into the different MPI processes' memory spaces that it is
1941 // aligned in one memory space, but not another. In that case, different
1942 // processes would align base_ptr differently, and adjust available_space
1943 // differently. We can check that by making sure that the max (or min) over
1944 // all processes is equal to every process's value. If that's not the case,
1945 // then the whole idea of aligning above is wrong and we need to rethink what
1946 // it means to align data in a shared memory space.
1947 //
1948 // One might be tempted to think that this is not how MPI implementations
1949 // actually arrange things. Alas, when developing this functionality in 2021,
1950 // this is really how at least OpenMPI ends up doing things. (This is with an
1951 // OpenMPI implementation of MPI 3.1, so it does not support the flag we set
1952 // in the MPI_Info structure above when allocating the memory window.) Indeed,
1953 // when running this code on three processes, one ends up with base_ptr values
1954 // of
1955 // base_ptr=0x7f0842f02108
1956 // base_ptr=0x7fc0a47881d0
1957 // base_ptr=0x7f64872db108
1958 // which, most annoyingly, are aligned to 8 and 16 byte boundaries -- so there
1959 // is no common offset std::align could find that leads to a 64-byte
1960 // aligned memory address in all three memory spaces. That's a tremendous
1961 // nuisance and there is really nothing we can do about this other than just
1962 // fall back on the (unaligned) base_ptr in that case.
1963 if (Utilities::MPI::min(available_space, shmem_group_communicator) !=
1964 Utilities::MPI::max(available_space, shmem_group_communicator))
1965 aligned_shmem_pointer = static_cast<T *>(base_ptr_backup);
1966
1967
1968 // **** Step 6 ****
1969 // If this is the shmem root process, we need to copy the data into the
1970 // shared memory space.
1971 if (is_shmem_root)
1972 {
1973 if (std::is_trivially_copyable_v<T> == true)
1974 std::memcpy(aligned_shmem_pointer, elements.get(), sizeof(T) * size());
1975 else
1976 for (std::size_t i = 0; i < size(); ++i)
1977 new (&aligned_shmem_pointer[i]) T(std::move(elements[i]));
1978 }
1979
1980 // Make sure that the shared memory host has copied the data before we try to
1981 // access it.
1982 const int ierr = MPI_Barrier(shmem_group_communicator);
1983 AssertThrowMPI(ierr);
1984
1985 // **** Step 7 ****
1986 // Finally, we need to set the pointers of this object to what we just
1987 // learned. This also releases all memory that may have been in use
1988 // previously.
1989 //
1990 // The part that is a bit tricky is how to write the deleter of this
1991 // shared memory object. When we want to get rid of it, we need to
1992 // also release the MPI_Win object along with the shmem_group_communicator
1993 // object. That's because as long as we use the shared memory, we still need
1994 // to hold on to the MPI_Win object, and the MPI_Win object is based on the
1995 // communicator. (The former is definitely true, the latter is not quite clear
1996 // from the MPI documentation, but seems reasonable.) So we need to have a
1997 // deleter for the pointer that ensures that upon release of the memory, we
1998 // not only call the destructor of these memory elements (but only once, on
1999 // the shmem root!) but also destroy the MPI_Win and the communicator. All of
2000 // that is encapsulated in the following call where the deleter makes copies
2001 // of the arguments in the lambda capture.
2002 elements = decltype(elements)(aligned_shmem_pointer,
2003 Deleter(this,
2004 is_shmem_root,
2005 aligned_shmem_pointer,
2006 shmem_group_communicator,
2007 shmem_window));
2008
2009 // We then also have to set the other two pointers that define the state of
2010 // the current object. Note that the new buffer size is exactly as large as
2011 // necessary, i.e., can store size() elements, regardless of the number of
2012 // allocated elements in the original objects.
2013 used_elements_end = elements.get() + new_size;
2014 allocated_elements_end = used_elements_end;
2015
2016 // **** Consistency check ****
2017 // At this point, each process should have a copy of the data.
2018 // Verify this in some sort of round-about way
2019 if constexpr (running_in_debug_mode())
2020 {
2021 replicated_across_communicator = true;
2022 const std::vector<char> packed_data = Utilities::pack(*this);
2023 const int hash =
2024 std::accumulate(packed_data.begin(), packed_data.end(), int(0));
2025 Assert(Utilities::MPI::max(hash, communicator) == hash,
2026 ExcInternalError());
2027 }
2028
2029 # else
2030 // No MPI -> nothing to replicate
2031 (void)communicator;
2032 (void)root_process;
2033 # endif
2034 }
2035
2036
2037
2038 template <class T>
2039 inline void
2040 AlignedVector<T>::swap(AlignedVector<T> &vec) noexcept
2041 {
2042 // Swap the data in the 'elements' objects. Then also make sure that
2043 // their respective deleter objects point to the right place.
2044 std::swap(elements, vec.elements);
2045 elements.get_deleter().reset_owning_object(this);
2046 vec.elements.get_deleter().reset_owning_object(&vec);
2047
2048 // Now also swap the remaining members.
2049 std::swap(used_elements_end, vec.used_elements_end);
2050 std::swap(allocated_elements_end, vec.allocated_elements_end);
2051 }
2052
2053
2054
2055 template <class T>
2056 inline bool
2057 AlignedVector<T>::empty() const
2058 {
2059 return used_elements_end == elements.get();
2060 }
2061
2062
2063
2064 template <class T>
2065 inline typename AlignedVector<T>::size_type
2066 AlignedVector<T>::size() const
2067 {
2068 return used_elements_end - elements.get();
2069 }
2070
2071
2072
2073 template <class T>
2074 inline typename AlignedVector<T>::size_type
2075 AlignedVector<T>::capacity() const
2076 {
2077 return allocated_elements_end - elements.get();
2078 }
2079
2080
2081
2082 template <class T>
2083 inline typename AlignedVector<T>::reference
2084 AlignedVector<T>::operator[](const size_type index)
2085 {
2086 AssertIndexRange(index, size());
2087 return elements[index];
2088 }
2089
2090
2091
2092 template <class T>
2093 inline typename AlignedVector<T>::const_reference
2094 AlignedVector<T>::operator[](const size_type index) const
2095 {
2096 AssertIndexRange(index, size());
2097 return elements[index];
2098 }
2099
2100
2101
2102 template <typename T>
2103 inline typename AlignedVector<T>::pointer
2104 AlignedVector<T>::data()
2105 {
2106 return elements.get();
2107 }
2108
2109
2110
2111 template <typename T>
2112 inline typename AlignedVector<T>::const_pointer
2113 AlignedVector<T>::data() const
2114 {
2115 return elements.get();
2116 }
2117
2118
2119
2120 template <class T>
2121 inline typename AlignedVector<T>::iterator
2122 AlignedVector<T>::begin()
2123 {
2124 return elements.get();
2125 }
2126
2127
2128
2129 template <class T>
2130 inline typename AlignedVector<T>::iterator
2131 AlignedVector<T>::end()
2132 {
2133 return used_elements_end;
2134 }
2135
2136
2137
2138 template <class T>
2139 inline typename AlignedVector<T>::const_iterator
2140 AlignedVector<T>::begin() const
2141 {
2142 return elements.get();
2143 }
2144
2145
2146
2147 template <class T>
2148 inline typename AlignedVector<T>::const_iterator
2149 AlignedVector<T>::end() const
2150 {
2151 return used_elements_end;
2152 }
2153
2154
2155
2156 template <class T>
2157 template <class Archive>
2158 inline void
2159 AlignedVector<T>::save(Archive &ar, const unsigned int) const
2160 {
2161 size_type vec_size = size();
2162 ar &vec_size;
2163 if (vec_size > 0)
2164 ar &boost::serialization::make_array(elements.get(), vec_size);
2165 }
2166
2167
2168
2169 template <class T>
2170 template <class Archive>
2171 inline void
2172 AlignedVector<T>::load(Archive &ar, const unsigned int)
2173 {
2174 size_type vec_size = 0;
2175 ar &vec_size;
2176
2177 if (vec_size > 0)
2178 {
2179 reserve(vec_size);
2180 ar &boost::serialization::make_array(elements.get(), vec_size);
2181 used_elements_end = elements.get() + vec_size;
2182 }
2183 }
2184
2185
2186
2187 template <class T>
2188 inline typename AlignedVector<T>::size_type
2189 AlignedVector<T>::memory_consumption() const
2190 {
2191 size_type memory = sizeof(*this);
2192 for (const T *t = elements.get(); t != used_elements_end; ++t)
2193 memory += dealii::MemoryConsumption::memory_consumption(*t);
2194 memory += sizeof(T) * (allocated_elements_end - used_elements_end);
2195 return memory;
2196 }
2197
2198
2199 #endif // ifndef DOXYGEN
2200
2201
2202 /**
2203 * Relational operator == for AlignedVector
2204 *
2205 * @relatesalso AlignedVector
2206 */
2207 template <class T>
2208 bool
2209 operator==(const AlignedVector<T> &lhs, const AlignedVector<T> &rhs)
2210 {
2211 if (lhs.size() != rhs.size())
2212 return false;
2213 for (typename AlignedVector<T>::const_iterator lit = lhs.begin(),
2214 rit = rhs.begin();
2215 lit != lhs.end();
2216 ++lit, ++rit)
2217 if (*lit != *rit)
2218 return false;
2219 return true;
2220 }
2221
2222
2223
2224 /**
2225 * Relational operator != for AlignedVector
2226 *
2227 * @relatesalso AlignedVector
2228 */
2229 template <class T>
2230 bool
2231 operator!=(const AlignedVector<T> &lhs, const AlignedVector<T> &rhs)
2232 {
2233 return !(operator==(lhs, rhs));
2234 }
2235
2236
2237 DEAL_II_NAMESPACE_CLOSE
2238
2239 #endif