Debian Codegraph @

packages/stokhos/src/sacado/kokkos/Stokhos_MemoryTraits.hpp

raw
1 // @HEADER
2 // *****************************************************************************
3 // Stokhos Package
4 //
5 // Copyright 2009 NTESS and the Stokhos contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9
10 #ifndef STOKHOS_MEMORY_TRAITS_HPP
11 #define STOKHOS_MEMORY_TRAITS_HPP
12
13 #include <cstdlib>
14
15 #include "Kokkos_Core_fwd.hpp"
16
17 // Currently always aligning
18 #define STOKHOS_ALIGN_MEMORY 1
19
20 // Uncomment this if you know all accesses will be aligned. Tthis is true
21 // if all Stokhos variables are coming from Kokkos allocations, new/delete
22 // or stack variables. However it may not be true for C allocations (e.g.,
23 // through MPI).
24 // #define STOKHOS_ASSUME_ALIGNED
25
26 // ivdep is necessary to get the intel compiler to vectorize through
27 // expresion template assignent operators
28 #if defined(__INTEL_COMPILER) && ! defined(__CUDA_ARCH__)
29 #define STOKHOS_HAVE_PRAGMA_IVDEP
30 #endif
31
32 // unrolling appears to slow everything down
33 #if 0 && ( defined(__INTEL_COMPILER) || defined(__CUDA_ARCH__) )
34 #define STOKHOS_HAVE_PRAGMA_UNROLL
35 #endif
36
37 // assume all memory accesses are aligned appropriately for aligned
38 // vector loads
39 #if defined(STOKHOS_ALIGN_MEMORY) && defined(STOKHOS_ASSUME_ALIGNED) && defined(__INTEL_COMPILER) && ! defined(__CUDA_ARCH__)
40 #define STOKHOS_HAVE_PRAGMA_VECTOR_ALIGNED
41 #endif
42
43 namespace Stokhos {
44
45 //! Traits class encapsulting memory alignment
46 template <typename MemorySpace>
47 struct MemoryTraits {
48
49 //! Bytes to which memory allocations are aligned
50 static const unsigned Alignment = 8;
51
52 //! Allocate aligned memory of given size
53 KOKKOS_INLINE_FUNCTION
54 static void* alloc(const size_t size) { return operator new(size); }
55
56 //! Free memory allocated by alloc()
57 KOKKOS_INLINE_FUNCTION
58 static void free(void *ptr) { operator delete(ptr); }
59 };
60
61 //! Specialization of MemoryTraits for host memory spaces
62 template <>
63 struct MemoryTraits< Kokkos::HostSpace > {
64
65 //! Bytes to which memory allocations are aligned
66 #if STOKHOS_ALIGN_MEMORY
67 #if defined(__MIC__)
68 static const unsigned Alignment = 64;
69 #elif defined(__AVX__)
70 static const unsigned Alignment = 32;
71 #elif defined(__SSE2__)
72 static const unsigned Alignment = 16;
73 #else
74 static const unsigned Alignment = 8;
75 #endif
76 #else
77 static const unsigned Alignment = 8;
78 #endif
79
80 //! Allocate aligned memory
81 /*!
82 * Note: We don't use mm_malloc or posix_memalign, because even though this
83 * implementation is host-only, it is potentially callable from host functions
84 * marked as device functions (via the KOKKOS_INLINE_FUNCTION maco).
85 *
86 * Also, we can't call new/delete as we may replace those with a version
87 * that calls this.
88 */
89 KOKKOS_INLINE_FUNCTION
90 static void* alloc(const size_t size) {
91 void* ptr = 0;
92 if (size > 0) {
93 #if STOKHOS_ALIGN_MEMORY
94 const size_t mask = Alignment-1;
95 const size_t total_size = size + mask + sizeof(std::ptrdiff_t);
96 char *ptr_alloc = reinterpret_cast<char*>(std::malloc(total_size));
97 char *ptr_storage = ptr_alloc + sizeof(std::ptrdiff_t);
98 char *ptr_body = reinterpret_cast<char*>(
99 ( reinterpret_cast<size_t>(ptr_storage) + mask ) & ~mask );
100 char *ptr_header = ptr_body - sizeof(std::ptrdiff_t);
101 const std::ptrdiff_t offset = ptr_body - ptr_alloc;
102 *reinterpret_cast<std::ptrdiff_t*>(ptr_header) = offset;
103 ptr = reinterpret_cast<void*>(ptr_body);
104 #else
105 ptr = operator new(size);
106 #endif
107 }
108 return ptr;
109 }
110
111 //! Free memory allocated by alloc()
112 KOKKOS_INLINE_FUNCTION
113 static void free(void *ptr) {
114 if (ptr != 0) {
115 #if STOKHOS_ALIGN_MEMORY
116 void *ptr_header = reinterpret_cast<char*>(ptr) - sizeof(std::ptrdiff_t);
117 const std::ptrdiff_t offset = *reinterpret_cast<std::ptrdiff_t*>(ptr_header);
118 void *ptr_alloc = reinterpret_cast<char*>(ptr) - offset;
119 std::free(ptr_alloc);
120 #else
121 operator delete(ptr);
122 #endif
123 }
124 }
125 };
126
127 //! An aligned STL allocator
128 template <typename T>
129 class aligned_allocator {
130 public:
131
132 typedef T value_type;
133 typedef T* pointer;
134 typedef const T* const_pointer;
135 typedef T& reference;
136 typedef const T& const_reference;
137 typedef size_t size_type;
138 typedef std::ptrdiff_t difference_type;
139
140 typedef Stokhos::MemoryTraits< Kokkos::HostSpace > Traits;
141
142 template <class U> struct rebind { typedef aligned_allocator<U> other; };
143
144 aligned_allocator() {}
145
146 template <class U> aligned_allocator(const aligned_allocator<U>&) {}
147
148 size_type max_size() const {
149 return (size_type(~0) - size_type(Traits::Alignment)) / sizeof(T);
150 }
151
152 pointer address(reference x) const { return &x; }
153
154 const_pointer address(const_reference x) const { return &x; }
155
156 pointer allocate(size_type n, const void* = 0) {
157 size_type count = n * sizeof(T);
158 void* ptr = Traits::alloc(count);
159 if (ptr == 0) throw std::bad_alloc();
160 return reinterpret_cast<pointer>(ptr);
161 }
162
163 void deallocate(pointer p, size_type) { Traits::free(p); }
164
165 void construct(pointer p, const_reference val) { new (p) T(val); }
166
167 void destroy(pointer p) { ((T*)p)->~T(); }
168 };
169
170 //! An aligned STL allocator
171 template <typename T>
172 class aligned_allocator< const T > {
173 public:
174
175 typedef T value_type;
176 typedef const T* pointer;
177 typedef const T* const_pointer;
178 typedef const T& reference;
179 typedef const T& const_reference;
180 typedef size_t size_type;
181 typedef std::ptrdiff_t difference_type;
182
183 typedef Stokhos::MemoryTraits< Kokkos::HostSpace > Traits;
184
185 template <class U> struct rebind { typedef aligned_allocator<U> other; };
186
187 aligned_allocator() {}
188
189 template <class U> aligned_allocator(const aligned_allocator<U>&) {}
190
191 size_type max_size() const {
192 return (size_type(~0) - size_type(Traits::Alignment)) / sizeof(T);
193 }
194
195 const_pointer address(const_reference x) const { return &x; }
196
197 pointer allocate(size_type n, const void* = 0) {
198 size_type count = n * sizeof(T);
199 void* ptr = Traits::alloc(count);
200 if (ptr == 0) throw std::bad_alloc();
201 return reinterpret_cast<pointer>(ptr);
202 }
203
204 void deallocate(pointer p, size_type) { Traits::free(p); }
205
206 void construct(pointer p, const_reference val) { new (p) T(val); }
207
208 void destroy(pointer p) { ((T*)p)->~T(); }
209 };
210
211 template <typename T, typename U>
212 inline bool
213 operator == (const aligned_allocator<T>&, const aligned_allocator<U>&)
214 { return true; }
215
216 template <typename T, typename U>
217 inline bool
218 operator != (const aligned_allocator<T>&, const aligned_allocator<U>&)
219 { return false; }
220
221 } // namespace Stokhos
222
223 #endif // STOKHOS_MEMORY_TRAITS_HPP