| 1 |
// @HEADER |
| 2 |
// ***************************************************************************** |
| 3 |
// Stokhos Package |
| 4 |
// |
| 5 |
// Copyright 2009 NTESS and the Stokhos contributors. |
| 6 |
// SPDX-License-Identifier: BSD-3-Clause |
| 7 |
// ***************************************************************************** |
| 8 |
// @HEADER |
| 9 |
|
| 10 |
#ifndef STOKHOS_MEMORY_TRAITS_HPP |
| 11 |
#define STOKHOS_MEMORY_TRAITS_HPP |
| 12 |
|
| 13 |
#include <cstdlib> |
| 14 |
|
| 15 |
#include "Kokkos_Core_fwd.hpp" |
| 16 |
|
| 17 |
// Currently always aligning |
| 18 |
#define STOKHOS_ALIGN_MEMORY 1 |
| 19 |
|
| 20 |
// Uncomment this if you know all accesses will be aligned. Tthis is true |
| 21 |
// if all Stokhos variables are coming from Kokkos allocations, new/delete |
| 22 |
// or stack variables. However it may not be true for C allocations (e.g., |
| 23 |
// through MPI). |
| 24 |
// #define STOKHOS_ASSUME_ALIGNED |
| 25 |
|
| 26 |
// ivdep is necessary to get the intel compiler to vectorize through |
| 27 |
// expresion template assignent operators |
| 28 |
#if defined(__INTEL_COMPILER) && ! defined(__CUDA_ARCH__) |
| 29 |
#define STOKHOS_HAVE_PRAGMA_IVDEP |
| 30 |
#endif |
| 31 |
|
| 32 |
// unrolling appears to slow everything down |
| 33 |
#if 0 && ( defined(__INTEL_COMPILER) || defined(__CUDA_ARCH__) ) |
| 34 |
#define STOKHOS_HAVE_PRAGMA_UNROLL |
| 35 |
#endif |
| 36 |
|
| 37 |
// assume all memory accesses are aligned appropriately for aligned |
| 38 |
// vector loads |
| 39 |
#if defined(STOKHOS_ALIGN_MEMORY) && defined(STOKHOS_ASSUME_ALIGNED) && defined(__INTEL_COMPILER) && ! defined(__CUDA_ARCH__) |
| 40 |
#define STOKHOS_HAVE_PRAGMA_VECTOR_ALIGNED |
| 41 |
#endif |
| 42 |
|
| 43 |
namespace Stokhos { |
| 44 |
|
| 45 |
//! Traits class encapsulting memory alignment |
| 46 |
template <typename MemorySpace> |
| 47 |
struct MemoryTraits { |
| 48 |
|
| 49 |
//! Bytes to which memory allocations are aligned |
| 50 |
static const unsigned Alignment = 8; |
| 51 |
|
| 52 |
//! Allocate aligned memory of given size |
| 53 |
KOKKOS_INLINE_FUNCTION |
| 54 |
static void* alloc(const size_t size) { return operator new(size); } |
| 55 |
|
| 56 |
//! Free memory allocated by alloc() |
| 57 |
KOKKOS_INLINE_FUNCTION |
| 58 |
static void free(void *ptr) { operator delete(ptr); } |
| 59 |
}; |
| 60 |
|
| 61 |
//! Specialization of MemoryTraits for host memory spaces |
| 62 |
template <> |
| 63 |
struct MemoryTraits< Kokkos::HostSpace > { |
| 64 |
|
| 65 |
//! Bytes to which memory allocations are aligned |
| 66 |
#if STOKHOS_ALIGN_MEMORY |
| 67 |
#if defined(__MIC__) |
| 68 |
static const unsigned Alignment = 64; |
| 69 |
#elif defined(__AVX__) |
| 70 |
static const unsigned Alignment = 32; |
| 71 |
#elif defined(__SSE2__) |
| 72 |
static const unsigned Alignment = 16; |
| 73 |
#else |
| 74 |
static const unsigned Alignment = 8; |
| 75 |
#endif |
| 76 |
#else |
| 77 |
static const unsigned Alignment = 8; |
| 78 |
#endif |
| 79 |
|
| 80 |
//! Allocate aligned memory |
| 81 |
/*! |
| 82 |
* Note: We don't use mm_malloc or posix_memalign, because even though this |
| 83 |
* implementation is host-only, it is potentially callable from host functions |
| 84 |
* marked as device functions (via the KOKKOS_INLINE_FUNCTION maco). |
| 85 |
* |
| 86 |
* Also, we can't call new/delete as we may replace those with a version |
| 87 |
* that calls this. |
| 88 |
*/ |
| 89 |
KOKKOS_INLINE_FUNCTION |
| 90 |
static void* alloc(const size_t size) { |
| 91 |
void* ptr = 0; |
| 92 |
if (size > 0) { |
| 93 |
#if STOKHOS_ALIGN_MEMORY |
| 94 |
const size_t mask = Alignment-1; |
| 95 |
const size_t total_size = size + mask + sizeof(std::ptrdiff_t); |
| 96 |
char *ptr_alloc = reinterpret_cast<char*>(std::malloc(total_size)); |
| 97 |
char *ptr_storage = ptr_alloc + sizeof(std::ptrdiff_t); |
| 98 |
char *ptr_body = reinterpret_cast<char*>( |
| 99 |
( reinterpret_cast<size_t>(ptr_storage) + mask ) & ~mask ); |
| 100 |
char *ptr_header = ptr_body - sizeof(std::ptrdiff_t); |
| 101 |
const std::ptrdiff_t offset = ptr_body - ptr_alloc; |
| 102 |
*reinterpret_cast<std::ptrdiff_t*>(ptr_header) = offset; |
| 103 |
ptr = reinterpret_cast<void*>(ptr_body); |
| 104 |
#else |
| 105 |
ptr = operator new(size); |
| 106 |
#endif |
| 107 |
} |
| 108 |
return ptr; |
| 109 |
} |
| 110 |
|
| 111 |
//! Free memory allocated by alloc() |
| 112 |
KOKKOS_INLINE_FUNCTION |
| 113 |
static void free(void *ptr) { |
| 114 |
if (ptr != 0) { |
| 115 |
#if STOKHOS_ALIGN_MEMORY |
| 116 |
void *ptr_header = reinterpret_cast<char*>(ptr) - sizeof(std::ptrdiff_t); |
| 117 |
const std::ptrdiff_t offset = *reinterpret_cast<std::ptrdiff_t*>(ptr_header); |
| 118 |
void *ptr_alloc = reinterpret_cast<char*>(ptr) - offset; |
| 119 |
std::free(ptr_alloc); |
| 120 |
#else |
| 121 |
operator delete(ptr); |
| 122 |
#endif |
| 123 |
} |
| 124 |
} |
| 125 |
}; |
| 126 |
|
| 127 |
//! An aligned STL allocator |
| 128 |
template <typename T> |
| 129 |
class aligned_allocator { |
| 130 |
public: |
| 131 |
|
| 132 |
typedef T value_type; |
| 133 |
typedef T* pointer; |
| 134 |
typedef const T* const_pointer; |
| 135 |
typedef T& reference; |
| 136 |
typedef const T& const_reference; |
| 137 |
typedef size_t size_type; |
| 138 |
typedef std::ptrdiff_t difference_type; |
| 139 |
|
| 140 |
typedef Stokhos::MemoryTraits< Kokkos::HostSpace > Traits; |
| 141 |
|
| 142 |
template <class U> struct rebind { typedef aligned_allocator<U> other; }; |
| 143 |
|
| 144 |
aligned_allocator() {} |
| 145 |
|
| 146 |
template <class U> aligned_allocator(const aligned_allocator<U>&) {} |
| 147 |
|
| 148 |
size_type max_size() const { |
| 149 |
return (size_type(~0) - size_type(Traits::Alignment)) / sizeof(T); |
| 150 |
} |
| 151 |
|
| 152 |
pointer address(reference x) const { return &x; } |
| 153 |
|
| 154 |
const_pointer address(const_reference x) const { return &x; } |
| 155 |
|
| 156 |
pointer allocate(size_type n, const void* = 0) { |
| 157 |
size_type count = n * sizeof(T); |
| 158 |
void* ptr = Traits::alloc(count); |
| 159 |
if (ptr == 0) throw std::bad_alloc(); |
| 160 |
return reinterpret_cast<pointer>(ptr); |
| 161 |
} |
| 162 |
|
| 163 |
void deallocate(pointer p, size_type) { Traits::free(p); } |
| 164 |
|
| 165 |
void construct(pointer p, const_reference val) { new (p) T(val); } |
| 166 |
|
| 167 |
void destroy(pointer p) { ((T*)p)->~T(); } |
| 168 |
}; |
| 169 |
|
| 170 |
//! An aligned STL allocator |
| 171 |
template <typename T> |
| 172 |
class aligned_allocator< const T > { |
| 173 |
public: |
| 174 |
|
| 175 |
typedef T value_type; |
| 176 |
typedef const T* pointer; |
| 177 |
typedef const T* const_pointer; |
| 178 |
typedef const T& reference; |
| 179 |
typedef const T& const_reference; |
| 180 |
typedef size_t size_type; |
| 181 |
typedef std::ptrdiff_t difference_type; |
| 182 |
|
| 183 |
typedef Stokhos::MemoryTraits< Kokkos::HostSpace > Traits; |
| 184 |
|
| 185 |
template <class U> struct rebind { typedef aligned_allocator<U> other; }; |
| 186 |
|
| 187 |
aligned_allocator() {} |
| 188 |
|
| 189 |
template <class U> aligned_allocator(const aligned_allocator<U>&) {} |
| 190 |
|
| 191 |
size_type max_size() const { |
| 192 |
return (size_type(~0) - size_type(Traits::Alignment)) / sizeof(T); |
| 193 |
} |
| 194 |
|
| 195 |
const_pointer address(const_reference x) const { return &x; } |
| 196 |
|
| 197 |
pointer allocate(size_type n, const void* = 0) { |
| 198 |
size_type count = n * sizeof(T); |
| 199 |
void* ptr = Traits::alloc(count); |
| 200 |
if (ptr == 0) throw std::bad_alloc(); |
| 201 |
return reinterpret_cast<pointer>(ptr); |
| 202 |
} |
| 203 |
|
| 204 |
void deallocate(pointer p, size_type) { Traits::free(p); } |
| 205 |
|
| 206 |
void construct(pointer p, const_reference val) { new (p) T(val); } |
| 207 |
|
| 208 |
void destroy(pointer p) { ((T*)p)->~T(); } |
| 209 |
}; |
| 210 |
|
| 211 |
template <typename T, typename U> |
| 212 |
inline bool |
| 213 |
operator == (const aligned_allocator<T>&, const aligned_allocator<U>&) |
| 214 |
{ return true; } |
| 215 |
|
| 216 |
template <typename T, typename U> |
| 217 |
inline bool |
| 218 |
operator != (const aligned_allocator<T>&, const aligned_allocator<U>&) |
| 219 |
{ return false; } |
| 220 |
|
| 221 |
} // namespace Stokhos |
| 222 |
|
| 223 |
#endif // STOKHOS_MEMORY_TRAITS_HPP |