| 1 |
//@HEADER |
| 2 |
// ************************************************************************ |
| 3 |
// |
| 4 |
// Kokkos v. 4.0 |
| 5 |
// Copyright (2022) National Technology & Engineering |
| 6 |
// Solutions of Sandia, LLC (NTESS). |
| 7 |
// |
| 8 |
// Under the terms of Contract DE-NA0003525 with NTESS, |
| 9 |
// the U.S. Government retains certain rights in this software. |
| 10 |
// |
| 11 |
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. |
| 12 |
// See https://kokkos.org/LICENSE for license information. |
| 13 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 14 |
// |
| 15 |
//@HEADER |
| 16 |
|
| 17 |
#include <Teuchos_UnitTestHarness.hpp> |
| 18 |
#include <KokkosCompat_TMM.hpp> |
| 19 |
#include <Teuchos_Array.hpp> |
| 20 |
|
| 21 |
namespace TestTeuchosKokkosCompat { |
| 22 |
|
| 23 |
// Test functor, that fills the given 1-D vector in parallel. The |
| 24 |
// functor is like APL's iota, except that it starts with an initial |
| 25 |
// constant instead of with zero. That makes it more "interesting" |
| 26 |
// so that tests are more likely to catch bugs. |
| 27 |
template<class Device> |
| 28 |
class FillFunctor { |
| 29 |
public: |
| 30 |
FillFunctor (const Kokkos::View<double*, Device>& vec) : vec_ (vec) {} |
| 31 |
KOKKOS_INLINE_FUNCTION void operator () (const int i) const { |
| 32 |
vec_[i] = 42.0 + static_cast<double> (i); |
| 33 |
} |
| 34 |
private: |
| 35 |
Kokkos::View<double*, Device> vec_; |
| 36 |
}; |
| 37 |
|
| 38 |
// Custom deallocator for Teuchos::ArrayRCP. It doesn't actually |
| 39 |
// deallocate the ArrayRCP's data (which belongs to the View passed |
| 40 |
// into Deallocator's constructor, not to the ArrayRCP). Instead, |
| 41 |
// it just keeps a reference to the View. That way, the data won't |
| 42 |
// go away until the View's reference count goes to zero. |
| 43 |
template<class Scalar, class ViewType> |
| 44 |
class Deallocator { |
| 45 |
public: |
| 46 |
// Constructor takes the View that owns the memory. |
| 47 |
Deallocator (const ViewType& view) : view_ (view) {} |
| 48 |
|
| 49 |
// "Deallocation function" doesn't actually deallocate its input |
| 50 |
// pointer; the View is responsible for deallocation of its |
| 51 |
// memory. |
| 52 |
void free (Scalar*) {} |
| 53 |
private: |
| 54 |
ViewType view_; // View that owns the memory |
| 55 |
}; |
| 56 |
|
| 57 |
} // namespace TestTeuchosKokkosCompat |
| 58 |
|
| 59 |
// Just test whether Teuchos memory management objects and Kokkos |
| 60 |
// Array Views can coexist in the same program. This test does not |
| 61 |
// have the Teuchos and Kokkos objects interact with each other. |
| 62 |
TEUCHOS_UNIT_TEST( LinkTeuchosAndKokkos, NoInteraction ) { |
| 63 |
typedef Teuchos::Array<double>::size_type size_type; |
| 64 |
|
| 65 |
const size_type numElts = 10; |
| 66 |
Teuchos::Array<double> x (numElts); |
| 67 |
for (size_type k = 0; k < numElts; ++k) { |
| 68 |
x[k] = 42.0 + static_cast<double> (k); |
| 69 |
} |
| 70 |
Teuchos::ArrayView<double> xView = x.view (3, 5); // view of [3, 4, 5, 6, 7] |
| 71 |
TEST_EQUALITY_CONST( xView.size(), 5 ); |
| 72 |
for (size_type k = 0; k < xView.size (); ++k) { |
| 73 |
TEST_EQUALITY( xView[k], x[k+3] ); |
| 74 |
} |
| 75 |
|
| 76 |
typedef Kokkos::View<double*> ka_view_type; |
| 77 |
ka_view_type y ("y", numElts); |
| 78 |
|
| 79 |
using execution_space = ka_view_type::execution_space; |
| 80 |
Kokkos::RangePolicy<execution_space, int> range (0, y.extent (0)); |
| 81 |
using functor_type = TestTeuchosKokkosCompat::FillFunctor<execution_space>; |
| 82 |
Kokkos::parallel_for ("FillFunctor", range, functor_type (y)); |
| 83 |
} |
| 84 |
|
| 85 |
|
| 86 |
// Get a Teuchos::ArrayView of a Kokkos::View, and make sure that |
| 87 |
// it points to the same data. |
| 88 |
TEUCHOS_UNIT_TEST( LinkTeuchosAndKokkos, ArrayViewOfView ) { |
| 89 |
typedef Teuchos::Array<double>::size_type size_type; |
| 90 |
typedef Kokkos::View<double*> ka_view_type; |
| 91 |
|
| 92 |
const size_type numElts = 10; |
| 93 |
ka_view_type y ("y", numElts); |
| 94 |
|
| 95 |
using execution_space = ka_view_type::execution_space; |
| 96 |
Kokkos::RangePolicy<execution_space, int> range (0, y.extent (0)); |
| 97 |
using functor_type = TestTeuchosKokkosCompat::FillFunctor<execution_space>; |
| 98 |
Kokkos::parallel_for ("FillFunctor", range, functor_type (y)); |
| 99 |
|
| 100 |
auto y_host = Kokkos::create_mirror_view (y); |
| 101 |
Kokkos::deep_copy (y_host, y); |
| 102 |
|
| 103 |
Teuchos::ArrayView<double> y_view (y_host.data (), y_host.extent (0)); |
| 104 |
TEST_EQUALITY_CONST( size_t (y_view.size ()), size_t (y_host.extent (0)) ); |
| 105 |
if (success) { |
| 106 |
for (size_type k = 0; k < size_type (y_host.extent (0)); ++k) { |
| 107 |
TEST_EQUALITY( y_view[k], y_host[k] ); |
| 108 |
} |
| 109 |
} |
| 110 |
} |
| 111 |
|
| 112 |
template<class DataType, class LayoutType = Kokkos::LayoutLeft> |
| 113 |
using ManagedHostView = |
| 114 |
Kokkos::View<DataType, LayoutType, |
| 115 |
Kokkos::Device<Kokkos::DefaultHostExecutionSpace, Kokkos::HostSpace> >; |
| 116 |
|
| 117 |
template<class DataType, class LayoutType = Kokkos::LayoutLeft> |
| 118 |
using UnmanagedHostView = |
| 119 |
Kokkos::View<DataType, LayoutType, |
| 120 |
Kokkos::Device<Kokkos::DefaultHostExecutionSpace, Kokkos::HostSpace>, |
| 121 |
Kokkos::MemoryTraits<Kokkos::Unmanaged> >; |
| 122 |
|
| 123 |
// Get a Kokkos::View of a Teuchos::ArrayView, and make sure that it |
| 124 |
// points to the same data. Thanks to Christian Trott for |
| 125 |
// implementing the necessary functionality (View constructor for |
| 126 |
// certain View specializations, that takes a raw pointer and |
| 127 |
// dimensions) in Kokkos Array. |
| 128 |
// |
| 129 |
// This example will be useful for implementing the |
| 130 |
// Tpetra::MultiVector methods get1dCopy and get2dCopy. |
| 131 |
TEUCHOS_UNIT_TEST( LinkTeuchosAndKokkos, ViewOfArrayView ) { |
| 132 |
using size_type = Teuchos::Array<double>::size_type; |
| 133 |
|
| 134 |
const size_type numElts = 10; |
| 135 |
Teuchos::Array<double> x (numElts); |
| 136 |
for (size_type k = 0; k < numElts; ++k) { |
| 137 |
x[k] = 42.0 + static_cast<double> (k); |
| 138 |
} |
| 139 |
// You can make an (unmanaged) View of a raw array with left or |
| 140 |
// right (Fortran or C) layout on the HostSpace device, just by |
| 141 |
// passing the array and stride(s) into the constructor. If you |
| 142 |
// want the dimensions to differ from the strides, you'll have to |
| 143 |
// create a subview with the desired dimensions. |
| 144 |
UnmanagedHostView<double*> x_view (x.getRawPtr (), x.size ()); |
| 145 |
|
| 146 |
TEST_EQUALITY( x.size(), static_cast<size_type> (x_view.extent (0)) ); |
| 147 |
for (size_type k = 0; k < x.size (); ++k) { |
| 148 |
TEST_EQUALITY( x_view(k), x[k] ); |
| 149 |
} |
| 150 |
|
| 151 |
UnmanagedHostView<const double*> x_view_const (x.getRawPtr (), x.size ()); |
| 152 |
|
| 153 |
TEST_EQUALITY( x.size (), static_cast<size_type> (x_view_const.extent (0)) ); |
| 154 |
for (size_type k = 0; k < x.size (); ++k) { |
| 155 |
TEST_EQUALITY( x_view_const(k), x[k] ); |
| 156 |
} |
| 157 |
} |
| 158 |
|
| 159 |
// Create a 2-D View (LayoutLeft, in HostSpace memory), and then create an |
| 160 |
// ArrayRCP which owns that View (using a custom destructor). This |
| 161 |
// will be useful for implementing Tpetra::MultiVector's getData, |
| 162 |
// getDataNonConst, get1dView, and get1dViewNonConst methods. |
| 163 |
TEUCHOS_UNIT_TEST( LinkTeuchosAndKokkos, ArrayRCP1D_of_2DView ) { |
| 164 |
using ka_view_type = ManagedHostView<double**>; |
| 165 |
|
| 166 |
const size_t numRows = 75; |
| 167 |
const size_t numCols = 5; |
| 168 |
const size_t stride = 100; |
| 169 |
const size_t ZERO = static_cast<size_t> (0); |
| 170 |
|
| 171 |
ka_view_type X ("X", stride, numCols); |
| 172 |
ka_view_type X_view = Kokkos::subview (X, std::make_pair (ZERO, numRows), |
| 173 |
std::make_pair (ZERO, numCols)); |
| 174 |
TEST_EQUALITY(X_view.extent(0), numRows); |
| 175 |
TEST_EQUALITY(X_view.extent(1), numCols); |
| 176 |
|
| 177 |
// Test that the strides of X_view are correct, for int. Kokkos |
| 178 |
// Array templates the stride() method on the integer type of the |
| 179 |
// input. I just want to make sure that this works for different |
| 180 |
// integer types; I'll test int here and size_t. |
| 181 |
{ |
| 182 |
int strides[3] = { 0 , 0 , 0 }; |
| 183 |
X_view.stride (strides); |
| 184 |
TEST_EQUALITY_CONST(strides[0], 1); // stride between X_view(i,j) and X_view(i+1,j) |
| 185 |
// The stride must be at least as given, but can be greater (due to possible padding for alignment). |
| 186 |
TEST_ASSERT(static_cast<size_t>(strides[1]) >= stride); // stride between X_view(i,j) and X_view(i,j+1) |
| 187 |
} |
| 188 |
|
| 189 |
// Test that the strides of X_view are correct, for size_t. |
| 190 |
{ |
| 191 |
size_t strides[3] = { 0 , 0 , 0 }; |
| 192 |
X_view.stride (strides); |
| 193 |
TEST_EQUALITY_CONST(strides[0], static_cast<size_t>(1)); // stride between X_view(i,j) and X_view(i+1,j) |
| 194 |
// The stride must be at least as given, but can be greater (due to possible padding for alignment). |
| 195 |
TEST_ASSERT(strides[1] >= stride); // stride between X_view(i,j) and X_view(i,j+1) |
| 196 |
} |
| 197 |
|
| 198 |
// Create a nonowning "view" of X's data. The ArrayRCP has a custom |
| 199 |
// destructor which simply holds a reference to the View and doesn't |
| 200 |
// actually deallocate memory. That way, the ArrayRCP can use the |
| 201 |
// View's raw pointer, but still defers to the View for memory |
| 202 |
// management. |
| 203 |
|
| 204 |
using dealloc_type = TestTeuchosKokkosCompat::Deallocator<double, ka_view_type>; |
| 205 |
Teuchos::ArrayRCP<double> Y_values (X.data (), 0, stride*numCols, |
| 206 |
dealloc_type (X), true); |
| 207 |
TEST_EQUALITY(Y_values.getRawPtr(), X.data()); |
| 208 |
TEST_EQUALITY(Y_values.getRawPtr(), X_view.data()); |
| 209 |
TEST_EQUALITY(Y_values.size(), stride*numCols); |
| 210 |
} |
| 211 |
|
| 212 |
|
| 213 |
// Create a 2-D View (LayoutLeft, in HostSpace memory), and then create an |
| 214 |
// ArrayRCP<ArrayRCP<double> >, each entry of which owns the |
| 215 |
// corresponding column of that View (using a custom destructor). |
| 216 |
// This will be useful for implementing Tpetra::MultiVector's |
| 217 |
// get2dView and get2dViewNonConst methods. |
| 218 |
TEUCHOS_UNIT_TEST( LinkTeuchosAndKokkos, ArrayRCP2D_of_2DView ) { |
| 219 |
// View<double*, LayoutLeft, ...> and View<double*, LayoutRight, |
| 220 |
// ...> always have unit stride. Furthermore, subview(X,j) returns |
| 221 |
// a view of a column for LayoutLeft, and a view of a row for |
| 222 |
// LayoutRight. Thus, to support both layouts, each column of the |
| 223 |
// multivector still needs to be a View<double**, ...>, and we have |
| 224 |
// to use the subview() overload that takes two ranges of row and |
| 225 |
// column indices. |
| 226 |
using ka_view_type = ManagedHostView<double**>; |
| 227 |
|
| 228 |
const size_t numRows = 75; |
| 229 |
const size_t numCols = 5; |
| 230 |
const size_t stride = 100; |
| 231 |
const size_t ZERO = static_cast<size_t> (0); |
| 232 |
|
| 233 |
ka_view_type X ("X", stride, numCols); |
| 234 |
ka_view_type X_view = Kokkos::subview (X, std::make_pair (ZERO, numRows), |
| 235 |
std::make_pair (ZERO, numCols)); |
| 236 |
TEST_EQUALITY( & X(0,0) , & X_view(0,0) ); |
| 237 |
TEST_EQUALITY(X_view.extent(0), numRows); |
| 238 |
TEST_EQUALITY(X_view.extent(1), numCols); |
| 239 |
|
| 240 |
// Test that the strides of X_view are correct, for size_t. |
| 241 |
{ |
| 242 |
size_t strides[3]; // stride[rank] is the maximum span |
| 243 |
strides[0] = static_cast<size_t> (0); |
| 244 |
strides[1] = static_cast<size_t> (0); |
| 245 |
strides[2] = static_cast<size_t> (0); |
| 246 |
X_view.stride (strides); |
| 247 |
TEST_EQUALITY_CONST(strides[0], static_cast<size_t>(1)); // stride between X_view(i,j) and X_view(i+1,j) |
| 248 |
// The stride must be at least as given, but can be greater (due to possible padding for alignment). |
| 249 |
TEST_ASSERT(strides[1] >= stride); // stride between X_view(i,j) and X_view(i,j+1) |
| 250 |
} |
| 251 |
|
| 252 |
// Make a 2-D "view" (array of arrays) of X_view. This is how we |
| 253 |
// will implement Tpetra::MultiVector methods like get2dView. |
| 254 |
Teuchos::ArrayRCP<Teuchos::ArrayRCP<double> > Y_2D (X_view.extent (1)); |
| 255 |
for (size_t j = 0; j < static_cast<size_t> (X_view.extent (1)); ++j) { |
| 256 |
ka_view_type X_j = Kokkos::subview (X_view, std::make_pair (ZERO, numRows), |
| 257 |
std::make_pair (j, j+1)); |
| 258 |
TEST_EQUALITY( & X_view(0,j) , & X_j(0,0) ); |
| 259 |
TEST_EQUALITY(static_cast<size_t>(X_j.extent(0)), numRows); |
| 260 |
TEST_EQUALITY_CONST(X_j.extent(1), 1); |
| 261 |
|
| 262 |
// Test that the strides of X_j are correct. |
| 263 |
{ |
| 264 |
size_t strides[3]; // stride[rank] is the maximum span |
| 265 |
strides[0] = static_cast<size_t> (0); |
| 266 |
strides[1] = static_cast<size_t> (0); |
| 267 |
strides[2] = static_cast<size_t> (0); |
| 268 |
X_j.stride (strides); |
| 269 |
TEST_EQUALITY_CONST(strides[0], static_cast<size_t>(1)); // stride between X_j(i,j) and X_j(i+1,j) |
| 270 |
// Stride between X_j(i,j) and X_j(i,j+1), even though X_j only |
| 271 |
// has one column. The stride must be at least as given, but |
| 272 |
// can be greater (due to possible padding for alignment). |
| 273 |
TEST_ASSERT(strides[1] >= stride); |
| 274 |
} |
| 275 |
|
| 276 |
// Create a nonowning "view" of X_j's data. The ArrayRCP has a |
| 277 |
// custom destructor which simply holds a reference to the View |
| 278 |
// X_j, and doesn't actually deallocate memory. That way, the |
| 279 |
// ArrayRCP can use the View's raw pointer, but still defers to |
| 280 |
// the View for memory management. |
| 281 |
using dealloc_type = TestTeuchosKokkosCompat::Deallocator<double, ka_view_type>; |
| 282 |
Teuchos::ArrayRCP<double> Y_j (X_j.data (), 0, numRows, |
| 283 |
dealloc_type (X_j), true); |
| 284 |
Y_2D[j] = Y_j; |
| 285 |
} |
| 286 |
} |
| 287 |
|
| 288 |
|
| 289 |
int |
| 290 |
main (int argc, char* argv[]) |
| 291 |
{ |
| 292 |
// Initialize MPI (if enabled) before initializing Kokkos. This |
| 293 |
// lets MPI control things like pinning processes to sockets. |
| 294 |
Teuchos::GlobalMPISession mpiSession (&argc, &argv); |
| 295 |
Kokkos::ScopeGuard kokkosScope (argc, argv); |
| 296 |
const int errCode = |
| 297 |
Teuchos::UnitTestRepository::runUnitTestsFromMain (argc, argv); |
| 298 |
return errCode; |
| 299 |
} |