/* * (C) Copyright 2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include #include #include #include "atlas/array.h" #include "atlas/array/ArrayDataStore.h" #include "atlas/array/ArrayView.h" #include "atlas/array/DataType.h" #include "atlas/array/MakeView.h" #include "atlas/array/gridtools/GridToolsArrayHelpers.h" #include "atlas/array/gridtools/GridToolsDataStore.h" #include "atlas/array/gridtools/GridToolsTraits.h" #include "atlas/array/helpers/ArrayInitializer.h" #include "atlas/array_fwd.h" #include "atlas/runtime/Log.h" #if ATLAS_HAVE_ACC #include "atlas_acc_support/atlas_acc.h" #endif //------------------------------------------------------------------------------ using namespace atlas::array::gridtools; using namespace atlas::array::helpers; namespace atlas { namespace array { //------------------------------------------------------------------------------ namespace gridtools { //------------------------------------------------------------------------------ template using UintSequence = std::make_integer_sequence; template class Storage, typename StorageInfo> static Array* wrap_array(::gridtools::data_store, StorageInfo>* ds, const ArraySpec& spec) { assert(ds); using data_store_t = typename std::remove_pointer::type; ArrayDataStore* data_store = new GridToolsDataStore(ds); return new ArrayT(data_store, spec); } //------------------------------------------------------------------------------ } // namespace gridtools template class ArrayT_impl { public: ArrayT_impl(ArrayT& array): array_(array) {} template > void construct(UInts... dims) { static_assert(sizeof...(UInts) > 0, "1"); auto gt_storage = create_gt_storage::type>(dims...); using data_store_t = typename std::remove_pointer::type; array_.data_store_ = std::make_unique>(gt_storage); array_.spec_ = make_spec(gt_storage, dims...); } template > void construct_aligned(UInts... dims) { static_assert(sizeof...(UInts) > 0, "1"); auto gt_storage = create_gt_storage::type, ::gridtools::alignment>(dims...); using data_store_t = typename std::remove_pointer::type; array_.data_store_ = std::make_unique>(gt_storage); array_.spec_ = make_spec(gt_storage, dims...); } template > void construct(ArrayAlignment alignment, UInts... dims) { static_assert(sizeof...(UInts) > 0, "1"); switch (int(alignment)) { case 1: construct(dims...); break; case 2: construct_aligned<2>(dims...); break; case 4: construct_aligned<4>(dims...); break; case 8: construct_aligned<8>(dims...); break; case 16: construct_aligned<16>(dims...); break; case 32: construct_aligned<32>(dims...); break; case 64: construct_aligned<64>(dims...); break; default: ATLAS_NOTIMPLEMENTED; } } void construct(const ArrayShape& shape) { assert(shape.size() > 0); switch (shape.size()) { case 1: return construct(shape[0]); case 2: return construct(shape[0], shape[1]); case 3: return construct(shape[0], shape[1], shape[2]); case 4: return construct(shape[0], shape[1], shape[2], shape[3]); case 5: return construct(shape[0], shape[1], shape[2], shape[3], shape[4]); default: { std::stringstream err; err << "shape not recognized"; throw_Exception(err.str(), Here()); } } } void construct(const ArraySpec& spec) { auto& shape = spec.shape(); assert(shape.size() > 0); switch (shape.size()) { case 1: return construct(spec.alignment(), shape[0]); case 2: return construct(spec.alignment(), shape[0], shape[1]); case 3: return construct(spec.alignment(), shape[0], shape[1], shape[2]); case 4: return construct(spec.alignment(), shape[0], shape[1], shape[2], shape[3]); case 5: return construct(spec.alignment(), shape[0], shape[1], shape[2], shape[3], shape[4]); default: { std::stringstream err; err << "shape not recognized"; throw_Exception(err.str(), Here()); } } } void construct(const ArrayShape& shape, const ArrayLayout& layout) { ATLAS_ASSERT(shape.size() > 0); ATLAS_ASSERT(shape.size() == layout.size()); switch (shape.size()) { case 1: return construct(shape[0]); case 2: { if (layout[0] == 0 && layout[1] == 1) { return construct_with_layout<::gridtools::layout_map<0, 1>>(shape[0], shape[1]); } if (layout[0] == 1 && layout[1] == 0) { return construct_with_layout<::gridtools::layout_map<1, 0>>(shape[0], shape[1]); } } case 3: { if (layout[0] == 0 && layout[1] == 1 && layout[2] == 2) { return construct_with_layout<::gridtools::layout_map<0, 1, 2>>(shape[0], shape[1], shape[2]); } if (layout[0] == 0 && layout[1] == 2 && layout[2] == 1) { return construct_with_layout<::gridtools::layout_map<0, 2, 1>>(shape[0], shape[1], shape[2]); } if (layout[0] == 1 && layout[1] == 0 && layout[2] == 2) { return construct_with_layout<::gridtools::layout_map<1, 0, 2>>(shape[0], shape[1], shape[2]); } if (layout[0] == 1 && layout[1] == 2 && layout[2] == 0) { return construct_with_layout<::gridtools::layout_map<1, 2, 0>>(shape[0], shape[1], shape[2]); } if (layout[0] == 2 && layout[1] == 0 && layout[2] == 1) { return construct_with_layout<::gridtools::layout_map<2, 0, 1>>(shape[0], shape[1], shape[2]); } if (layout[0] == 2 && layout[1] == 1 && layout[2] == 0) { return construct_with_layout<::gridtools::layout_map<2, 1, 0>>(shape[0], shape[1], shape[2]); } } case 4: { if (layout[0] == 0 && layout[1] == 1 && layout[2] == 2 && layout[3] == 3) { return construct_with_layout<::gridtools::layout_map<0, 1, 2, 3>>(shape[0], shape[1], shape[2], shape[3]); } if (layout[0] == 3 && layout[1] == 2 && layout[2] == 1 && layout[3] == 0) { return construct_with_layout<::gridtools::layout_map<3, 2, 1, 0>>(shape[0], shape[1], shape[2], shape[3]); } } case 5: { if (layout[0] == 0 && layout[1] == 1 && layout[2] == 2 && layout[3] == 3 && layout[4] == 4) { return construct_with_layout<::gridtools::layout_map<0, 1, 2, 3, 4>>(shape[0], shape[1], shape[2], shape[3], shape[4]); } if (layout[0] == 4 && layout[1] == 3 && layout[2] == 2 && layout[3] == 1 && layout[4] == 0) { return construct_with_layout<::gridtools::layout_map<4, 3, 2, 1, 0>>(shape[0], shape[1], shape[2], shape[3], shape[4]); } } default: { std::stringstream err; if (shape.size() > 5) { err << "shape not recognized"; } else { err << "Layout < "; for (size_t j = 0; j < layout.size(); ++j) { err << layout[j] << " "; } err << "> not implemented in Atlas."; } throw_Exception(err.str(), Here()); } } } template > void construct_with_layout(UInts... dims) { auto gt_data_store_ptr = create_gt_storage(dims...); using data_store_t = typename std::remove_pointer::type; array_.data_store_ = std::make_unique>(gt_data_store_ptr); array_.spec_ = make_spec(gt_data_store_ptr, dims...); } template void resize_variadic(Ints... c) { if (sizeof...(c) != array_.rank()) { std::stringstream err; err << "Trying to resize an array of Rank " << array_.rank() << " by dimensions with Rank " << sizeof...(c) << std::endl; throw_Exception(err.str(), Here()); } if (array_.valid()) { array_.syncHostDevice(); } Array* resized = Array::create(ArrayShape{(idx_t)c...}); array_initializer::apply(array_, *resized); array_.replace(*resized); delete resized; } template void apply_resize(const ArrayShape& shape, std::integer_sequence) { return resize_variadic(shape[Indices]...); } private: ArrayT& array_; }; //------------------------------------------------------------------------------ template Array* Array::create(idx_t dim0) { return new ArrayT(dim0); } template Array* Array::create(idx_t dim0, idx_t dim1) { return new ArrayT(dim0, dim1); } template Array* Array::create(idx_t dim0, idx_t dim1, idx_t dim2) { return new ArrayT(dim0, dim1, dim2); } template Array* Array::create(idx_t dim0, idx_t dim1, idx_t dim2, idx_t dim3) { return new ArrayT(dim0, dim1, dim2, dim3); } template Array* Array::create(idx_t dim0, idx_t dim1, idx_t dim2, idx_t dim3, idx_t dim4) { return new ArrayT(dim0, dim1, dim2, dim3, dim4); } template Array* Array::create(const ArrayShape& shape) { return new ArrayT(shape); } template Array* Array::create(const ArrayShape& shape, const ArrayLayout& layout) { return new ArrayT(shape, layout); } template Array* Array::wrap(Value* data, const ArraySpec& spec) { ArrayShape const& shape = spec.shape(); ArrayStrides const& strides = spec.strides(); assert(shape.size() > 0); switch (shape.size()) { case 1: return wrap_array( wrap_gt_storage(data, get_array_from_vector<1>(shape), get_array_from_vector<1>(strides)), spec); case 2: return wrap_array( wrap_gt_storage(data, get_array_from_vector<2>(shape), get_array_from_vector<2>(strides)), spec); case 3: return wrap_array( wrap_gt_storage(data, get_array_from_vector<3>(shape), get_array_from_vector<3>(strides)), spec); case 4: return wrap_array( wrap_gt_storage(data, get_array_from_vector<4>(shape), get_array_from_vector<4>(strides)), spec); case 5: return wrap_array( wrap_gt_storage(data, get_array_from_vector<5>(shape), get_array_from_vector<5>(strides)), spec); case 6: return wrap_array( wrap_gt_storage(data, get_array_from_vector<6>(shape), get_array_from_vector<6>(strides)), spec); case 7: return wrap_array( wrap_gt_storage(data, get_array_from_vector<7>(shape), get_array_from_vector<7>(strides)), spec); case 8: return wrap_array( wrap_gt_storage(data, get_array_from_vector<8>(shape), get_array_from_vector<8>(strides)), spec); case 9: return wrap_array( wrap_gt_storage(data, get_array_from_vector<9>(shape), get_array_from_vector<9>(strides)), spec); default: { std::stringstream err; err << "shape not recognized"; throw_Exception(err.str(), Here()); } } } template Array* Array::wrap(Value* data, const ArrayShape& shape) { return wrap(data, ArraySpec(shape)); } Array* Array::create(DataType datatype, const ArrayShape& shape) { switch (datatype.kind()) { case DataType::KIND_REAL64: return create(shape); case DataType::KIND_REAL32: return create(shape); case DataType::KIND_INT32: return create(shape); case DataType::KIND_INT64: return create(shape); case DataType::KIND_UINT64: return create(shape); default: { std::stringstream err; err << "data kind " << datatype.kind() << " not recognised."; throw_Exception(err.str(), Here()); } } } Array* Array::create(DataType datatype, const ArrayShape& shape, const ArrayLayout& layout) { switch (datatype.kind()) { case DataType::KIND_REAL64: return create(shape, layout); case DataType::KIND_REAL32: return create(shape, layout); case DataType::KIND_INT32: return create(shape, layout); case DataType::KIND_INT64: return create(shape, layout); case DataType::KIND_UINT64: return create(shape, layout); default: { std::stringstream err; err << "data kind " << datatype.kind() << " not recognised."; throw_Exception(err.str(), Here()); } } } Array* Array::create(DataType datatype, ArraySpec&& spec) { switch (datatype.kind()) { case DataType::KIND_REAL64: return new ArrayT(std::move(spec)); case DataType::KIND_REAL32: return new ArrayT(std::move(spec)); case DataType::KIND_INT32: return new ArrayT(std::move(spec)); case DataType::KIND_INT64: return new ArrayT(std::move(spec)); case DataType::KIND_UINT64: return new ArrayT(std::move(spec)); default: { std::stringstream err; err << "data kind " << datatype.kind() << " not recognised."; throw_NotImplemented(err.str(), Here()); } } } Array* Array::create(ArraySpec&& spec) { return create(spec.datatype(), std::move(spec)); } //------------------------------------------------------------------------------ Array::~Array() = default; //------------------------------------------------------------------------------ template size_t ArrayT::footprint() const { size_t size = sizeof(*this); size += bytes(); return size; } //------------------------------------------------------------------------------ template void ArrayT::accMap() const { data_store_->accMap(); } template void ArrayT::accUnmap() const { data_store_->accUnmap(); } template bool ArrayT::accMapped() const { return data_store_->accMapped(); } //------------------------------------------------------------------------------ template void ArrayT::insert(idx_t idx1, idx_t size1) { // if( hostNeedsUpdate() ) { // updateHost(); //} if (not hasDefaultLayout()) { ATLAS_NOTIMPLEMENTED; } ArrayShape nshape = shape(); if (idx1 > nshape[0]) { throw_Exception("can not insert into an array at a position beyond its size", Here()); } nshape[0] += size1; Array* resized = Array::create(nshape); array_initializer_partitioned<0>::apply(*this, *resized, idx1, size1); replace(*resized); delete resized; } //------------------------------------------------------------------------------ template void ArrayT::resize(idx_t dim0) { ArrayT_impl(*this).resize_variadic(dim0); } template void ArrayT::resize(idx_t dim0, idx_t dim1) { ArrayT_impl(*this).resize_variadic(dim0, dim1); } template void ArrayT::resize(idx_t dim0, idx_t dim1, idx_t dim2) { ArrayT_impl(*this).resize_variadic(dim0, dim1, dim2); } template void ArrayT::resize(idx_t dim0, idx_t dim1, idx_t dim2, idx_t dim3) { ArrayT_impl(*this).resize_variadic(dim0, dim1, dim2, dim3); } template void ArrayT::resize(idx_t dim0, idx_t dim1, idx_t dim2, idx_t dim3, idx_t dim4) { ArrayT_impl(*this).resize_variadic(dim0, dim1, dim2, dim3, dim4); } template void ArrayT::copy(const Array& other, const Array::CopyPolicy&) { array_initializer::apply(other, *this); } template void ArrayT::dump(std::ostream& out) const { switch (rank()) { case 1: make_host_view(*this).dump(out); break; case 2: make_host_view(*this).dump(out); break; case 3: make_host_view(*this).dump(out); break; case 4: make_host_view(*this).dump(out); break; case 5: make_host_view(*this).dump(out); break; case 6: make_host_view(*this).dump(out); break; case 7: make_host_view(*this).dump(out); break; case 8: make_host_view(*this).dump(out); break; case 9: make_host_view(*this).dump(out); break; default: ATLAS_NOTIMPLEMENTED; } } template void ArrayT::resize(const ArrayShape& shape) { assert(shape.size() > 0); switch (shape.size()) { case 1: return ArrayT_impl(*this).apply_resize(shape, UintSequence<1>()); case 2: return ArrayT_impl(*this).apply_resize(shape, UintSequence<2>()); case 3: return ArrayT_impl(*this).apply_resize(shape, UintSequence<3>()); case 4: return ArrayT_impl(*this).apply_resize(shape, UintSequence<4>()); case 5: return ArrayT_impl(*this).apply_resize(shape, UintSequence<5>()); case 6: return ArrayT_impl(*this).apply_resize(shape, UintSequence<6>()); case 7: return ArrayT_impl(*this).apply_resize(shape, UintSequence<7>()); case 8: return ArrayT_impl(*this).apply_resize(shape, UintSequence<8>()); case 9: return ArrayT_impl(*this).apply_resize(shape, UintSequence<9>()); default: { std::stringstream err; err << "shape not recognized"; throw_Exception(err.str(), Here()); } } } //------------------------------------------------------------------------------ template ArrayT::ArrayT(ArrayDataStore* ds, const ArraySpec& spec) { data_store_ = std::unique_ptr(ds); spec_ = spec; } template ArrayT::ArrayT(idx_t dim0) { ArrayT_impl(*this).construct(dim0); } template ArrayT::ArrayT(idx_t dim0, idx_t dim1) { ArrayT_impl(*this).construct(dim0, dim1); } template ArrayT::ArrayT(idx_t dim0, idx_t dim1, idx_t dim2) { ArrayT_impl(*this).construct(dim0, dim1, dim2); } template ArrayT::ArrayT(idx_t dim0, idx_t dim1, idx_t dim2, idx_t dim3) { ArrayT_impl(*this).construct(dim0, dim1, dim2, dim3); } template ArrayT::ArrayT(idx_t dim0, idx_t dim1, idx_t dim2, idx_t dim3, idx_t dim4) { ArrayT_impl(*this).construct(dim0, dim1, dim2, dim3, dim4); } template ArrayT::ArrayT(const ArrayShape& shape) { ArrayT_impl(*this).construct(shape); } template ArrayT::ArrayT(const ArrayShape& shape, const ArrayAlignment& alignment) { ArrayT_impl(*this).construct(ArraySpec(shape, alignment)); } template ArrayT::ArrayT(const ArrayShape& shape, const ArrayLayout& layout) { ArrayT_impl(*this).construct(shape, layout); } template ArrayT::ArrayT(ArraySpec&& spec) { ArrayT_impl(*this).construct(spec); } //------------------------------------------------------------------------------ } // namespace array } // namespace atlas //------------------------------------------------------------------------------ // Explicit template instantiations namespace atlas { namespace array { template class ArrayT; template class ArrayT; template class ArrayT; template class ArrayT; template class ArrayT; template Array* Array::create(idx_t); template Array* Array::create(idx_t); template Array* Array::create(idx_t); template Array* Array::create(idx_t); template Array* Array::create(idx_t); template Array* Array::create(idx_t, idx_t); template Array* Array::create(idx_t, idx_t); template Array* Array::create(idx_t, idx_t); template Array* Array::create(idx_t, idx_t); template Array* Array::create(idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t, idx_t); template Array* Array::create(idx_t, idx_t, idx_t, idx_t, idx_t); template Array* Array::create(const ArrayShape&); template Array* Array::create(const ArrayShape&); template Array* Array::create(const ArrayShape&); template Array* Array::create(const ArrayShape&); template Array* Array::create(const ArrayShape&); template Array* Array::create(const ArrayShape&, const ArrayLayout&); template Array* Array::create(const ArrayShape&, const ArrayLayout&); template Array* Array::create(const ArrayShape&, const ArrayLayout&); template Array* Array::create(const ArrayShape&, const ArrayLayout&); template Array* Array::create(const ArrayShape&, const ArrayLayout&); template Array* Array::wrap(int*, const ArrayShape&); template Array* Array::wrap(long*, const ArrayShape&); template Array* Array::wrap(float*, const ArrayShape&); template Array* Array::wrap(double*, const ArrayShape&); template Array* Array::wrap(long unsigned*, const ArrayShape&); template Array* Array::wrap(int*, const ArraySpec&); template Array* Array::wrap(long*, const ArraySpec&); template Array* Array::wrap(float*, const ArraySpec&); template Array* Array::wrap(double*, const ArraySpec&); template Array* Array::wrap(long unsigned*, const ArraySpec&); } // namespace array } // namespace atlas