Debian Codegraph @

src/atlas/array/native/NativeDataStore.h

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #pragma once
12
13 #include <algorithm> // std::fill
14 #include <limits> // std::numeric_limits<T>::signaling_NaN
15 #include <string>
16 #include <iostream>
17
18 #include "pluto/pluto.h"
19
20 #include "atlas/array/ArrayDataStore.h"
21 #include "atlas/array/ArraySpec.h"
22 #include "atlas/library.h"
23 #include "atlas/library/config.h"
24 #include "atlas/parallel/acc/acc.h"
25 #include "atlas/runtime/Exception.h"
26 #include "atlas/runtime/Trace.h"
27
28 #define ATLAS_ACC_DEBUG 0
29
30 //------------------------------------------------------------------------------
31
32 namespace atlas {
33 namespace array {
34 namespace native {
35
36 template <typename Value>
37 static constexpr Value invalid_value() {
38 return std::numeric_limits<Value>::has_signaling_NaN ? std::numeric_limits<Value>::signaling_NaN()
39 : std::numeric_limits<Value>::has_quiet_NaN ? std::numeric_limits<Value>::quiet_NaN()
40 : std::numeric_limits<Value>::has_infinity ? std::numeric_limits<Value>::infinity()
41 : std::numeric_limits<Value>::max();
42 }
43
44 #if ATLAS_INIT_SNAN
45 template <typename Value>
46 void initialise(Value array[], size_t size) {
47 std::fill_n(array, size, invalid_value<Value>());
48 }
49 #else
50 template <typename Value>
51 void initialise(Value[], size_t) {}
52 #endif
53
54 static bool test_device_is_shared() {
55 // This is currently a bit hacky, to be improved!
56 auto& device_resource = *pluto::device::get_default_resource();
57
58 if (device_resource == *pluto::pinned_resource()) {
59 return true;
60 }
61 if (device_resource == *pluto::managed_resource()) {
62 return true;
63 }
64 if (device_resource == *pluto::pinned_pool_resource()) {
65 return true;
66 }
67 if (device_resource == *pluto::managed_pool_resource()) {
68 return true;
69 }
70
71 // If above shortcuts failed, we can do an introspection on a dummy allocation
72 if (ATLAS_HAVE_GPU && pluto::devices()) {
73 void* device_ptr = device_resource.allocate(8);
74 bool is_shared = pluto::is_managed(device_ptr) || (pluto::is_pinned(device_ptr));
75 device_resource.deallocate(device_ptr,8);
76 return is_shared;
77 }
78 return false;
79 }
80
81 static pluto::memory_resource* host_resource(bool shared) {
82 if (shared) {
83 return pluto::device::get_default_resource();
84 }
85 else {
86 return pluto::host::get_default_resource();
87 }
88 }
89
90 static pluto::memory_resource* device_resource(bool shared) {
91 if (shared) {
92 return pluto::null_memory_resource();
93 }
94 else {
95 return pluto::device::get_default_resource();
96 }
97 }
98
99
100 template <typename Value>
101 class DataStore final : public ArrayDataStore {
102 public:
103 DataStore(size_t size): size_(size),
104 is_shared_data_{test_device_is_shared()},
105 host_allocator_{host_resource(is_shared_data_)},
106 device_allocator_{device_resource(is_shared_data_)} {
107
108 label_ = array::label::get();
109 allocateHost();
110 initialise(host_data_, size_);
111 if (ATLAS_HAVE_GPU && pluto::devices()) {
112 device_updated_ = false;
113 }
114 else {
115 device_data_ = host_data_;
116 }
117 }
118
119 virtual ~DataStore() {
120 deallocateDevice();
121 deallocateHost();
122 }
123
124 void updateDevice() const override {
125 if (ATLAS_HAVE_GPU && pluto::devices()) {
126 if (not device_allocated_) {
127 allocateDevice();
128 }
129 #if ATLAS_HAVE_TRACE
130 std::unique_ptr<atlas::Trace> trace;
131 if (atlas::Library::instance().traceMemory()) {
132 trace.reset(new atlas::Trace(Here(), "update_device", {"memory","memory.transfer", "memory.transfer.h2d"}));
133 }
134 #endif
135 pluto::copy_host_to_device(label_, device_data_, host_data_, size_);
136 device_updated_ = true;
137 }
138 }
139
140 void updateHost() const override {
141 if constexpr (ATLAS_HAVE_GPU) {
142 if (device_allocated_) {
143 #if ATLAS_HAVE_TRACE
144 std::unique_ptr<atlas::Trace> trace;
145 if (atlas::Library::instance().traceMemory()) {
146 trace.reset(new atlas::Trace(Here(), "update_host", {"memory","memory.transfer", "memory.transfer.d2h"}));
147 }
148 #endif
149 pluto::copy_device_to_host(label_, host_data_, device_data_, size_);
150 host_updated_ = true;
151 }
152 }
153 }
154
155 bool valid() const override { return true; }
156
157 void syncHostDevice() const override {
158 if (host_updated_ and device_updated_) {
159 return; // nothing to do
160 }
161 if (not (host_updated_ or device_updated_)) {
162 throw_AssertionFailed("syncHostDevice() could not figure out which of host or device is up to date. "
163 "Probably it was forgotten to use setDeviceNeedsUpdate(true) or setDeviceNeedsUpdate(true)",
164 Here());
165 }
166
167 if (not device_updated_) {
168 updateDevice();
169 }
170 else if (not host_updated_) {
171 updateHost();
172 }
173 }
174
175 void syncHost() const override {
176 if (not host_updated_) {
177 ATLAS_ASSERT(device_updated_,"Unexpected state when calling syncHost() because (deviceNeedsUpdate() and hostNeedsUpdate())");
178 updateHost();
179 }
180 }
181
182 void syncDevice() const override {
183 if (not device_updated_) {
184 ATLAS_ASSERT(host_updated_,"Unexpected state when calling syncDevice() because (deviceNeedsUpdate() and hostNeedsUpdate())");
185 if (not host_updated_) {
186 throw_AssertionFailed("Unexpected state when calling syncDevice() because (deviceNeedsUpdate() and hostNeedsUpdate())",
187 Here());
188 }
189 updateDevice();
190 }
191 }
192
193 bool deviceAllocated() const override { return device_allocated_; }
194
195 void allocateDevice() const override {
196 if (ATLAS_HAVE_GPU && pluto::devices()) {
197 if (device_allocated_) {
198 return;
199 }
200 if (size_) {
201 if(is_shared_data_) {
202 device_data_ = pluto::get_registered_device_pointer(host_data_);
203 }
204 else {
205 #if ATLAS_HAVE_TRACE
206 std::unique_ptr<atlas::Trace> trace;
207 if (atlas::Library::instance().traceMemory()) {
208 trace.reset(new atlas::Trace(Here(), "allocate_device", {"memory","memory.allocator","memory.allocator.device","memory.allocator.device.allocate"}));
209 }
210 #endif
211 device_data_ = device_allocator_.allocate(label_, size_);
212 }
213 ATLAS_ASSERT(pluto::is_device_accessible(device_data_));
214 device_allocated_ = true;
215 accMap();
216 }
217 }
218 }
219
220 void deallocateDevice() const override {
221 if (device_allocated_) {
222 accUnmap();
223 if (not is_shared_data_) {
224 #if ATLAS_HAVE_TRACE
225 std::unique_ptr<atlas::Trace> trace;
226 if (atlas::Library::instance().traceMemory()) {
227 trace.reset(new atlas::Trace(Here(), "deallocate_device", {"memory","memory.allocator","memory.allocator.device","memory.allocator.device.deallocate"}));
228 }
229 #endif
230 device_allocator_.deallocate(label_, device_data_,size_);
231 }
232 device_data_ = nullptr;
233 device_allocated_ = false;
234 device_updated_ = false;
235 }
236 }
237
238 bool hostNeedsUpdate() const override { return (not host_updated_); }
239
240 bool deviceNeedsUpdate() const override { return (not device_updated_); }
241
242 void setHostNeedsUpdate(bool v) const override { host_updated_ = (not v); }
243
244 void setDeviceNeedsUpdate(bool v) const override { device_updated_ = (not v); }
245
246 void reactivateDeviceWriteViews() const override {}
247
248 void reactivateHostWriteViews() const override {}
249
250 void* voidDataStore() override { return static_cast<void*>(host_data_); }
251
252 void* voidHostData() override { return static_cast<void*>(host_data_); }
253
254 void* voidDeviceData() override { return static_cast<void*>(device_data_); }
255
256 void accMap() const override {
257 if ((not acc_mapped_ && acc::devices())) {
258 if (is_shared_data_) {
259 // nvidia compiler should not accMap for managed memory
260 if (pluto::is_managed(host_data_) && acc::compiler_id() == acc::CompilerId::nvidia) {
261 return;
262 }
263 }
264 ATLAS_ASSERT(deviceAllocated(),"Could not accMap as device data is not allocated");
265 ATLAS_ASSERT(!atlas::acc::is_present(host_data_, size_ * sizeof(Value)));
266 if constexpr(ATLAS_ACC_DEBUG) {
267 std::cout << " + acc_map_data(hostptr:"<<host_data_<<", device:"<<device_data_<<", bytes:"<<footprint()<<")" <<std::endl;
268 }
269 atlas::acc::map((void*)host_data_, (void*)device_data_, size_ * sizeof(Value));
270 ATLAS_ASSERT(atlas::acc::is_present(host_data_, size_ * sizeof(Value)));
271 ATLAS_ASSERT(atlas::acc::deviceptr(host_data_) == device_data_);
272 acc_mapped_ = true;
273 }
274 }
275
276 bool accMapped() const override {
277 return acc_mapped_;
278 }
279
280 void accUnmap() const override {
281 if (acc_mapped_) {
282 if constexpr(ATLAS_ACC_DEBUG) {
283 std::cout << " - acc_unmap_data(hostptr:"<<host_data_<<", device:"<<device_data_<<", bytes:"<<footprint()<<")" <<std::endl;
284 }
285 ATLAS_ASSERT(atlas::acc::is_present(host_data_, size_ * sizeof(Value)));
286 atlas::acc::unmap(host_data_);
287 ATLAS_ASSERT(!atlas::acc::is_present(host_data_, size_ * sizeof(Value)));
288 acc_mapped_ = false;
289 }
290 }
291
292
293 private:
294
295 void allocateHost() {
296 if (size_ > 0) {
297 #if ATLAS_HAVE_TRACE
298 std::unique_ptr<atlas::Trace> trace;
299 if (atlas::Library::instance().traceMemory()) {
300 trace.reset(new atlas::Trace(Here(), "allocate_host", {"memory","memory.allocator","memory.allocator.host","memory.allocator.host.allocate"}));
301 }
302 #endif
303 host_data_ = host_allocator_.allocate(label_, size_);
304 }
305 else {
306 host_data_ = nullptr;
307 }
308 }
309
310 void deallocateHost() {
311 if (host_data_) {
312 #if ATLAS_HAVE_TRACE
313 std::unique_ptr<atlas::Trace> trace;
314 if (atlas::Library::instance().traceMemory()) {
315 trace.reset(new atlas::Trace(Here(), "deallocate_host", {"memory","memory.allocator","memory.allocator.host","memory.allocator.host.deallocate"}));
316 }
317 #endif
318 host_allocator_.deallocate(label_, host_data_, size_);
319 host_data_ = nullptr;
320 }
321 }
322
323 size_t footprint() const { return sizeof(Value) * size_; }
324
325 size_t size_;
326 Value* host_data_;
327 mutable Value* device_data_{nullptr};
328
329 mutable bool host_updated_{true};
330 mutable bool device_updated_{true};
331 mutable bool device_allocated_{false};
332 mutable bool acc_mapped_{false};
333 bool is_shared_data_{false};
334
335 pluto::allocator<Value> host_allocator_;
336 mutable pluto::allocator<Value> device_allocator_;
337
338 std::string label_;
339 };
340
341 //------------------------------------------------------------------------------
342
343 template <typename Value>
344 class WrappedDataStore final : public ArrayDataStore {
345 public:
346
347 void init_device() {
348 if (ATLAS_HAVE_GPU && pluto::devices()) {
349 device_updated_ = false;
350 }
351 else {
352 device_data_ = host_data_;
353 }
354 }
355
356 WrappedDataStore(Value* host_data, size_t size): host_data_(host_data), size_(size),
357 device_allocator_{pluto::device::get_default_resource()} {
358 label_ = array::label::get();
359 init_device();
360 }
361
362 WrappedDataStore(Value* host_data, const ArraySpec& spec):
363 host_data_(host_data),
364 size_(spec.size()),
365 device_allocator_{pluto::device::get_default_resource()} {
366 label_ = array::label::get();
367 init_device();
368 contiguous_ = spec.contiguous();
369 if (! contiguous_) {
370 int break_idx = 0;
371 bool break_found = false;
372 size_t expected_stride = 1;
373 size_t shp_mult_rhs = 1;
374 for (int i = spec.rank(); i > 0; i--) {
375 if (expected_stride != spec.strides()[i - 1]) {
376 if (! break_found) {
377 break_idx = i - 1;
378 break_found = true;
379 shp_mult_rhs = expected_stride;
380 expected_stride = spec.strides()[i - 1]; // prepare to search another discontiguity
381 }
382 else {
383 // double striding detected --> can not do memcpy anymore
384 memcpy_2d_ = false;
385 }
386 }
387 expected_stride *= spec.shape()[i - 1];
388 }
389 memcpy_h2d_pitch_ = shp_mult_rhs;
390 memcpy_d2h_pitch_ = spec.strides()[break_idx];
391 memcpy_width_ = shp_mult_rhs;
392 if (spec.strides()[spec.rank() - 1] > 1) {
393 memcpy_height_ = spec.shape()[0] * spec.device_strides()[0];
394 }
395 else {
396 size_t shp_mult_lhs = spec.shape()[0];
397 for (int i = 1; i <= break_idx; i++) {
398 shp_mult_lhs *= spec.shape()[i];
399 }
400 memcpy_height_ = shp_mult_lhs;
401 }
402 }
403 }
404
405 virtual ~WrappedDataStore() {
406 deallocateDevice();
407 }
408
409 void updateDevice() const override {
410 if (ATLAS_HAVE_GPU && pluto::devices()) {
411 if (not device_allocated_) {
412 allocateDevice();
413 }
414 #if ATLAS_HAVE_TRACE
415 std::unique_ptr<atlas::Trace> trace;
416 if (atlas::Library::instance().traceMemory()) {
417 trace.reset(new atlas::Trace(Here(), "update_device [host=wrapped]", {"memory","memory.transfer", "memory.transfer.h2d"}));
418 }
419 #endif
420 if (contiguous_) {
421 pluto::copy_host_to_device(label_, device_data_, host_data_, size_);
422 }
423 else {
424 ATLAS_ASSERT(memcpy_2d_, "Atlas fields with more than one discontiguous dimension are not supported");
425 pluto::copy_host_to_device_2D(
426 label_,
427 device_data_, memcpy_h2d_pitch_,
428 host_data_, memcpy_d2h_pitch_,
429 memcpy_width_, memcpy_height_);
430 }
431 device_updated_ = true;
432 }
433 }
434
435 void updateHost() const override {
436 if constexpr (ATLAS_HAVE_GPU) {
437 if (device_allocated_) {
438 #if ATLAS_HAVE_TRACE
439 std::unique_ptr<atlas::Trace> trace;
440 if (atlas::Library::instance().traceMemory()) {
441 trace.reset(new atlas::Trace(Here(), "update_host [host=wrapped]", {"memory","memory.transfer", "memory.transfer.d2h"}));
442 }
443 #endif
444 if (contiguous_) {
445 pluto::copy_device_to_host(label_, host_data_, device_data_, size_);
446 }
447 else {
448 ATLAS_ASSERT(memcpy_2d_, "Atlas fields with more than one discontiguous dimension are not supported");
449 pluto::copy_device_to_host_2D(
450 label_,
451 host_data_, memcpy_d2h_pitch_ ,
452 device_data_, memcpy_h2d_pitch_,
453 memcpy_width_, memcpy_height_);
454 }
455 host_updated_ = true;
456 }
457 }
458 }
459
460
461 bool valid() const override { return true; }
462
463 void syncHostDevice() const override {
464 if (host_updated_ and device_updated_) {
465 return; // nothing to do
466 }
467 if (not (host_updated_ or device_updated_)) {
468 throw_AssertionFailed("syncHostDevice() could not figure out which of host or device is up to date. "
469 "Probably it was forgotten to use setDeviceNeedsUpdate(true) or setDeviceNeedsUpdate(true)",
470 Here());
471 }
472
473 if (not device_updated_) {
474 updateDevice();
475 }
476 else if (not host_updated_) {
477 updateHost();
478 }
479 }
480
481 void syncHost() const override {
482 if (not host_updated_) {
483 ATLAS_ASSERT(device_updated_,"Unexpected state when calling syncHost() because (deviceNeedsUpdate() and hostNeedsUpdate())");
484 updateHost();
485 }
486 }
487
488 void syncDevice() const override {
489 if (not device_updated_) {
490 ATLAS_ASSERT(host_updated_,"Unexpected state when calling syncDevice() because (deviceNeedsUpdate() and hostNeedsUpdate())");
491 updateDevice();
492 }
493 }
494
495 bool deviceAllocated() const override { return device_allocated_; }
496
497 void allocateDevice() const override {
498 if (ATLAS_HAVE_GPU && pluto::devices()) {
499 if (device_allocated_) {
500 return;
501 }
502 if (size_) {
503 #if ATLAS_HAVE_TRACE
504 std::unique_ptr<atlas::Trace> trace;
505 if (atlas::Library::instance().traceMemory()) {
506 trace.reset(new atlas::Trace(Here(), "allocate_device [host=wrapped]", {"memory","memory.allocator","memory.allocator.device","memory.allocator.device.allocate"}));
507 }
508 #endif
509 device_data_ = device_allocator_.allocate(label_, size_);
510 device_allocated_ = true;
511 if (contiguous_) {
512 accMap();
513 }
514 }
515 }
516 }
517
518 void deallocateDevice() const override {
519 if (device_allocated_) {
520 if (contiguous_) {
521 accUnmap();
522 }
523 #if ATLAS_HAVE_TRACE
524 std::unique_ptr<atlas::Trace> trace;
525 if (atlas::Library::instance().traceMemory()) {
526 trace.reset(new atlas::Trace(Here(), "deallocate_device [host=wrapped]", {"memory","memory.allocator","memory.allocator.device","memory.allocator.device.deallocate"}));
527 }
528 #endif
529 device_allocator_.deallocate(label_, device_data_, size_);
530 device_data_ = nullptr;
531 device_allocated_ = false;
532 device_updated_ = false;
533 }
534 }
535
536 bool hostNeedsUpdate() const override { return (not host_updated_); }
537
538 bool deviceNeedsUpdate() const override { return (not device_updated_); }
539
540 void setHostNeedsUpdate(bool v) const override { host_updated_ = (not v); }
541
542 void setDeviceNeedsUpdate(bool v) const override { device_updated_ = (not v); }
543
544 void reactivateDeviceWriteViews() const override {}
545
546 void reactivateHostWriteViews() const override {}
547
548 void* voidDataStore() override { return static_cast<void*>(host_data_); }
549
550 void* voidHostData() override { return static_cast<void*>(host_data_); }
551
552 void* voidDeviceData() override { return static_cast<void*>(device_data_); }
553
554 void accMap() const override {
555 if (not acc_mapped_ && acc::devices()) {
556 ATLAS_ASSERT(deviceAllocated(),"Could not accMap as device data is not allocated");
557 ATLAS_ASSERT(!atlas::acc::is_present(host_data_, size_ * sizeof(Value)));
558 if constexpr(ATLAS_ACC_DEBUG) {
559 std::cout << " + acc_map_data(hostptr:"<<host_data_<<", device:"<<device_data_<<", bytes:"<<size_ * sizeof(Value)<<")" <<std::endl;
560 }
561 atlas::acc::map((void*)host_data_, (void*)device_data_, size_ * sizeof(Value));
562 ATLAS_ASSERT(atlas::acc::is_present(host_data_, size_ * sizeof(Value)));
563 ATLAS_ASSERT(atlas::acc::deviceptr(host_data_) == device_data_);
564 acc_mapped_ = true;
565 }
566 }
567
568 bool accMapped() const override {
569 return acc_mapped_;
570 }
571
572 void accUnmap() const override {
573 if (acc_mapped_) {
574 ATLAS_ASSERT(atlas::acc::is_present(host_data_, size_ * sizeof(Value)));
575 if constexpr(ATLAS_ACC_DEBUG) {
576 std::cout << " - acc_unmap_data(hostptr:"<<host_data_<<", device:"<<device_data_<<", bytes:"<<size_ * sizeof(Value)<<")" <<std::endl;
577 }
578 atlas::acc::unmap(host_data_);
579 acc_mapped_ = false;
580 }
581 }
582
583 private:
584 Value* host_data_;
585 size_t size_;
586 mutable Value* device_data_;
587
588 bool contiguous_{true};
589 bool memcpy_2d_{true};
590 size_t memcpy_h2d_pitch_;
591 size_t memcpy_d2h_pitch_;
592 size_t memcpy_height_;
593 size_t memcpy_width_;
594
595 mutable bool host_updated_{true};
596 mutable bool device_updated_{true};
597 mutable bool device_allocated_{false};
598 mutable bool acc_mapped_{false};
599
600 mutable pluto::allocator<Value> device_allocator_;
601
602 std::string label_;
603 };
604
605 } // namespace native
606 } // namespace array
607 } // namespace atlas