CPPuddle
Loading...
Searching...
No Matches
aggregation_executors_and_allocators.hpp
Go to the documentation of this file.
1// LANL Copyright Notice:
2//
3//© 2026. Triad National Security, LLC. All rights reserved.
4//
5// This program was produced under U.S. Government contract 89233218CNA000001
6// for Los Alamos National Laboratory (LANL), which is operated by Triad
7// National Security, LLC for the U.S. Department of Energy/National Nuclear
8// Security Administration. All rights in the program are reserved by Triad
9// National Security, LLC, and the U.S. Department of Energy/National Nuclear
10// Security Administration. The Government is granted for itself and others
11// acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license
12// in this material to reproduce, prepare. derivative works, distribute copies
13// to the public, perform publicly and display publicly, and to permit others
14// to do so.
15//
16//
17// Distributed under the Boost Software License, Version 1.0. (See accompanying
18// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
19
20#ifndef AGGREGATION_EXECUTOR_AND_ALLOCATOR_HPP
21#define AGGREGATION_EXECUTOR_AND_ALLOCATOR_HPP
22
23#ifndef CPPUDDLE_HAVE_HPX
24#error "Work aggregation allocators/executors require CPPUDDLE_WITH_HPX=ON"
25#endif
26
27#include <stdexcept>
28// When defined, CPPuddle will run more checks
29// about the order of aggregated method calls.
30// Best defined before including this header when needed
31// (hence commented out here)
32//#define DEBUG_AGGREGATION_CALLS 1
33
34#include <stdio.h>
35
36#include <any>
37#include <atomic>
38#include <chrono>
39#include <cstdio>
40#include <iostream>
41#include <memory>
42#include <mutex>
43#include <optional>
44#include <ostream>
45#include <string>
46#include <tuple>
47#include <type_traits>
48#include <typeinfo>
49#include <utility>
50#include <unordered_map>
51
52#include <hpx/futures/future.hpp>
53#include <hpx/hpx_init.hpp>
54#include <hpx/include/async.hpp>
55#include <hpx/include/iostreams.hpp>
56#include <hpx/include/lcos.hpp>
57#include <hpx/lcos/promise.hpp>
58#include <hpx/mutex.hpp>
59
60#if defined(HPX_HAVE_CUDA) || defined(HPX_HAVE_HIP)
61// required for defining type traits using cuda executor as underlying
62// aggregation executors
63#include <hpx/async_cuda/cuda_executor.hpp>
64#endif
65
66#include <boost/core/demangle.hpp>
67#include <boost/format.hpp>
68
70// get direct access to the buffer manangment
72// get normal access to the executor pools
74
75#ifndef CPPUDDLE_HAVE_HPX_MUTEX
76#pragma message \
77 "Work aggregation will use hpx::mutex internally, despite CPPUDDLE_WITH_HPX_MUTEX=OFF"
78#pragma message \
79 "Consider using CPPUDDLE_WITH_HPX_MUTEX=ON, to make the rest of CPPuddle also use hpx::mutex"
80#endif
81namespace cppuddle {
82namespace kernel_aggregation {
83namespace detail {
84 using aggregation_mutex_t = hpx::mutex;
85
86//===============================================================================
87//===============================================================================
88// Helper functions/classes
89
92template <typename... Ts>
93std::tuple<Ts...> make_tuple_supporting_references(Ts &&...ts) {
94 return std::tuple<Ts...>{std::forward<Ts>(ts)...};
95}
96
99template <typename T> std::string print_if_possible(T val) {
100 if constexpr (std::is_convertible_v<T, std::string>) {
101 return val;
102 } else if constexpr (std::is_integral_v<T> || std::is_floating_point_v<T>) {
103 return std::to_string(val);
104 } else if constexpr (std::is_pointer_v<T>) {
105 // Pretty printing pointer sort of only works well with %p
106 // TODO Try using std::format as soon as we can move to C++20
107 std::unique_ptr<char[]> debug_string(new char[128]());
108 snprintf(debug_string.get(), 128, "%p", val);
109 return std::string(debug_string.get());
110 } else {
111 return std::string("cannot print value");
112 }
113}
114
117template <class TupType, size_t... I>
118void print_tuple(const TupType &_tup, std::index_sequence<I...>) {
119 (..., (hpx::cout << (I == 0 ? "" : ", ")
120 << print_if_possible(std::get<I + 1>(_tup))));
121}
122
125template <class... T> void print_tuple(const std::tuple<T...> &_tup) {
126 // Use pointer and sprintf as boost::format refused to NOT cast the pointer
127 // address to 1...
128 // TODO Try using std::format as soon as we can move to C++20
129 std::unique_ptr<char[]> debug_string(new char[128]());
130 snprintf(debug_string.get(), 128, "Function address: %p -- Arguments: (",
131 std::get<0>(_tup));
132 hpx::cout << debug_string.get();
133 print_tuple(_tup, std::make_index_sequence<sizeof...(T) - 1>());
134 hpx::cout << ")";
135}
136
137//===============================================================================
138//===============================================================================
139template <typename Executor, typename F, typename... Ts>
140void exec_post_wrapper(Executor & exec, F &&f, Ts &&...ts) {
141 hpx::apply(exec, std::forward<F>(f), std::forward<Ts>(ts)...);
142}
143
144template <typename Executor, typename F, typename... Ts>
145hpx::future<void> exec_async_wrapper(Executor & exec, F &&f, Ts &&...ts) {
146 return hpx::async(exec, std::forward<F>(f), std::forward<Ts>(ts)...);
147}
148
151/** Launch conditions: All slice executors must have called the same function
152 * (tracked by future all_slices_ready)
153 * AND
154 * Previous aggregated_function_call on the same Executor must have been
155 * launched (tracked by future stream_future)
156 * All function calls received from the slice executors are checked if they
157 * match the first one in both types and values (throws exception otherwise)
158 */
159
160template <typename Executor> class aggregated_function_call {
161private:
162 std::atomic<size_t> slice_counter = 0;
163
165 /* hpx::promise<void> slices_ready_promise; */
167 /* hpx::future<void> all_slices_ready = slices_ready_promise.get_future(); */
169 const size_t number_slices;
170 const bool async_mode;
171
172 Executor &underlying_executor;
173
174#if !(defined(NDEBUG)) && defined(DEBUG_AGGREGATION_CALLS)
175#pragma message \
176 "Building slow work aggegator build with additional runtime checks! Build with NDEBUG defined for fast build..."
179 std::any function_tuple;
181 std::string debug_type_information;
182 aggregation_mutex_t debug_mut;
183#endif
184
185 std::vector<hpx::promise<void>> potential_async_promises{};
186
187public:
188 aggregated_function_call(const size_t number_slices, bool async_mode, Executor &exec)
189 : number_slices(number_slices), async_mode(async_mode), underlying_executor(exec) {
190 if (async_mode)
191 potential_async_promises.resize(number_slices);
192 }
193 ~aggregated_function_call(void) {
194 // All slices should have done this call
195 assert(slice_counter == number_slices);
196 // assert(!all_slices_ready.valid());
197 }
198
199 bool sync_aggregation_slices(hpx::future<void> &stream_future) {
200 assert(!async_mode);
201 assert(potential_async_promises.empty());
202 const size_t local_counter = slice_counter++;
203 if (local_counter == number_slices - 1) {
204 return true;
205 }
206 else return false;
207 }
208 template <typename F, typename... Ts>
209 void post_when(hpx::future<void> &stream_future, F &&f, Ts &&...ts) {
210#if !(defined(NDEBUG)) && defined(DEBUG_AGGREGATION_CALLS)
211 // needed for concurrent access to function_tuple and debug_type_information
212 // Not required for normal use
213 std::lock_guard<aggregation_mutex_t> guard(debug_mut);
214#endif
215 assert(!async_mode);
216 assert(potential_async_promises.empty());
217 const size_t local_counter = slice_counter++;
218
219 if (local_counter == 0) {
220#if !(defined(NDEBUG)) && defined(DEBUG_AGGREGATION_CALLS)
221 auto tmp_tuple =
222 make_tuple_supporting_references(f, std::forward<Ts>(ts)...);
223 function_tuple = tmp_tuple;
224 debug_type_information = typeid(decltype(tmp_tuple)).name();
225#endif
226
227 } else {
228 //
229 // This scope checks if both the type and the values of the current call
230 // match the original call To be used in debug build...
231 //
232#if !(defined(NDEBUG)) && defined(DEBUG_AGGREGATION_CALLS)
233 auto comparison_tuple =
234 make_tuple_supporting_references(f, std::forward<Ts>(ts)...);
235 try {
236 auto orig_call_tuple =
237 std::any_cast<decltype(comparison_tuple)>(function_tuple);
238 if (comparison_tuple != orig_call_tuple) {
239 throw std::runtime_error(
240 "Values of post function arguments (or function "
241 "itself) do not match ");
242 }
243 } catch (const std::bad_any_cast &e) {
244 hpx::cout
245 << "\nMismatched types error in aggregated post call of executor "
246 << ": " << e.what() << "\n";
247 hpx::cout << "Expected types:\t\t "
248 << boost::core::demangle(debug_type_information.c_str());
249 hpx::cout << "\nGot types:\t\t "
250 << boost::core::demangle(
251 typeid(decltype(comparison_tuple)).name())
252 << "\n"
253 << std::endl;
254 // throw;
255 } catch (const std::runtime_error &e) {
256 hpx::cout
257 << "\nMismatched values error in aggregated post call of executor "
258 << ": " << e.what() << std::endl;
259 hpx::cout << "Types (matched):\t "
260 << boost::core::demangle(debug_type_information.c_str());
261 auto orig_call_tuple =
262 std::any_cast<decltype(comparison_tuple)>(function_tuple);
263 hpx::cout << "\nExpected values:\t ";
264 print_tuple(orig_call_tuple);
265 hpx::cout << "\nGot values:\t\t ";
266 print_tuple(comparison_tuple);
267 hpx::cout << std::endl << std::endl;
268 // throw;
269 }
270#endif
271 }
272 assert(local_counter < number_slices);
273 assert(slice_counter < number_slices + 1);
274 // Check exit criteria: Launch function call continuation by setting the
275 // slices promise
276 if (local_counter == number_slices - 1) {
277 exec_post_wrapper<Executor, F, Ts...>(underlying_executor, std::forward<F>(f), std::forward<Ts>(ts)...);
278 //slices_ready_promise.set_value();
279 }
280 }
281 template <typename F, typename... Ts>
282 hpx::future<void> async_when(hpx::future<void> &stream_future,
283 F &&f, Ts &&...ts) {
284#if !(defined(NDEBUG)) && defined(DEBUG_AGGREGATION_CALLS)
285 // needed for concurrent access to function_tuple and debug_type_information
286 // Not required for normal use
287 std::lock_guard<aggregation_mutex_t> guard(debug_mut);
288#endif
289 assert(async_mode);
290 assert(!potential_async_promises.empty());
291 const size_t local_counter = slice_counter++;
292 if (local_counter == 0) {
293#if !(defined(NDEBUG)) && defined(DEBUG_AGGREGATION_CALLS)
294 auto tmp_tuple =
295 make_tuple_supporting_references(f, std::forward<Ts>(ts)...);
296 function_tuple = tmp_tuple;
297 debug_type_information = typeid(decltype(tmp_tuple)).name();
298#endif
299 } else {
300 //
301 // This scope checks if both the type and the values of the current call
302 // match the original call To be used in debug build...
303 //
304#if !(defined(NDEBUG)) && defined(DEBUG_AGGREGATION_CALLS)
305 auto comparison_tuple =
306 make_tuple_supporting_references(f, std::forward<Ts>(ts)...);
307 try {
308 auto orig_call_tuple =
309 std::any_cast<decltype(comparison_tuple)>(function_tuple);
310 if (comparison_tuple != orig_call_tuple) {
311 throw std::runtime_error(
312 "Values of async function arguments (or function "
313 "itself) do not match ");
314 }
315 } catch (const std::bad_any_cast &e) {
316 hpx::cout
317 << "\nMismatched types error in aggregated async call of executor "
318 << ": " << e.what() << "\n";
319 hpx::cout << "Expected types:\t\t "
320 << boost::core::demangle(debug_type_information.c_str());
321 hpx::cout << "\nGot types:\t\t "
322 << boost::core::demangle(
323 typeid(decltype(comparison_tuple)).name())
324 << "\n"
325 << std::endl;
326 // throw;
327 } catch (const std::runtime_error &e) {
328 hpx::cout
329 << "\nMismatched values error in aggregated async call of executor "
330 << ": " << e.what() << std::endl;
331 hpx::cout << "Types (matched):\t "
332 << boost::core::demangle(debug_type_information.c_str());
333 auto orig_call_tuple =
334 std::any_cast<decltype(comparison_tuple)>(function_tuple);
335 hpx::cout << "\nExpected values:\t ";
336 print_tuple(orig_call_tuple);
337 hpx::cout << "\nGot values:\t\t ";
338 print_tuple(comparison_tuple);
339 hpx::cout << std::endl << std::endl;
340 // throw;
341 }
342#endif
343 }
344 assert(local_counter < number_slices);
345 assert(slice_counter < number_slices + 1);
346 assert(potential_async_promises.size() == number_slices);
347 hpx::future<void> ret_fut =
348 potential_async_promises[local_counter].get_future();
349 if (local_counter == number_slices - 1) {
350 /* slices_ready_promise.set_value(); */
351 auto fut = exec_async_wrapper<Executor, F, Ts...>(
352 underlying_executor, std::forward<F>(f), std::forward<Ts>(ts)...);
353 fut.then([this](auto &&fut) {
354 for (auto &promise : potential_async_promises) {
355 promise.set_value();
356 }
357 });
358 }
359 // Check exit criteria: Launch function call continuation by setting the
360 // slices promise
361 return ret_fut;
362 }
363 template <typename F, typename... Ts>
364 hpx::shared_future<void> wrap_async(hpx::future<void> &stream_future,
365 F &&f, Ts &&...ts) {
366 assert(async_mode);
367 assert(!potential_async_promises.empty());
368 const size_t local_counter = slice_counter++;
369 assert(local_counter < number_slices);
370 assert(slice_counter < number_slices + 1);
371 assert(potential_async_promises.size() == number_slices);
372 hpx::shared_future<void> ret_fut =
373 potential_async_promises[local_counter].get_shared_future();
374 if (local_counter == number_slices - 1) {
375 auto fut = f(std::forward<Ts>(ts)...);
376 fut.then([this](auto &&fut) {
377 // TODO just use one promise
378 for (auto &promise : potential_async_promises) {
379 promise.set_value();
380 }
381 });
382 }
383 return ret_fut;
384 }
385 // We need to be able to copy or no-except move for std::vector..
386 aggregated_function_call(const aggregated_function_call &other) = default;
387 aggregated_function_call &
388 operator=(const aggregated_function_call &other) = default;
389 aggregated_function_call(aggregated_function_call &&other) = default;
390 aggregated_function_call &
391 operator=(aggregated_function_call &&other) = default;
392};
393
394//===============================================================================
395//===============================================================================
396
397enum class aggregated_executor_modes { EAGER = 1, STRICT, ENDLESS };
399template <typename T, typename Host_Allocator, typename Executor>
400class allocator_slice;
401
403/** Executor is not meant to be used directly. Instead it yields multiple
404 * executor_slice objects. These serve as interfaces. Slices from the same
405 * aggregated_executor are meant to execute the same function calls but on
406 * different data (i.e. different tasks)
407 */
408template <typename Executor> class aggregated_executor {
409private:
410 //===============================================================================
411 // Misc private variables:
412 //
413 std::atomic<bool> slices_exhausted;
414
415 std::atomic<bool> executor_slices_alive;
416 std::atomic<bool> buffers_in_use;
417 std::atomic<size_t> dealloc_counter;
418
419 const aggregated_executor_modes mode;
420 const size_t max_slices;
421 std::atomic<size_t> current_slices;
425 std::unique_ptr<cppuddle::executor_recycling::executor_interface<
426 Executor, cppuddle::executor_recycling::round_robin_pool_impl<Executor>>>
427 executor_wrapper;
428
429public:
430 size_t gpu_id;
431 // Subclasses
432
434 class executor_slice {
435 public:
436 aggregated_executor<Executor> &parent;
437 private:
441 size_t launch_counter{0};
442 size_t buffer_counter{0};
443 bool notify_parent_about_destruction{true};
444
445 public:
448 size_t number_slices;
449 size_t max_slices;
450 size_t id;
451 using executor_t = Executor;
452 executor_slice(aggregated_executor &parent, const size_t slice_id,
453 const size_t number_slices, const size_t max_number_slices)
454 : parent(parent), notify_parent_about_destruction(true),
455 number_slices(number_slices), id(slice_id), max_slices(max_number_slices) {
456 assert(parent.max_slices == max_slices);
457 assert(number_slices >= 1);
458 assert(number_slices <= max_slices);
459 }
460 ~executor_slice(void) {
461 // Don't notify parent if we moved away from this executor_slice
462 if (notify_parent_about_destruction) {
463 // Executor should be done by the time of destruction
464 // -> check here before notifying parent
465
466 assert(parent.max_slices == max_slices);
467 assert(number_slices >= 1);
468 assert(number_slices <= max_slices);
469 // parent still in execution mode?
470 assert(parent.slices_exhausted == true);
471 // all kernel launches done?
472 assert(launch_counter == parent.function_calls.size());
473 // Notifiy parent that this aggregation slice is one
474 parent.reduce_usage_counter();
475 }
476 }
477 executor_slice(const executor_slice &other) = delete;
478 executor_slice &operator=(const executor_slice &other) = delete;
479 executor_slice(executor_slice &&other)
480 : parent(other.parent), launch_counter(std::move(other.launch_counter)),
481 buffer_counter(std::move(other.buffer_counter)),
482 number_slices(std::move(other.number_slices)),
483 id(std::move(other.id)), max_slices(std::move(other.max_slices)) {
484 other.notify_parent_about_destruction = false;
485 }
486 executor_slice &operator=(executor_slice &&other) {
487 parent = other.parent;
488 launch_counter = std::move(other.launch_counter);
489 buffer_counter = std::move(other.buffer_counter);
490 number_slices = std::move(other.number_slices);
491 id = std::move(other.id);
492 max_slices = std::move(other.max_slices);
493 other.notify_parent_about_destruction = false;
494 }
495 template <typename T, typename Host_Allocator>
496 allocator_slice<T, Host_Allocator, Executor> make_allocator() {
497 return allocator_slice<T, Host_Allocator, Executor>(*this);
498 }
499 bool sync_aggregation_slices() {
500 assert(parent.slices_exhausted == true);
501 auto ret = parent.sync_aggregation_slices(launch_counter);
502 launch_counter++;
503 return ret;
504 }
505 template <typename F, typename... Ts> void post(F &&f, Ts &&...ts) {
506 // we should only execute function calls once all slices
507 // have been given away (-> Executor Slices start)
508 assert(parent.slices_exhausted == true);
509 parent.post(launch_counter, std::forward<F>(f), std::forward<Ts>(ts)...);
510 launch_counter++;
511 }
512 template <typename F, typename... Ts>
513 hpx::future<void> async(F &&f, Ts &&...ts) {
514 // we should only execute function calls once all slices
515 // have been given away (-> Executor Slices start)
516 assert(parent.slices_exhausted == true);
517 hpx::future<void> ret_fut = parent.async(
518 launch_counter, std::forward<F>(f), std::forward<Ts>(ts)...);
519 launch_counter++;
520 return ret_fut;
521 }
522
523 // OneWay Execution
524 template <typename F, typename... Ts>
525 friend decltype(auto) tag_invoke(hpx::parallel::execution::post_t,
526 executor_slice& exec, F&& f, Ts&&... ts)
527 {
528 return exec.post(std::forward<F>(f), std::forward<Ts>(ts)...);
529 }
530
531 // TwoWay Execution
532 template <typename F, typename... Ts>
533 friend decltype(auto) tag_invoke(
534 hpx::parallel::execution::async_execute_t, executor_slice& exec,
535 F&& f, Ts&&... ts)
536 {
537 return exec.async(
538 std::forward<F>(f), std::forward<Ts>(ts)...);
539 }
540
541 template <typename F, typename... Ts>
542 hpx::shared_future<void> wrap_async(F &&f, Ts &&...ts) {
543 // we should only execute function calls once all slices
544 // have been given away (-> Executor Slices start)
545 assert(parent.slices_exhausted == true);
546 hpx::shared_future<void> ret_fut = parent.wrap_async(
547 launch_counter, std::forward<F>(f), std::forward<Ts>(ts)...);
548 launch_counter++;
549 return ret_fut;
550 }
551
554 template <typename T, typename Host_Allocator> T *get(const size_t size) {
555 assert(parent.slices_exhausted == true);
556 T *aggregated_buffer =
557 parent.get<T, Host_Allocator>(size, buffer_counter);
558 buffer_counter++;
559 assert(buffer_counter > 0);
560 return aggregated_buffer;
561 }
562
563 Executor& get_underlying_executor(void) {
564 assert(parent.executor_wrapper);
565 return *(parent.executor_wrapper);
566 }
567 };
568
569 // deprecated name...
570 using Executor_Slice [[deprectated("Renamed: Use executor_slice instead")]] = executor_slice;
571
572 //===============================================================================
573
574 // hpx::promise<void> slices_full_promise;
577 std::vector<hpx::promise<executor_slice>> executor_slices;
580 std::deque<aggregated_function_call<Executor>> function_calls;
582 aggregation_mutex_t mut;
583
586 using buffer_entry_t =
587 std::tuple<void*, const size_t, std::atomic<size_t>, bool, const size_t, size_t>;
589 std::deque<buffer_entry_t> buffer_allocations;
591 std::unordered_map<void*,size_t> buffer_allocations_map;
593 aggregation_mutex_t buffer_mut;
594 std::atomic<size_t> buffer_counter = 0;
595
597 template <typename T, typename Host_Allocator>
598 T *get(const size_t size, const size_t slice_alloc_counter) {
599 assert(slices_exhausted == true);
600 assert(executor_wrapper);
601 assert(executor_slices_alive == true);
602 // Add aggreated buffer entry in case it hasn't happened yet for this call
603 // First: Check if it already has happened
604 if (buffer_counter <= slice_alloc_counter) {
605 // we might be the first! Lock...
606 std::lock_guard<aggregation_mutex_t> guard(buffer_mut);
607 // ... and recheck
608 if (buffer_counter <= slice_alloc_counter) {
609 constexpr bool manage_content_lifetime = false;
610 buffers_in_use = true;
611
612 // Default location -- useful for GPU builds as we otherwise create way too
613 // many different buffers for different aggregation sizes on different GPUs
614 /* size_t location_id = gpu_id * instances_per_gpu; */
615 // Use integer conversion to only use 0 16 32 ... as buckets
616 size_t location_id = ((hpx::get_worker_thread_num() % cppuddle::number_instances) / 16) * 16;
617#ifdef CPPUDDLE_HAVE_HPX_AWARE_ALLOCATORS
618 if (max_slices == 1) {
619 // get prefered location: aka the current hpx threads location
620 // Usually handy for CPU builds where we want to use the buffers
621 // close to the current CPU core
622 /* location_id = (hpx::get_worker_thread_num() / instances_per_gpu) * instances_per_gpu; */
623 /* location_id = (gpu_id) * instances_per_gpu; */
624 // division makes sure that we always use the same instance to store our gpu buffers.
625 }
626#endif
627 // Get shiny and new buffer that will be shared between all slices
628 // Buffer might be recycled from previous allocations by the
629 // buffer_interface...
630 T *aggregated_buffer =
631 cppuddle::memory_recycling::detail::buffer_interface::get<
632 T, Host_Allocator>(size, manage_content_lifetime, location_id,
633 gpu_id);
634 // Create buffer entry for this buffer
635 buffer_allocations.emplace_back(static_cast<void *>(aggregated_buffer),
636 size, 1, true, location_id, gpu_id);
637
638#ifndef NDEBUG
639 // if previousely used the buffer should not be in usage anymore
640 const auto exists = buffer_allocations_map.count(
641 static_cast<void *>(aggregated_buffer));
642 if (exists > 0) {
643 const auto previous_usage_id =
644 buffer_allocations_map[static_cast<void *>(aggregated_buffer)];
645 const auto &valid =
646 std::get<3>(buffer_allocations[previous_usage_id]);
647 assert(!valid);
648 }
649#endif
650 buffer_allocations_map.insert_or_assign(static_cast<void *>(aggregated_buffer),
651 buffer_counter);
652
653 assert (buffer_counter == slice_alloc_counter);
654 buffer_counter = buffer_allocations.size();
655
656 // Return buffer
657 return aggregated_buffer;
658 }
659 }
660 assert(buffers_in_use == true);
661 assert(std::get<3>(buffer_allocations[slice_alloc_counter])); // valid
662 assert(std::get<2>(buffer_allocations[slice_alloc_counter]) >= 1);
663
664 // Buffer entry should already exist:
665 T *aggregated_buffer = static_cast<T *>(
666 std::get<0>(buffer_allocations[slice_alloc_counter]));
667 // Error handling: Size is wrong?
668 assert(size == std::get<1>(buffer_allocations[slice_alloc_counter]));
669 // Notify that one more slice has visited this buffer allocation
670 std::get<2>(buffer_allocations[slice_alloc_counter])++;
671 return aggregated_buffer;
672 }
673
675 template <typename T, typename Host_Allocator>
676 void mark_unused(T *p, const size_t size) {
677 assert(slices_exhausted == true);
678 assert(executor_wrapper);
679
680 void *ptr_key = static_cast<void*>(p);
681 size_t slice_alloc_counter = buffer_allocations_map[p];
682
683 assert(slice_alloc_counter < buffer_allocations.size());
684 /*auto &[buffer_pointer_any, buffer_size, buffer_allocation_counter, valid] =
685 buffer_allocations[slice_alloc_counter];*/
686 auto buffer_pointer_void = std::get<0>(buffer_allocations[slice_alloc_counter]);
687 const auto buffer_size = std::get<1>(buffer_allocations[slice_alloc_counter]);
688 auto &buffer_allocation_counter = std::get<2>(buffer_allocations[slice_alloc_counter]);
689 auto &valid = std::get<3>(buffer_allocations[slice_alloc_counter]);
690 const auto &location_id = std::get<4>(buffer_allocations[slice_alloc_counter]);
691 const auto &gpu_id = std::get<5>(buffer_allocations[slice_alloc_counter]);
692 assert(valid);
693 T *buffer_pointer = static_cast<T *>(buffer_pointer_void);
694
695 assert(buffer_size == size);
696 assert(p == buffer_pointer);
697 // assert(buffer_pointer == p || buffer_pointer == nullptr);
698 // Slice is done with this buffer
699 buffer_allocation_counter--;
700 // Check if all slices are done with this buffer?
701 if (buffer_allocation_counter == 0) {
702 // Yes! "Deallocate" by telling the recylcer the buffer is fit for reusage
703 std::lock_guard<aggregation_mutex_t> guard(buffer_mut);
704 // Only mark unused if another buffer has not done so already (and marked
705 // it as invalid)
706 if (valid) {
707 assert(buffers_in_use == true);
708 cppuddle::memory_recycling::detail::buffer_interface::mark_unused<
709 T, Host_Allocator>(buffer_pointer, buffer_size, location_id,
710 gpu_id);
711 // mark buffer as invalid to prevent any other slice from marking the
712 // buffer as unused
713 valid = false;
714
715 const size_t current_deallocs = ++dealloc_counter;
716 if (current_deallocs == buffer_counter) {
717 std::lock_guard<aggregation_mutex_t> guard(mut);
718 buffers_in_use = false;
719 if (!executor_slices_alive && !buffers_in_use) {
720 slices_exhausted = false;
721 // Release executor
722 executor_wrapper.reset(nullptr);
723 }
724 }
725 }
726 }
727 }
728
729 //===============================================================================
730 // Public Interface
731public:
732 hpx::future<void> current_continuation;
733 hpx::future<void> last_stream_launch_done;
734 std::atomic<size_t> overall_launch_counter = 0;
735
737 bool sync_aggregation_slices(const size_t slice_launch_counter) {
738 std::lock_guard<aggregation_mutex_t> guard(mut);
739 assert(slices_exhausted == true);
740 assert(executor_wrapper);
741 // Add function call object in case it hasn't happened for this launch yet
742 if (overall_launch_counter <= slice_launch_counter) {
743 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
744 if (overall_launch_counter <= slice_launch_counter) {
745 function_calls.emplace_back(current_slices, false, *executor_wrapper);
746 overall_launch_counter = function_calls.size();
747 return function_calls[slice_launch_counter].sync_aggregation_slices(
748 last_stream_launch_done);
749 }
750 }
751
752 return function_calls[slice_launch_counter].sync_aggregation_slices(
753 last_stream_launch_done);
754 }
755
757 template <typename F, typename... Ts>
758 void post(const size_t slice_launch_counter, F &&f, Ts &&...ts) {
759 std::lock_guard<aggregation_mutex_t> guard(mut);
760 assert(slices_exhausted == true);
761 assert(executor_wrapper);
762 // Add function call object in case it hasn't happened for this launch yet
763 if (overall_launch_counter <= slice_launch_counter) {
764 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
765 if (overall_launch_counter <= slice_launch_counter) {
766 function_calls.emplace_back(current_slices, false, *executor_wrapper);
767 overall_launch_counter = function_calls.size();
768 function_calls[slice_launch_counter].post_when(
769 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
770 return;
771 }
772 }
773
774 function_calls[slice_launch_counter].post_when(
775 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
776 return;
777 }
778
780 template <typename F, typename... Ts>
781 hpx::future<void> async(const size_t slice_launch_counter, F &&f,
782 Ts &&...ts) {
783 std::lock_guard<aggregation_mutex_t> guard(mut);
784 assert(slices_exhausted == true);
785 assert(executor_wrapper);
786 // Add function call object in case it hasn't happened for this launch yet
787 if (overall_launch_counter <= slice_launch_counter) {
788 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
789 if (overall_launch_counter <= slice_launch_counter) {
790 function_calls.emplace_back(current_slices, true, *executor_wrapper);
791 overall_launch_counter = function_calls.size();
792 return function_calls[slice_launch_counter].async_when(
793 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
794 }
795 }
796
797 return function_calls[slice_launch_counter].async_when(
798 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
799 }
800
801 template <typename F, typename... Ts>
802 hpx::shared_future<void> wrap_async(const size_t slice_launch_counter, F &&f,
803 Ts &&...ts) {
804 std::lock_guard<aggregation_mutex_t> guard(mut);
805 assert(slices_exhausted == true);
806 assert(executor_wrapper);
807 // Add function call object in case it hasn't happened for this launch yet
808 if (overall_launch_counter <= slice_launch_counter) {
809 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
810 if (overall_launch_counter <= slice_launch_counter) {
811 function_calls.emplace_back(current_slices, true, *executor_wrapper);
812 overall_launch_counter = function_calls.size();
813 return function_calls[slice_launch_counter].wrap_async(
814 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
815 }
816 }
817
818 return function_calls[slice_launch_counter].wrap_async(
819 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
820 }
821
822 bool slice_available(void) {
823 std::lock_guard<aggregation_mutex_t> guard(mut);
824 return !slices_exhausted;
825 }
826
827 std::optional<hpx::future<executor_slice>> request_executor_slice() {
828 std::lock_guard<aggregation_mutex_t> guard(mut);
829 if (!slices_exhausted) {
830 const size_t local_slice_id = ++current_slices;
831 if (local_slice_id == 1) {
832 // Cleanup leftovers from last run if any
833 // TODO still required? Should be clean here already
834 function_calls.clear();
835 overall_launch_counter = 0;
836 std::lock_guard<aggregation_mutex_t> guard(buffer_mut);
837#ifndef NDEBUG
838 for (const auto &buffer_entry : buffer_allocations) {
839 const auto &[buffer_pointer_any, buffer_size,
840 buffer_allocation_counter, valid, location_id, device_id] =
841 buffer_entry;
842 assert(!valid);
843 }
844#endif
845 buffer_allocations.clear();
846 buffer_allocations_map.clear();
847 buffer_counter = 0;
848
849 assert(executor_slices_alive == false);
850 assert(buffers_in_use == false);
851 executor_slices_alive = true;
852 buffers_in_use = false;
853 dealloc_counter = 0;
854 launched_slices = 0;
855
856
857 // if (mode == aggregated_executor_modes::STRICT ) {
858 // slices_full_promise = hpx::promise<void>{};
859 // }
860 }
861
862 // Create Executor Slice future -- that will be returned later
863 hpx::future<executor_slice> ret_fut;
864 if (local_slice_id < max_slices) {
865 executor_slices.emplace_back(hpx::promise<executor_slice>{});
866 ret_fut =
867 executor_slices[local_slice_id - 1].get_future();
868 } else {
869 launched_slices = current_slices;
870 ret_fut = hpx::make_ready_future(executor_slice{*this,
871 executor_slices.size(), launched_slices, max_slices});
872 }
873
874 // Are we the first slice? If yes, add continuation set the
875 // executor_slice
876 // futures to ready if the launch conditions are met
877 if (local_slice_id == 1) {
878 // Redraw executor
879 assert(!executor_wrapper);
880 cppuddle::executor_recycling::executor_pool::select_device<
881 Executor, cppuddle::executor_recycling::round_robin_pool_impl<Executor>>(
882 gpu_id);
883 executor_wrapper.reset(
884 new cppuddle::executor_recycling::executor_interface<
885 Executor,
886 cppuddle::executor_recycling::round_robin_pool_impl<Executor>>(
887 gpu_id));
888 // Renew promise that all slices will be ready as the primary launch
889 // criteria...
890 hpx::shared_future<void> fut;
891 if (mode == aggregated_executor_modes::EAGER ||
892 mode == aggregated_executor_modes::ENDLESS) {
893 // Fallback launch condidtion: Launch as soon as the underlying stream
894 // is ready
895 /* auto slices_full_fut = slices_full_promise.get_future(); */
896 cppuddle::executor_recycling::executor_pool::select_device<
897 Executor,
898 cppuddle::executor_recycling::round_robin_pool_impl<Executor>>(gpu_id);
899 auto exec_fut = (*executor_wrapper).get_future();
900 /* auto fut = hpx::when_any(exec_fut, slices_full_fut); */
901 fut = std::move(exec_fut);
902 // Launch all executor slices within this continuation
903 current_continuation = fut.then([this](auto &&fut) {
904 std::lock_guard<aggregation_mutex_t> guard(mut);
905 // if (!slices_exhausted) {
906 slices_exhausted = true;
907 launched_slices = current_slices;
908 size_t id = 0;
909 for (auto &slice_promise : executor_slices) {
910 slice_promise.set_value(
911 executor_slice{*this, id, launched_slices, max_slices});
912 id++;
913 }
914 executor_slices.clear();
915 });
916 }
917 }
918 if (local_slice_id >= max_slices &&
919 mode != aggregated_executor_modes::ENDLESS) {
920 slices_exhausted = true;
921 launched_slices = current_slices;
922 size_t id = 0;
923 for (auto &slice_promise : executor_slices) {
924 slice_promise.set_value(
925 executor_slice{*this, id, launched_slices, max_slices});
926 id++;
927 }
928 executor_slices.clear();
929 }
930 return ret_fut;
931 } else {
932 // Return empty optional as failure
933 return std::optional<hpx::future<executor_slice>>{};
934 }
935 }
936 size_t launched_slices;
937 void reduce_usage_counter(void) {
938 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
939 assert(slices_exhausted == true);
940 assert(executor_wrapper);
941 assert(executor_slices_alive == true);
942 assert(launched_slices >= 1);
943 assert(current_slices >= 0 && current_slices <= launched_slices);
944 const size_t local_slice_id = --current_slices;
945 // Last slice goes out scope?
946 if (local_slice_id == 0) {
947 // Mark executor fit for reusage
948 if (mode != aggregated_executor_modes::STRICT )
949 current_continuation.get(); // clear remaining task in case it was not run yet
950 std::lock_guard<aggregation_mutex_t> guard(mut);
951 executor_slices_alive = false;
952 if (!executor_slices_alive && !buffers_in_use) {
953 // Release executor
954 slices_exhausted = false;
955 executor_wrapper.reset(nullptr);
956 }
957 }
958 }
959 ~aggregated_executor(void) {
960
961 assert(current_slices == 0);
962 assert(executor_slices_alive == false);
963 assert(buffers_in_use == false);
964
965 // if (mode != aggregated_executor_modes::STRICT ) {
966 // slices_full_promise.set_value(); // Trigger slices launch condition continuation
967 // }
968
969 // Cleanup leftovers from last run if any
970 function_calls.clear();
971 overall_launch_counter = 0;
972#ifndef NDEBUG
973 for (const auto &buffer_entry : buffer_allocations) {
974 const auto &[buffer_pointer_any, buffer_size, buffer_allocation_counter,
975 valid, location_id, device_id] = buffer_entry;
976 assert(!valid);
977 }
978#endif
979 buffer_allocations.clear();
980 buffer_allocations_map.clear();
981 buffer_counter = 0;
982
983 assert(buffer_allocations.empty());
984 assert(buffer_allocations_map.empty());
985 }
986
987 aggregated_executor(const size_t number_slices,
988 aggregated_executor_modes mode, const size_t gpu_id = 0)
989 : max_slices(number_slices), current_slices(0), slices_exhausted(false),
990 dealloc_counter(0), mode(mode), executor_slices_alive(false),
991 buffers_in_use(false), gpu_id(gpu_id),
992 executor_wrapper(nullptr),
993 current_continuation(hpx::make_ready_future()),
994 last_stream_launch_done(hpx::make_ready_future()) {}
995 // Not meant to be copied or moved
996 aggregated_executor(const aggregated_executor &other) = delete;
997 aggregated_executor &operator=(const aggregated_executor &other) = delete;
998 aggregated_executor(aggregated_executor &&other) = delete;
999 aggregated_executor &operator=(aggregated_executor &&other) = delete;
1000};
1001
1002template <typename T, typename Host_Allocator, typename Executor>
1003class allocator_slice {
1004private:
1005 typename aggregated_executor<Executor>::executor_slice &executor_reference;
1006 aggregated_executor<Executor> &executor_parent;
1007
1008public:
1009 using value_type = T;
1010 allocator_slice(
1011 typename aggregated_executor<Executor>::executor_slice &executor)
1012 : executor_reference(executor), executor_parent(executor.parent) {}
1013 template <typename U>
1014 explicit allocator_slice(
1015 allocator_slice<U, Host_Allocator, Executor> const &) noexcept {}
1016 T *allocate(std::size_t n) {
1017 T *data = executor_reference.template get<T, Host_Allocator>(n);
1018 return data;
1019 }
1020 void deallocate(T *p, std::size_t n) {
1021 /* executor_reference.template mark_unused<T, Host_Allocator>(p, n); */
1022 executor_parent.template mark_unused<T, Host_Allocator>(p, n);
1023 }
1024 template <typename... Args>
1025 inline void construct(T *p, Args... args) noexcept {
1026 // Do nothing here - we reuse the content of the last owner
1027 }
1028 void destroy(T *p) {
1029 // Do nothing here - Contents will be destroyed when the buffer manager is
1030 // destroyed, not before
1031 }
1032};
1033template <typename T, typename U, typename Host_Allocator, typename Executor>
1034constexpr bool
1035operator==(allocator_slice<T, Host_Allocator, Executor> const &,
1036 allocator_slice<U, Host_Allocator, Executor> const &) noexcept {
1037 return false;
1038}
1039template <typename T, typename U, typename Host_Allocator, typename Executor>
1040constexpr bool
1041operator!=(allocator_slice<T, Host_Allocator, Executor> const &,
1042 allocator_slice<U, Host_Allocator, Executor> const &) noexcept {
1043 return true;
1044}
1045
1046} // namespace detail
1047} // namespace kernel_aggregation
1048} // namespace cppuddle
1049
1050
1051
1052namespace CPPUDDLE_HPX_EXECUTOR_SPECIALIZATION_NS {
1053 // TODO Unfortunately does not work that way! Create trait that works for Executor Slices with
1054 // compatible underlying executor types
1055 /* template<typename E> */
1056 /* struct is_one_way_executor<typename aggregated_executor<E>::executor_slice> */
1057 /* : std::true_type */
1058 /* {}; */
1059 /* template<typename E> */
1060 /* struct is_two_way_executor<typename aggregated_executor<E>::executor_slice> */
1061 /* : std::true_type */
1062 /* {}; */
1063
1064#if defined(HPX_HAVE_CUDA) || defined(HPX_HAVE_HIP)
1065 // Workaround for the meantime: Manually create traits for compatible types:
1066template <>
1067struct is_one_way_executor<
1068 typename cppuddle::kernel_aggregation::detail::aggregated_executor<
1069 hpx::cuda::experimental::cuda_executor>::executor_slice>
1070 : std::true_type {};
1071template <>
1072struct is_two_way_executor<
1073 typename cppuddle::kernel_aggregation::detail::aggregated_executor<
1074 hpx::cuda::experimental::cuda_executor>::executor_slice>
1075 : std::true_type {};
1076#endif
1077}
1078
1079#endif
executor_slice(aggregated_executor &parent, const size_t slice_id, const size_t number_slices, const size_t max_number_slices)
Definition aggregation_executors_and_allocators.hpp:452