434 class executor_slice {
436 aggregated_executor<Executor> &parent;
441 size_t launch_counter{0};
442 size_t buffer_counter{0};
443 bool notify_parent_about_destruction{true};
448 size_t number_slices;
451 using executor_t = Executor;
452 executor_slice(aggregated_executor &parent, const size_t slice_id,
453 const size_t number_slices, const size_t max_number_slices)
454 : parent(parent), notify_parent_about_destruction(true),
455 number_slices(number_slices), id(slice_id), max_slices(max_number_slices) {
456 assert(parent.max_slices == max_slices);
457 assert(number_slices >= 1);
458 assert(number_slices <= max_slices);
460 ~executor_slice(void) {
461 // Don't notify parent if we moved away from this executor_slice
462 if (notify_parent_about_destruction) {
463 // Executor should be done by the time of destruction
464 // -> check here before notifying parent
466 assert(parent.max_slices == max_slices);
467 assert(number_slices >= 1);
468 assert(number_slices <= max_slices);
469 // parent still in execution mode?
470 assert(parent.slices_exhausted == true);
471 // all kernel launches done?
472 assert(launch_counter == parent.function_calls.size());
473 // Notifiy parent that this aggregation slice is one
474 parent.reduce_usage_counter();
477 executor_slice(const executor_slice &other) = delete;
478 executor_slice &operator=(const executor_slice &other) = delete;
479 executor_slice(executor_slice &&other)
480 : parent(other.parent), launch_counter(std::move(other.launch_counter)),
481 buffer_counter(std::move(other.buffer_counter)),
482 number_slices(std::move(other.number_slices)),
483 id(std::move(other.id)), max_slices(std::move(other.max_slices)) {
484 other.notify_parent_about_destruction = false;
486 executor_slice &operator=(executor_slice &&other) {
487 parent = other.parent;
488 launch_counter = std::move(other.launch_counter);
489 buffer_counter = std::move(other.buffer_counter);
490 number_slices = std::move(other.number_slices);
491 id = std::move(other.id);
492 max_slices = std::move(other.max_slices);
493 other.notify_parent_about_destruction = false;
495 template <typename T, typename Host_Allocator>
496 allocator_slice<T, Host_Allocator, Executor> make_allocator() {
497 return allocator_slice<T, Host_Allocator, Executor>(*this);
499 bool sync_aggregation_slices() {
500 assert(parent.slices_exhausted == true);
501 auto ret = parent.sync_aggregation_slices(launch_counter);
505 template <typename F, typename... Ts> void post(F &&f, Ts &&...ts) {
506 // we should only execute function calls once all slices
507 // have been given away (-> Executor Slices start)
508 assert(parent.slices_exhausted == true);
509 parent.post(launch_counter, std::forward<F>(f), std::forward<Ts>(ts)...);
512 template <typename F, typename... Ts>
513 hpx::future<void> async(F &&f, Ts &&...ts) {
514 // we should only execute function calls once all slices
515 // have been given away (-> Executor Slices start)
516 assert(parent.slices_exhausted == true);
517 hpx::future<void> ret_fut = parent.async(
518 launch_counter, std::forward<F>(f), std::forward<Ts>(ts)...);
524 template <typename F, typename... Ts>
525 friend decltype(auto) tag_invoke(hpx::parallel::execution::post_t,
526 executor_slice& exec, F&& f, Ts&&... ts)
528 return exec.post(std::forward<F>(f), std::forward<Ts>(ts)...);
532 template <typename F, typename... Ts>
533 friend decltype(auto) tag_invoke(
534 hpx::parallel::execution::async_execute_t, executor_slice& exec,
538 std::forward<F>(f), std::forward<Ts>(ts)...);
541 template <typename F, typename... Ts>
542 hpx::shared_future<void> wrap_async(F &&f, Ts &&...ts) {
543 // we should only execute function calls once all slices
544 // have been given away (-> Executor Slices start)
545 assert(parent.slices_exhausted == true);
546 hpx::shared_future<void> ret_fut = parent.wrap_async(
547 launch_counter, std::forward<F>(f), std::forward<Ts>(ts)...);
554 template <typename T, typename Host_Allocator> T *get(const size_t size) {
555 assert(parent.slices_exhausted == true);
556 T *aggregated_buffer =
557 parent.get<T, Host_Allocator>(size, buffer_counter);
559 assert(buffer_counter > 0);
560 return aggregated_buffer;
563 Executor& get_underlying_executor(void) {
564 assert(parent.executor_wrapper);
565 return *(parent.executor_wrapper);
598 T *get(const size_t size, const size_t slice_alloc_counter) {
599 assert(slices_exhausted == true);
600 assert(executor_wrapper);
601 assert(executor_slices_alive == true);
602 // Add aggreated buffer entry in case it hasn't happened yet for this call
603 // First: Check if it already has happened
604 if (buffer_counter <= slice_alloc_counter) {
605 // we might be the first! Lock...
606 std::lock_guard<aggregation_mutex_t> guard(buffer_mut);
608 if (buffer_counter <= slice_alloc_counter) {
609 constexpr bool manage_content_lifetime = false;
610 buffers_in_use = true;
612 // Default location -- useful for GPU builds as we otherwise create way too
613 // many different buffers for different aggregation sizes on different GPUs
614 /* size_t location_id = gpu_id * instances_per_gpu; */
615 // Use integer conversion to only use 0 16 32 ... as buckets
616 size_t location_id = ((hpx::get_worker_thread_num() % cppuddle::number_instances) / 16) * 16;
617#ifdef CPPUDDLE_HAVE_HPX_AWARE_ALLOCATORS
618 if (max_slices == 1) {
619 // get prefered location: aka the current hpx threads location
620 // Usually handy for CPU builds where we want to use the buffers
621 // close to the current CPU core
622 /* location_id = (hpx::get_worker_thread_num() / instances_per_gpu) * instances_per_gpu; */
623 /* location_id = (gpu_id) * instances_per_gpu; */
624 // division makes sure that we always use the same instance to store our gpu buffers.
627 // Get shiny and new buffer that will be shared between all slices
628 // Buffer might be recycled from previous allocations by the
629 // buffer_interface...
630 T *aggregated_buffer =
631 cppuddle::memory_recycling::detail::buffer_interface::get<
632 T, Host_Allocator>(size, manage_content_lifetime, location_id,
634 // Create buffer entry for this buffer
635 buffer_allocations.emplace_back(static_cast<void *>(aggregated_buffer),
636 size, 1, true, location_id, gpu_id);
639 // if previousely used the buffer should not be in usage anymore
640 const auto exists = buffer_allocations_map.count(
641 static_cast<void *>(aggregated_buffer));
643 const auto previous_usage_id =
644 buffer_allocations_map[static_cast<void *>(aggregated_buffer)];
646 std::get<3>(buffer_allocations[previous_usage_id]);
650 buffer_allocations_map.insert_or_assign(static_cast<void *>(aggregated_buffer),
653 assert (buffer_counter == slice_alloc_counter);
654 buffer_counter = buffer_allocations.size();
657 return aggregated_buffer;
660 assert(buffers_in_use == true);
661 assert(std::get<3>(buffer_allocations[slice_alloc_counter])); // valid
662 assert(std::get<2>(buffer_allocations[slice_alloc_counter]) >= 1);
664 // Buffer entry should already exist:
665 T *aggregated_buffer = static_cast<T *>(
666 std::get<0>(buffer_allocations[slice_alloc_counter]));
667 // Error handling: Size is wrong?
668 assert(size == std::get<1>(buffer_allocations[slice_alloc_counter]));
669 // Notify that one more slice has visited this buffer allocation
670 std::get<2>(buffer_allocations[slice_alloc_counter])++;
671 return aggregated_buffer;
676 void mark_unused(T *p, const size_t size) {
677 assert(slices_exhausted == true);
678 assert(executor_wrapper);
680 void *ptr_key = static_cast<void*>(p);
681 size_t slice_alloc_counter = buffer_allocations_map[p];
683 assert(slice_alloc_counter < buffer_allocations.size());
684 /*auto &[buffer_pointer_any, buffer_size, buffer_allocation_counter, valid] =
685 buffer_allocations[slice_alloc_counter];*/
686 auto buffer_pointer_void = std::get<0>(buffer_allocations[slice_alloc_counter]);
687 const auto buffer_size = std::get<1>(buffer_allocations[slice_alloc_counter]);
688 auto &buffer_allocation_counter = std::get<2>(buffer_allocations[slice_alloc_counter]);
689 auto &valid = std::get<3>(buffer_allocations[slice_alloc_counter]);
690 const auto &location_id = std::get<4>(buffer_allocations[slice_alloc_counter]);
691 const auto &gpu_id = std::get<5>(buffer_allocations[slice_alloc_counter]);
693 T *buffer_pointer = static_cast<T *>(buffer_pointer_void);
695 assert(buffer_size == size);
696 assert(p == buffer_pointer);
697 // assert(buffer_pointer == p || buffer_pointer == nullptr);
698 // Slice is done with this buffer
699 buffer_allocation_counter--;
700 // Check if all slices are done with this buffer?
701 if (buffer_allocation_counter == 0) {
702 // Yes! "Deallocate
" by telling the recylcer the buffer is fit for reusage
703 std::lock_guard<aggregation_mutex_t> guard(buffer_mut);
704 // Only mark unused if another buffer has not done so already (and marked
707 assert(buffers_in_use == true);
708 cppuddle::memory_recycling::detail::buffer_interface::mark_unused<
709 T, Host_Allocator>(buffer_pointer, buffer_size, location_id,
711 // mark buffer as invalid to prevent any other slice from marking the
715 const size_t current_deallocs = ++dealloc_counter;
716 if (current_deallocs == buffer_counter) {
717 std::lock_guard<aggregation_mutex_t> guard(mut);
718 buffers_in_use = false;
719 if (!executor_slices_alive && !buffers_in_use) {
720 slices_exhausted = false;
722 executor_wrapper.reset(nullptr);
737 bool sync_aggregation_slices(const size_t slice_launch_counter) {
738 std::lock_guard<aggregation_mutex_t> guard(mut);
739 assert(slices_exhausted == true);
740 assert(executor_wrapper);
741 // Add function call object in case it hasn't happened for this launch yet
742 if (overall_launch_counter <= slice_launch_counter) {
743 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
744 if (overall_launch_counter <= slice_launch_counter) {
745 function_calls.emplace_back(current_slices, false, *executor_wrapper);
746 overall_launch_counter = function_calls.size();
747 return function_calls[slice_launch_counter].sync_aggregation_slices(
748 last_stream_launch_done);
752 return function_calls[slice_launch_counter].sync_aggregation_slices(
753 last_stream_launch_done);
758 void post(const size_t slice_launch_counter, F &&f, Ts &&...ts) {
759 std::lock_guard<aggregation_mutex_t> guard(mut);
760 assert(slices_exhausted == true);
761 assert(executor_wrapper);
762 // Add function call object in case it hasn't happened for this launch yet
763 if (overall_launch_counter <= slice_launch_counter) {
764 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
765 if (overall_launch_counter <= slice_launch_counter) {
766 function_calls.emplace_back(current_slices, false, *executor_wrapper);
767 overall_launch_counter = function_calls.size();
768 function_calls[slice_launch_counter].post_when(
769 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
774 function_calls[slice_launch_counter].post_when(
775 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
781 hpx::future<void> async(const size_t slice_launch_counter, F &&f,
783 std::lock_guard<aggregation_mutex_t> guard(mut);
784 assert(slices_exhausted == true);
785 assert(executor_wrapper);
786 // Add function call object in case it hasn't happened for this launch yet
787 if (overall_launch_counter <= slice_launch_counter) {
788 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
789 if (overall_launch_counter <= slice_launch_counter) {
790 function_calls.emplace_back(current_slices, true, *executor_wrapper);
791 overall_launch_counter = function_calls.size();
792 return function_calls[slice_launch_counter].async_when(
793 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
797 return function_calls[slice_launch_counter].async_when(
798 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
802 hpx::shared_future<void> wrap_async(const size_t slice_launch_counter, F &&f,
804 std::lock_guard<aggregation_mutex_t> guard(mut);
805 assert(slices_exhausted == true);
806 assert(executor_wrapper);
807 // Add function call object in case it hasn't happened for this launch yet
808 if (overall_launch_counter <= slice_launch_counter) {
809 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
810 if (overall_launch_counter <= slice_launch_counter) {
811 function_calls.emplace_back(current_slices, true, *executor_wrapper);
812 overall_launch_counter = function_calls.size();
813 return function_calls[slice_launch_counter].wrap_async(
814 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
818 return function_calls[slice_launch_counter].wrap_async(
819 last_stream_launch_done, std::forward<F>(f), std::forward<Ts>(ts)...);
827 std::optional<hpx::future<executor_slice>> request_executor_slice() {
828 std::lock_guard<aggregation_mutex_t> guard(mut);
829 if (!slices_exhausted) {
830 const size_t local_slice_id = ++current_slices;
831 if (local_slice_id == 1) {
832 // Cleanup leftovers from last run if any
833 // TODO still required? Should be clean here already
834 function_calls.clear();
835 overall_launch_counter = 0;
836 std::lock_guard<aggregation_mutex_t> guard(buffer_mut);
838 for (const auto &buffer_entry : buffer_allocations) {
839 const auto &[buffer_pointer_any, buffer_size,
840 buffer_allocation_counter, valid, location_id, device_id] =
845 buffer_allocations.clear();
846 buffer_allocations_map.clear();
849 assert(executor_slices_alive == false);
850 assert(buffers_in_use == false);
851 executor_slices_alive = true;
852 buffers_in_use = false;
857 // if (mode == aggregated_executor_modes::STRICT ) {
858 // slices_full_promise = hpx::promise<void>{};
862 // Create Executor Slice future -- that will be returned later
863 hpx::future<executor_slice> ret_fut;
864 if (local_slice_id < max_slices) {
865 executor_slices.emplace_back(hpx::promise<executor_slice>{});
867 executor_slices[local_slice_id - 1].get_future();
869 launched_slices = current_slices;
870 ret_fut = hpx::make_ready_future(executor_slice{*this,
871 executor_slices.size(), launched_slices, max_slices});
874 // Are we the first slice? If yes, add continuation set the
876 // futures to ready if the launch conditions are met
877 if (local_slice_id == 1) {
879 assert(!executor_wrapper);
880 cppuddle::executor_recycling::executor_pool::select_device<
881 Executor, cppuddle::executor_recycling::round_robin_pool_impl<Executor>>(
883 executor_wrapper.reset(
884 new cppuddle::executor_recycling::executor_interface<
886 cppuddle::executor_recycling::round_robin_pool_impl<Executor>>(
888 // Renew promise that all slices will be ready as the primary launch
890 hpx::shared_future<void> fut;
891 if (mode == aggregated_executor_modes::EAGER ||
892 mode == aggregated_executor_modes::ENDLESS) {
893 // Fallback launch condidtion: Launch as soon as the underlying stream
895 /* auto slices_full_fut = slices_full_promise.get_future(); */
896 cppuddle::executor_recycling::executor_pool::select_device<
898 cppuddle::executor_recycling::round_robin_pool_impl<Executor>>(gpu_id);
899 auto exec_fut = (*executor_wrapper).get_future();
900 /* auto fut = hpx::when_any(exec_fut, slices_full_fut); */
901 fut = std::move(exec_fut);
902 // Launch all executor slices within this continuation
903 current_continuation = fut.then([this](auto &&fut) {
904 std::lock_guard<aggregation_mutex_t> guard(mut);
905 // if (!slices_exhausted) {
906 slices_exhausted = true;
907 launched_slices = current_slices;
909 for (auto &slice_promise : executor_slices) {
910 slice_promise.set_value(
911 executor_slice{*this, id, launched_slices, max_slices});
914 executor_slices.clear();
918 if (local_slice_id >= max_slices &&
919 mode != aggregated_executor_modes::ENDLESS) {
920 slices_exhausted = true;
921 launched_slices = current_slices;
923 for (auto &slice_promise : executor_slices) {
924 slice_promise.set_value(
925 executor_slice{*this, id, launched_slices, max_slices});
928 executor_slices.clear();
932 // Return empty optional as failure
933 return std::optional<hpx::future<executor_slice>>{};
937 void reduce_usage_counter(void) {
938 /* std::lock_guard<aggregation_mutex_t> guard(mut); */
939 assert(slices_exhausted == true);
940 assert(executor_wrapper);
941 assert(executor_slices_alive == true);
942 assert(launched_slices >= 1);
943 assert(current_slices >= 0 && current_slices <= launched_slices);
944 const size_t local_slice_id = --current_slices;
945 // Last slice goes out scope?
946 if (local_slice_id == 0) {
947 // Mark executor fit for reusage
948 if (mode != aggregated_executor_modes::STRICT )
949 current_continuation.get(); // clear remaining task in case it was not run yet
950 std::lock_guard<aggregation_mutex_t> guard(mut);
951 executor_slices_alive = false;
952 if (!executor_slices_alive && !buffers_in_use) {
954 slices_exhausted = false;
955 executor_wrapper.reset(nullptr);
959 ~aggregated_executor(void) {
961 assert(current_slices == 0);
962 assert(executor_slices_alive == false);
963 assert(buffers_in_use == false);
965 // if (mode != aggregated_executor_modes::STRICT ) {
966 // slices_full_promise.set_value(); // Trigger slices launch condition continuation
969 // Cleanup leftovers from last run if any
970 function_calls.clear();
971 overall_launch_counter = 0;
973 for (const auto &buffer_entry : buffer_allocations) {
974 const auto &[buffer_pointer_any, buffer_size, buffer_allocation_counter,
975 valid, location_id, device_id] = buffer_entry;
979 buffer_allocations.clear();
980 buffer_allocations_map.clear();
983 assert(buffer_allocations.empty());
984 assert(buffer_allocations_map.empty());
987 aggregated_executor(const size_t number_slices,
988 aggregated_executor_modes mode, const size_t gpu_id = 0)
989 : max_slices(number_slices), current_slices(0), slices_exhausted(false),
990 dealloc_counter(0), mode(mode), executor_slices_alive(false),
991 buffers_in_use(false), gpu_id(gpu_id),
992 executor_wrapper(nullptr),
993 current_continuation(hpx::make_ready_future()),
994 last_stream_launch_done(hpx::make_ready_future()) {}