Shamrock 2025.10.0
Astrophysical Code
Loading...
Searching...
No Matches
scan_exclusive_sum_in_place.cpp
Go to the documentation of this file.
1// -------------------------------------------------------//
2//
3// SHAMROCK code for hydrodynamics
4// Copyright (c) 2021-2026 Timothée David--Cléris <tim.shamrock@proton.me>
5// SPDX-License-Identifier: CeCILL Free Software License Agreement v2.1
6// Shamrock is licensed under the CeCILL 2.1 License, see LICENSE for more information
7//
8// -------------------------------------------------------//
9
16
27#include <numeric>
28
29#if defined(__has_include)
30 #if __has_include(<AdaptiveCpp/algorithms/numeric.hpp>)
31 #include <AdaptiveCpp/algorithms/numeric.hpp>
32 #define ACPP_ALG_AVAILABLE
33 #endif
34#endif
35
36namespace {
37
38#ifdef __ACPP__
39 template<class T>
40 void scan_exclusive_sum_in_place_std_scan_single_task_acpp(
41 sham::DeviceBuffer<T> &buf1, u32 len) {
42
43 auto &q = buf1.get_dev_scheduler_ptr()->get_queue();
44
45 sycl::queue &q_s = q.q;
46
47 if (q_s.is_host()) {
48 sham::EventList deps{};
49 T *in_out_ptr = buf1.get_write_access(deps);
50
51 auto e = q.submit(deps, [&](sycl::handler &cgh) {
52 cgh.single_task([=]() {
53 std::exclusive_scan(in_out_ptr, in_out_ptr + len, in_out_ptr, T{});
54 });
55 });
56
58 } else {
59 auto acc_src = buf1.copy_to_stdvec_idx_range(0, len);
60 std::exclusive_scan(acc_src.begin(), acc_src.end(), acc_src.begin(), T{});
61 buf1.copy_from_stdvec(acc_src, len);
62 }
63 }
64#endif
65
66 template<class T>
67 void scan_exclusive_sum_in_place_fallback(sham::DeviceBuffer<T> &buf1, u32 len) {
68 auto acc_src = buf1.copy_to_stdvec_idx_range(0, len);
69 std::exclusive_scan(acc_src.begin(), acc_src.end(), acc_src.begin(), 0);
70 buf1.copy_from_stdvec(acc_src, len);
71 }
72
73#ifdef SYCL2020_FEATURE_GROUP_REDUCTION
74 template<class T>
75 void scan_exclusive_sum_in_place_decoupled_lookback_512(sham::DeviceBuffer<T> &buf1, u32 len) {
76 shamalgs::numeric::details::exclusive_sum_atomic_decoupled_v5_usm_in_place<T, 512>(
77 buf1, len);
78 }
79#endif
80
81#ifdef ACPP_ALG_AVAILABLE
82 template<class T>
83 void scan_exclusive_sum_in_place_adaptivecpp(sham::DeviceBuffer<T> &buf1, u32 len) {
84 auto &q = buf1.get_dev_scheduler_ptr()->get_queue().q;
85
86 acpp::algorithms::util::allocation_cache cache{
87 acpp::algorithms::util::allocation_type::device};
88 acpp::algorithms::util::allocation_group scratch{&cache, q.get_device()};
89
91
92 sham::EventList deps{};
93 const T *in_out_ptr = buf1.get_read_access(deps);
94 T *temp_ptr = temp.get_write_access(deps);
95
96 sycl::event e = adaptivecpp::algorithms::exclusive_scan(
97 q, scratch, in_out_ptr, in_out_ptr + len, temp_ptr, T{}, deps.get_events());
98 deps.set_consumed(true);
99
100 buf1.complete_event_state(e);
101 temp.complete_event_state(e);
102
103 buf1.copy_from(temp, len);
104 }
105#endif
106} // namespace
107
108namespace shamalgs::primitives {
109
111 namespace impl {
112
114 struct StdScan {
115 static constexpr std::string_view variant_type_name = "std_scan";
116 };
117
118#ifdef __ACPP__
121 struct StdScanSingleTaskAcpp {
122 static constexpr std::string_view variant_type_name = "std_scan_single_task_acpp";
123 };
124#endif
125
126#ifdef SYCL2020_FEATURE_GROUP_REDUCTION
128 struct DecoupledLookback512 {
129 static constexpr std::string_view variant_type_name = "decoupled_lookback_512";
130 };
131#endif
132
133#ifdef ACPP_ALG_AVAILABLE
135 struct AdaptiveCppAlg {
136 static constexpr std::string_view variant_type_name = "acpp_alg";
137 };
138#endif
139
141 constexpr std::string_view scan_exclusive_sum_in_place_impl_name
142 = "scan_exclusive_sum_in_place";
143
145 StdScan
146#ifdef __ACPP__
147 ,
148 StdScanSingleTaskAcpp
149#endif
150#ifdef SYCL2020_FEATURE_GROUP_REDUCTION
151 ,
152 DecoupledLookback512
153#endif
154#ifdef ACPP_ALG_AVAILABLE
155 ,
156 AdaptiveCppAlg
157#endif
158 >
159 scan_exclusive_sum_in_place_impl{[](const sham::DeviceScheduler_ptr &, auto &self) {
160#ifdef __MACH__ // decoupled lookback perf on mac os is awful
161 #ifdef __ACPP__ // for acpp we gain using enqueue custom operation instead of copying
162 self.set(StdScanSingleTaskAcpp{});
163 #else
164 self.set(StdScan{});
165 #endif
166#else
167 #ifdef SYCL2020_FEATURE_GROUP_REDUCTION
168 self.set(DecoupledLookback512{});
169 #else
170 self.set(StdScan{});
171 #endif
172#endif
173 }};
174
175 // Must come after the global it registers: same TU, so it is initialized after it
177 scan_exclusive_sum_in_place_impl_name, scan_exclusive_sum_in_place_impl);
178
179 } // namespace impl
180
181 template<class T>
183
184 if (len == 0) {
185 return;
186 }
187
188 if (len > buf1.get_size()) {
190 "The buffer is smaller than the length of the scan\n"
191 "len > buf1.get_size(), len = {}, buf1.get_size() = {}",
192 len,
193 buf1.get_size()));
194 }
195
196 if (!impl::scan_exclusive_sum_in_place_impl.is_set()) {
199 }
200
201 std::visit(
203 [&](impl::StdScan) {
204 scan_exclusive_sum_in_place_fallback(buf1, len);
205 },
206#ifdef __ACPP__
207 [&](impl::StdScanSingleTaskAcpp) {
208 scan_exclusive_sum_in_place_std_scan_single_task_acpp(buf1, len);
209 },
210#endif
211#ifdef SYCL2020_FEATURE_GROUP_REDUCTION
212 [&](impl::DecoupledLookback512) {
213 scan_exclusive_sum_in_place_decoupled_lookback_512(buf1, len);
214 },
215#endif
216#ifdef ACPP_ALG_AVAILABLE
217 [&](impl::AdaptiveCppAlg) {
218 scan_exclusive_sum_in_place_adaptivecpp(buf1, len);
219 },
220#endif
221 },
222 impl::scan_exclusive_sum_in_place_impl.get());
223 }
224
225 template void scan_exclusive_sum_in_place<u32>(sham::DeviceBuffer<u32> &buf1, u32 len);
226
227} // namespace shamalgs::primitives
Generic std::variant-based implementation selector.
std::uint32_t u32
32 bit unsigned integer
A buffer allocated in USM (Unified Shared Memory).
void complete_event_state(sycl::event e) const
Complete the event state of the buffer.
void copy_from_stdvec(const std::vector< T > &vec)
Copy the content of a std::vector into the buffer.
T * get_write_access(sham::EventList &depends_list, SourceLocation src_loc=SourceLocation{})
Get a read-write pointer to the buffer's data.
std::shared_ptr< DeviceScheduler > & get_dev_scheduler_ptr()
Gets the Device scheduler pointer corresponding to the held allocation.
void copy_from(const DeviceBuffer< T, new_target > &other, size_t copy_size)
Copies the content of another buffer to this one.
size_t get_size() const
Gets the number of elements in the buffer.
std::vector< T > copy_to_stdvec_idx_range(size_t begin, size_t end) const
Copies a specified range of elements from the buffer to a std::vector.
const T * get_read_access(sham::EventList &depends_list, SourceLocation src_loc=SourceLocation{}) const
Get a read-only pointer to the buffer's data.
Class to manage a list of SYCL events.
Definition EventList.hpp:32
void set_consumed(bool consumed)
Set the consumed state of the EventList (to be used with interop).
std::vector< sycl::event > & get_events()
Get the list of events.
Drop-in replacement for the hand-rolled "global variable + enum + name mapping.
This header file contains utility functions related to exception handling in the code.
Name-keyed registry of the implementation selectors (IImplVariant) of every algorithm.
#define SHAMALGS_REGISTER_IMPL(name, impl)
Register the implementation selector impl under name at static initialization.
void autoselect_impl(std::string_view alg, const sham::DeviceScheduler_ptr &sched)
Select the default implementation of alg for the device behind sched.
namespace to control implementation behavior
constexpr std::string_view scan_exclusive_sum_in_place_impl_name
Registry name, shared by its registration and the dispatch site(s).
namespace for primitive algorithm (e.g. sort, scan, reductions, ...)
void scan_exclusive_sum_in_place(sham::DeviceBuffer< T > &buf1, u32 len)
Compute exclusive prefix sum in-place on a device buffer.
void throw_with_loc(std::string message, SourceLocation loc=SourceLocation{})
Throw an exception and append the source location to it.
In-place exclusive scan (prefix sum) algorithm for device buffers.
std::exclusive_scan on a host copy of the buffer (portable fallback)
Build an overload set out of several callables, for use with std::visit.