39 #ifdef SYCL2020_FEATURE_GROUP_REDUCTION
40 return details::exclusive_sum_atomic_decoupled_v5<T, 512>(q, buf1, len);
53 #ifdef SYCL2020_FEATURE_GROUP_REDUCTION
54 return details::exclusive_sum_atomic_decoupled_v5_usm<T, 512>(sched, buf1, len);
61 template sycl::buffer<u32>
scan_exclusive(sycl::queue &q, sycl::buffer<u32> &buf1,
u32 len);
66 sycl::queue &q, sycl::buffer<u32> &buf_flags,
u32 len) {
75 template<
class Tret,
class T>
77 const sham::DeviceScheduler_ptr &sched,
102 const T *__restrict values,
103 const T *__restrict bin_edges,
104 Tret *__restrict counts) {
106 if (values[i] < bin_edges[0] || values[i] >= bin_edges[nbins]) {
111 u32 end_range = nbins + 1;
113 while (end_range - start_range > 1) {
114 u32 mid_range = (start_range + end_range) / 2;
116 if (values[i] < bin_edges[mid_range]) {
117 end_range = mid_range;
119 start_range = mid_range;
127 sycl::memory_order_relaxed,
128 sycl::memory_scope_device,
129 sycl::access::address_space::global_space>
130 cnt(counts[start_range]);
139 const sham::DeviceScheduler_ptr &sched,
145 const sham::DeviceScheduler_ptr &sched,
151 const sham::DeviceScheduler_ptr &sched,
157 const sham::DeviceScheduler_ptr &sched,
165 const sham::DeviceScheduler_ptr &sched,
174 auto value_filter = [&]() {
187 const T *__restrict keys,
188 const T *__restrict bin_edges,
189 u32 *__restrict key_filter) {
191 if (keys[i] < bin_edges[0] || keys[i] >= bin_edges[nbins]) {
201 return valid_key_idxs;
215 if (valid_key_count > 0) {
222 const T *__restrict keys,
223 const T *__restrict values,
224 const u32 *__restrict valid_keys_idxs,
225 T *__restrict valid_keys,
226 T *__restrict valid_values) {
227 u32 src_key = valid_keys_idxs[i];
228 valid_keys[i] = keys[src_key];
229 valid_values[i] = values[src_key];
238 bin_counts.set_val_at_idx(bin_counts.
get_size() - 1, 0);
247 if (valid_key_count > 0) {
251 if (pow2_len_key > valid_key_count) {
252 valid_keys.
resize(pow2_len_key);
253 valid_values.
resize(pow2_len_key);
259 pow2_len_key - valid_key_count,
260 [offset_start = valid_key_count](
261 u32 i, T *__restrict valid_keys, T *__restrict valid_values) {
262 u32 key_id = offset_start + i;
263 valid_keys[key_id] = shambase::get_max<T>();
264 valid_values[key_id] = shambase::get_max<T>();
272 sched, valid_keys, valid_values, pow2_len_key);
275 return {std::move(valid_values), std::move(offsets_bins)};
279 const sham::DeviceScheduler_ptr &sched,
286 const sham::DeviceScheduler_ptr &sched,
std::uint32_t u32
32 bit unsigned integer
std::uint64_t u64
64 bit unsigned integer
Shamrock assertion utility.
#define SHAM_ASSERT(x)
Shorthand for SHAM_ASSERT_NAMED without a message.
A buffer allocated in USM (Unified Shared Memory).
void resize(size_t new_size, bool keep_data=true)
Resizes the buffer to a given size.
void fill(T value, std::array< size_t, 2 > idx_range)
Fill a subpart of the buffer with a given value.
T get_val_at_idx(size_t idx) const
Get the value at a given index in the buffer.
size_t get_size() const
Gets the number of elements in the buffer.
void expand(u32 add_sz)
Expand the buffer by add_sz elements.
main include file for the shamalgs algorithms
void kernel_call(sham::DeviceQueue &q, RefIn in, RefOut in_out, u32 n, Functor &&func, SourceLocation &&callsite=SourceLocation{})
Submit a kernel to a SYCL queue.
void sort_by_key_pow2_len(sycl::queue &q, sycl::buffer< Tkey > &buf_key, sycl::buffer< Tval > &buf_values, u32 len)
Sort the buffer according to the key order.
namespace containing the numeric algorithms of shamalgs
BinnedCompute< T > binned_init_compute(const sham::DeviceScheduler_ptr &sched, const sham::DeviceBuffer< T > &bin_edges, u64 nbins, const sham::DeviceBuffer< T > &values, const sham::DeviceBuffer< T > &keys, u32 len)
Prepare binned data for per-bin computation.
sham::DeviceBuffer< Tret > device_histogram(const sham::DeviceScheduler_ptr &sched, const sham::DeviceBuffer< T > &bin_edges, u64 nbins, const sham::DeviceBuffer< T > &values, u32 len)
Compute the histogram of values between bin_edges.
sycl::buffer< T > scan_exclusive(sycl::queue &q, sycl::buffer< T > &buf1, u32 len)
Computes the exclusive sum of elements in a SYCL buffer.
std::tuple< std::optional< sycl::buffer< u32 > >, u32 > stream_compact(sycl::queue &q, sycl::buffer< u32 > &buf_flags, u32 len)
Stream compaction algorithm.
constexpr T roundup_pow2(T v) noexcept
round up to the next power of two Source : https://graphics.stanford.edu/~seander/bithacks....
T & get_check_ref(const std::unique_ptr< T > &ptr, SourceLocation loc=SourceLocation())
Takes a std::unique_ptr and returns a reference to the object it holds. It throws a std::runtime_erro...
sham::DeviceBuffer< T > exclusive_sum_fallback_usm(const sham::DeviceScheduler_ptr &sched, sham::DeviceBuffer< T > &buf1, u32 len)
Exclusive sum fallback on USM.
sycl::buffer< T > exclusive_sum_fallback(sycl::queue &q, sycl::buffer< T > &buf1, u32 len)
Exclusive sum fallback on SYCL buffer.
std::tuple< std::optional< sycl::buffer< u32 > >, u32 > stream_compact_excl_scan(sycl::queue &q, sycl::buffer< u32 > &buf_flags, u32 len)
Stream compaction algorithm using exclusive summation.
A class that references multiple buffers or similar objects.
Structure holding the result of binning values for further computation.