41 static constexpr std::string_view variant_type_name =
"reference";
45 static constexpr std::string_view variant_type_name =
"naive_gpu";
49 static constexpr std::string_view variant_type_name =
"gpu_team_fetching";
53 static constexpr std::string_view variant_type_name =
"gpu_oversubscribe";
66 template<
class T,
class Tbins,
class... Targs,
class Tfunctor>
67 inline void compute_histogram_reference(
78 auto cpu_basic_impl = [&](
const std::vector<Tbins> &bin_edge_inf,
79 const std::vector<Tbins> &bin_edge_sup,
80 const std::vector<Targs> &...in_data,
81 std::vector<T> &result) {
82 for (
size_t ibin = 0; ibin < nbins; ibin++) {
83 Tbins edge_inf = bin_edge_inf[ibin];
84 Tbins edge_sup = bin_edge_sup[ibin];
88 for (
size_t i = 0; i < element_count; i++) {
89 bool has_value =
false;
90 auto tmp = functor(edge_inf, edge_sup, in_data[i]..., has_value);
96 result[ibin] = accumulator;
109 template<
class T,
class Tbins,
class... Targs,
class Tfunctor>
110 inline void compute_histogram_naive_gpu(
111 const sham::DeviceScheduler_ptr &dev_sched,
115 size_t element_count,
121 dev_sched->get_queue(),
125 [element_count, functor](
127 const Tbins *__restrict bin_edge_inf,
128 const Tbins *__restrict bin_edge_sup,
129 const Targs *__restrict... in_data,
130 T *__restrict result) {
131 Tbins edge_inf = bin_edge_inf[ibin];
132 Tbins edge_sup = bin_edge_sup[ibin];
136 for (size_t i = 0; i < element_count; i++) {
137 bool has_value = false;
138 T tmp = functor(edge_inf, edge_sup, in_data[i]..., has_value);
144 result[ibin] = accumulator;
148 template<
class T,
class Tbins,
class... Targs,
class Tfunctor>
149 inline void compute_histogram_gpu_team_fetching(
150 const sham::DeviceScheduler_ptr &dev_sched,
154 size_t element_count,
159 sham::kernel_call_hndl(
160 dev_sched->get_queue(),
164 [element_count, functor](
166 const Tbins *__restrict bin_edge_inf,
167 const Tbins *__restrict bin_edge_sup,
168 const Targs *__restrict... in_data,
169 T *__restrict result) {
170 return [=, in_data = std::tuple{in_data...}](sycl::handler &cgh) {
171 u32 group_size = 128;
172 u32 group_cnt = shambase::group_count(nbins, group_size);
175 group_cnt = (group_cnt + 3) / 4 * 4;
176 u32 corrected_len = group_cnt * group_size;
179 = sycl::local_accessor<std::tuple<Targs...>, 1>(group_size, cgh);
182 sycl::nd_range<1>{corrected_len, group_size},
183 [=](sycl::nd_item<1> item) {
184 u32 local_id = item.get_local_id(0);
185 u32 group_tile_id = item.get_group_linear_id();
186 u32 ibin = group_tile_id * group_size + local_id;
188 bool is_valid_point = (ibin < nbins);
189 Tbins edge_inf = is_valid_point ? bin_edge_inf[ibin] : Tbins{};
190 Tbins edge_sup = is_valid_point ? bin_edge_sup[ibin] : Tbins{};
194 for (
size_t i = 0; i < element_count; i += group_size) {
196 item.barrier(sycl::access::fence_space::local_space);
198 if (i + local_id < element_count) {
200 [&](
auto &...in_data) {
202 = std::tuple{in_data[i + local_id]...};
207 item.barrier(sycl::access::fence_space::local_space);
209 if (is_valid_point) {
210 for (
size_t lane = 0; lane < group_size; lane++) {
211 if (i + lane >= element_count) {
214 bool has_value =
false;
216 [&](
auto &...local_accs) {
230 item.barrier(sycl::access::fence_space::local_space);
233 if (is_valid_point) {
234 result[ibin] = local_sum;
241 template<
class T,
class Tbins,
class... Targs,
class Tfunctor>
242 inline void compute_histogram_gpu_oversubscribe(
243 const sham::DeviceScheduler_ptr &dev_sched,
248 size_t element_count,
253 sham::kernel_call_hndl(
254 dev_sched->get_queue(),
258 [element_count, functor, group_size, nbins](
259 u32 nbins_oversubscribed,
260 const Tbins *__restrict bin_edge_inf,
261 const Tbins *__restrict bin_edge_sup,
262 const Targs *__restrict... in_data,
263 T *__restrict result) {
264 return [=, in_data = std::tuple{in_data...}](sycl::handler &cgh) {
265 u32 group_cnt = shambase::group_count(nbins_oversubscribed, group_size);
268 group_cnt = (group_cnt + 3) / 4 * 4;
270 u32 corrected_len = group_cnt * group_size;
273 sycl::nd_range<1>{corrected_len, group_size},
274 [=](sycl::nd_item<1> item) {
275 u32 local_id = item.get_local_id(0);
276 u32 ibin = item.get_group_linear_id();
278 bool is_valid_point = (ibin < nbins);
279 Tbins edge_inf = is_valid_point ? bin_edge_inf[ibin] : Tbins{};
280 Tbins edge_sup = is_valid_point ? bin_edge_sup[ibin] : Tbins{};
286 for (
size_t i = 0; i < element_count; i += group_size) {
288 if (i + local_id < element_count) {
290 bool has_value =
false;
295 [&](
auto &...in_data) {
299 in_data[i + local_id]...,
313 auto group_sum = sycl::reduce_over_group(
314 item.get_group(), local_sum, sycl::plus<T>{});
316 if (is_valid_point && local_id == 0) {
317 result[ibin] = group_sum;