Shamrock 2025.10.0
Astrophysical Code
Loading...
Searching...
No Matches
NeighbourCache.cpp
Go to the documentation of this file.
1// -------------------------------------------------------//
2//
3// SHAMROCK code for hydrodynamics
4// Copyright (c) 2021-2026 Timothée David--Cléris <tim.shamrock@proton.me>
5// SPDX-License-Identifier: CeCILL Free Software License Agreement v2.1
6// Shamrock is licensed under the CeCILL 2.1 License, see LICENSE for more information
7//
8// -------------------------------------------------------//
9
17
19#include "shambase/assert.hpp"
20#include "shambase/memory.hpp"
29
30template<class Tvec, class Tmorton, template<class> class SPHKernel>
31void shammodels::sph::modules::NeighbourCache<Tvec, Tmorton, SPHKernel>::start_neighbors_cache() {
32
33 // interface_control
34 using GhostHandle = sph::BasicSPHGhostHandler<Tvec>;
35 using GhostHandleCache = typename GhostHandle::CacheMap;
37
38 shambase::Timer time_neigh;
39 time_neigh.start();
40
41 StackEntry stack_loc{};
42
43 // do cache
44 auto build_neigh_cache = [&](u64 patch_id) {
45 shamlog_debug_ln("BasicSPH", "build particle cache id =", patch_id);
46
47 NamedStackEntry cache_build_stack_loc{"build cache"};
48
49 auto &mfield = storage.merged_xyzh.get().get(patch_id);
50
51 sham::DeviceBuffer<Tvec> &buf_xyz = mfield.template get_field_buf_ref<Tvec>(0);
52 sham::DeviceBuffer<Tscal> &buf_hpart = mfield.template get_field_buf_ref<Tscal>(1);
53
54 sham::DeviceBuffer<Tscal> &tree_field_rint
55 = storage.rtree_rint_field.get().get(patch_id).buf_field;
56
57 RTree &tree = storage.merged_pos_trees.get().get(patch_id);
58 auto obj_it = tree.get_object_iterator();
59
60 u32 obj_cnt = shambase::get_check_ref(storage.part_counts).indexes.get(patch_id);
61
62 sycl::range range_npart{obj_cnt};
63
64 Tscal h_tolerance = solver_config.htol_up_coarse_cycle;
65
66 NamedStackEntry stack_loc1{"init cache"};
67
68 using namespace shamrock;
69
70 sham::DeviceQueue &q = shamsys::instance::get_compute_scheduler().get_queue();
71
72 sham::DeviceBuffer<u32> neigh_count(
73 obj_cnt, shamsys::instance::get_compute_scheduler_ptr());
74
75 shamlog_debug_sycl_ln("Cache", "generate cache for N=", obj_cnt);
77 q,
78 sham::MultiRef{buf_xyz, buf_hpart, tree_field_rint, obj_it},
79 sham::MultiRef{neigh_count},
80 obj_cnt,
81 [h_tolerance](
82 u32 id_a,
83 const Tvec *__restrict xyz,
84 const Tscal *__restrict hpart,
85 const Tscal *__restrict rint_tree,
86 auto particle_looper,
87 u32 *__restrict neigh_cnt) {
88 constexpr Tscal Rker2 = Kernel::Rkern * Kernel::Rkern;
89
90 Tscal rint_a = hpart[id_a] * h_tolerance;
91
92 Tvec xyz_a = xyz[id_a];
93
94 Tvec inter_box_a_min = xyz_a - rint_a * Kernel::Rkern;
95 Tvec inter_box_a_max = xyz_a + rint_a * Kernel::Rkern;
96
97 u32 cnt = 0;
98
99 particle_looper.rtree_for(
100 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
101 Tscal int_r_max_cell = rint_tree[node_id] * Kernel::Rkern;
102
103 using namespace walker::interaction_crit;
104
105 return sph_radix_cell_crit(
106 xyz_a,
107 inter_box_a_min,
108 inter_box_a_max,
109 node_aabb.lower,
110 node_aabb.upper,
111 int_r_max_cell);
112 },
113 [&](u32 id_b) {
114 // compute only omega_a
115 Tvec dr = xyz_a - xyz[id_b];
116 Tscal rab2 = sycl::dot(dr, dr);
117 Tscal rint_b = hpart[id_b] * h_tolerance;
118
119 bool no_interact
120 = rab2 > rint_a * rint_a * Rker2 && rab2 > rint_b * rint_b * Rker2;
121
122 cnt += (no_interact) ? 0 : 1;
123 });
124
125 neigh_cnt[id_a] = cnt;
126 });
127
128 tree::ObjectCache pcache = tree::prepare_object_cache(std::move(neigh_count), obj_cnt);
129
130 NamedStackEntry stack_loc2{"fill cache"};
132 q,
133 sham::MultiRef{buf_xyz, buf_hpart, tree_field_rint, pcache.scanned_cnt, obj_it},
134 sham::MultiRef{pcache.index_neigh_map},
135 obj_cnt,
136 [h_tolerance](
137 u32 id_a,
138 const Tvec *__restrict xyz,
139 const Tscal *__restrict hpart,
140 const Tscal *__restrict rint_tree,
141 const u32 *__restrict scanned_neigh_cnt,
142 auto particle_looper,
143 u32 *__restrict neigh) {
144 constexpr Tscal Rker2 = Kernel::Rkern * Kernel::Rkern;
145
146 Tscal rint_a = hpart[id_a] * h_tolerance;
147
148 Tvec xyz_a = xyz[id_a];
149
150 Tvec inter_box_a_min = xyz_a - rint_a * Kernel::Rkern;
151 Tvec inter_box_a_max = xyz_a + rint_a * Kernel::Rkern;
152
153 u32 cnt = scanned_neigh_cnt[id_a];
154
155 particle_looper.rtree_for(
156 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
157 Tscal int_r_max_cell = rint_tree[node_id] * Kernel::Rkern;
158
159 using namespace walker::interaction_crit;
160
161 return sph_radix_cell_crit(
162 xyz_a,
163 inter_box_a_min,
164 inter_box_a_max,
165 node_aabb.lower,
166 node_aabb.upper,
167 int_r_max_cell);
168 },
169 [&](u32 id_b) {
170 // compute only omega_a
171 Tvec dr = xyz_a - xyz[id_b];
172 Tscal rab2 = sycl::dot(dr, dr);
173 Tscal rint_b = hpart[id_b] * h_tolerance;
174
175 bool no_interact
176 = rab2 > rint_a * rint_a * Rker2 && rab2 > rint_b * rint_b * Rker2;
177
178 if (!no_interact) {
179 neigh[cnt] = id_b;
180 }
181 cnt += (no_interact) ? 0 : 1;
182 });
183 });
184
185 return pcache;
186 };
187
188 shambase::get_check_ref(storage.neigh_cache).free_alloc();
189
190 using namespace shamrock::patch;
191 scheduler().for_each_patchdata_nonempty([&](Patch cur_p, PatchDataLayer &pdat) {
192 auto &ncache = shambase::get_check_ref(storage.neigh_cache);
193 ncache.neigh_cache.add_obj(cur_p.id_patch, build_neigh_cache(cur_p.id_patch));
194 });
195
196 time_neigh.stop();
197 storage.timings_details.neighbors += time_neigh.elapsed_sec();
198}
199
200template<class Tvec, class Tmorton, template<class> class SPHKernel>
203
204 // interface_control
205 using GhostHandle = sph::BasicSPHGhostHandler<Tvec>;
206 using GhostHandleCache = typename GhostHandle::CacheMap;
208
209 shambase::Timer time_neigh;
210 time_neigh.start();
211
212 StackEntry stack_loc{};
213
214 // do cache
215 auto build_neigh_cache = [&](u64 patch_id) {
216 shamlog_debug_ln("BasicSPH", "build particle cache id =", patch_id);
217
218 NamedStackEntry cache_build_stack_loc{"build cache"};
219
220 auto &mfield = storage.merged_xyzh.get().get(patch_id);
221
222 sham::DeviceBuffer<Tvec> &buf_xyz = mfield.template get_field_buf_ref<Tvec>(0);
223 sham::DeviceBuffer<Tscal> &buf_hpart = mfield.template get_field_buf_ref<Tscal>(1);
224
225 sham::DeviceBuffer<Tscal> &tree_field_rint
226 = storage.rtree_rint_field.get().get(patch_id).buf_field;
227
228 RTree &tree = storage.merged_pos_trees.get().get(patch_id);
229 auto obj_it = tree.get_object_iterator();
230
231 // a depth first traversal holds at most depth + 1 entries in its stack
232 u32 tree_depth = tree.get_exact_tree_depth();
233 u32 stack_size = tree_depth + 1;
234
235 // shamlog_info_ln("Cache", "patch", patch_id, "tree depth =", tree_depth);
236
237 u32 obj_cnt = shambase::get_check_ref(storage.part_counts).indexes.get(patch_id);
238
239 sycl::range range_npart{obj_cnt};
240
241 Tscal h_tolerance = solver_config.htol_up_coarse_cycle;
242
243 NamedStackEntry stack_loc1{"init cache"};
244
245 using namespace shamrock;
246
247 sham::DeviceQueue &q = shamsys::instance::get_compute_scheduler().get_queue();
248
249 sham::DeviceBuffer<u32> neigh_count(
250 obj_cnt, shamsys::instance::get_compute_scheduler_ptr());
251
252 shamlog_debug_sycl_ln("Cache", "generate cache for N=", obj_cnt);
253 sham::kernel_call_hndl(
254 q,
255 sham::MultiRef{buf_xyz, buf_hpart, tree_field_rint, obj_it},
256 sham::MultiRef{neigh_count},
257 obj_cnt,
258 [h_tolerance, stack_size](
259 u32 n,
260 const Tvec *__restrict xyz,
261 const Tscal *__restrict hpart,
262 const Tscal *__restrict rint_tree,
263 auto particle_looper,
264 u32 *__restrict neigh_cnt) {
265 return [=](sycl::handler &cgh) {
266 constexpr Tscal Rker2 = Kernel::Rkern * Kernel::Rkern;
267 constexpr u32 group_size = 256;
268
269 sycl::local_accessor<u32, 1> stack_local(stack_size * group_size, cgh);
270
271 cgh.parallel_for(sham::make_ndrange(group_size, n), [=](sycl::nd_item<1> item) {
272 u32 id_a = (u32) item.get_global_linear_id();
273
274 if (id_a >= n) {
275 return;
276 }
277
278 u32 group_id = (u32) item.get_local_id(0);
279 u32 *stack_ptr = &stack_local[group_id * stack_size];
280 auto stack_id = [&stack_ptr](u32 id) -> u32 &{
281 return stack_ptr[id];
282 };
283
284 Tscal rint_a = hpart[id_a] * h_tolerance;
285
286 Tvec xyz_a = xyz[id_a];
287
288 Tvec inter_box_a_min = xyz_a - rint_a * Kernel::Rkern;
289 Tvec inter_box_a_max = xyz_a + rint_a * Kernel::Rkern;
290
291 u32 cnt = 0;
292
293 particle_looper.rtree_for(
294 stack_id,
295 stack_size,
296 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
297 Tscal int_r_max_cell = rint_tree[node_id] * Kernel::Rkern;
298
299 using namespace walker::interaction_crit;
300
301 return sph_radix_cell_crit(
302 xyz_a,
303 inter_box_a_min,
304 inter_box_a_max,
305 node_aabb.lower,
306 node_aabb.upper,
307 int_r_max_cell);
308 },
309 [&](u32 id_b) {
310 // compute only omega_a
311 Tvec dr = xyz_a - xyz[id_b];
312 Tscal rab2 = sycl::dot(dr, dr);
313 Tscal rint_b = hpart[id_b] * h_tolerance;
314
315 bool no_interact = rab2 > rint_a * rint_a * Rker2
316 && rab2 > rint_b * rint_b * Rker2;
317
318 cnt += (no_interact) ? 0 : 1;
319 });
320
321 neigh_cnt[id_a] = cnt;
322 });
323 };
324 });
325
326 tree::ObjectCache pcache = tree::prepare_object_cache(std::move(neigh_count), obj_cnt);
327
328 NamedStackEntry stack_loc2{"fill cache"};
329 sham::kernel_call_hndl(
330 q,
331 sham::MultiRef{buf_xyz, buf_hpart, tree_field_rint, pcache.scanned_cnt, obj_it},
332 sham::MultiRef{pcache.index_neigh_map},
333 obj_cnt,
334 [h_tolerance, stack_size](
335 u32 n,
336 const Tvec *__restrict xyz,
337 const Tscal *__restrict hpart,
338 const Tscal *__restrict rint_tree,
339 const u32 *__restrict scanned_neigh_cnt,
340 auto particle_looper,
341 u32 *__restrict neigh) {
342 return [=](sycl::handler &cgh) {
343 constexpr Tscal Rker2 = Kernel::Rkern * Kernel::Rkern;
344 constexpr u32 group_size = 256;
345
346 sycl::local_accessor<u32, 1> stack_local(stack_size * group_size, cgh);
347
348 cgh.parallel_for(sham::make_ndrange(group_size, n), [=](sycl::nd_item<1> item) {
349 u32 id_a = (u32) item.get_global_linear_id();
350
351 if (id_a >= n) {
352 return;
353 }
354
355 u32 group_id = (u32) item.get_local_id(0);
356 u32 *stack_ptr = &stack_local[group_id * stack_size];
357 auto stack_id = [&stack_ptr](u32 id) -> u32 &{
358 return stack_ptr[id];
359 };
360
361 Tscal rint_a = hpart[id_a] * h_tolerance;
362
363 Tvec xyz_a = xyz[id_a];
364
365 Tvec inter_box_a_min = xyz_a - rint_a * Kernel::Rkern;
366 Tvec inter_box_a_max = xyz_a + rint_a * Kernel::Rkern;
367
368 u32 cnt = scanned_neigh_cnt[id_a];
369
370 particle_looper.rtree_for(
371 stack_id,
372 stack_size,
373 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
374 Tscal int_r_max_cell = rint_tree[node_id] * Kernel::Rkern;
375
376 using namespace walker::interaction_crit;
377
378 return sph_radix_cell_crit(
379 xyz_a,
380 inter_box_a_min,
381 inter_box_a_max,
382 node_aabb.lower,
383 node_aabb.upper,
384 int_r_max_cell);
385 },
386 [&](u32 id_b) {
387 // compute only omega_a
388 Tvec dr = xyz_a - xyz[id_b];
389 Tscal rab2 = sycl::dot(dr, dr);
390 Tscal rint_b = hpart[id_b] * h_tolerance;
391
392 bool no_interact = rab2 > rint_a * rint_a * Rker2
393 && rab2 > rint_b * rint_b * Rker2;
394
395 if (!no_interact) {
396 neigh[cnt] = id_b;
397 }
398 cnt += (no_interact) ? 0 : 1;
399 });
400 });
401 };
402 });
403
404 return pcache;
405 };
406
407 shambase::get_check_ref(storage.neigh_cache).free_alloc();
408
409 using namespace shamrock::patch;
410 scheduler().for_each_patchdata_nonempty([&](Patch cur_p, PatchDataLayer &pdat) {
411 auto &ncache = shambase::get_check_ref(storage.neigh_cache);
412 ncache.neigh_cache.add_obj(cur_p.id_patch, build_neigh_cache(cur_p.id_patch));
413 });
414
415 time_neigh.stop();
416 storage.timings_details.neighbors += time_neigh.elapsed_sec();
417}
418
419template<class Tvec, class Tmorton, template<class> class SPHKernel>
422
423 // interface_control
424 using GhostHandle = sph::BasicSPHGhostHandler<Tvec>;
425 using GhostHandleCache = typename GhostHandle::CacheMap;
427
428 shambase::Timer time_neigh;
429 time_neigh.start();
430
431 StackEntry stack_loc{};
432
433 // do cache
434 auto build_neigh_cache = [&](u64 patch_id) {
435 shamlog_debug_ln("BasicSPH", "build particle cache id =", patch_id);
436
437 NamedStackEntry cache_build_stack_loc{"build cache"};
438
439 auto &mfield = storage.merged_xyzh.get().get(patch_id);
440
441 sham::DeviceBuffer<Tvec> &buf_xyz = mfield.template get_field_buf_ref<Tvec>(0);
442 sham::DeviceBuffer<Tscal> &buf_hpart = mfield.template get_field_buf_ref<Tscal>(1);
443
444 sham::DeviceBuffer<Tscal> &tree_field_rint
445 = storage.rtree_rint_field.get().get(patch_id).buf_field;
446
447 RTree &tree = storage.merged_pos_trees.get().get(patch_id);
448 auto obj_it = tree.get_object_iterator();
449 auto leaf_it = tree.get_traverser();
450
451 u32 leaf_cnt = tree.get_leaf_cell_count();
452 u32 intnode_cnt = tree.get_internal_cell_count();
453
454 u32 obj_cnt = shambase::get_check_ref(storage.part_counts).indexes.get(patch_id);
455
456 sycl::range range_nleaf{leaf_cnt};
457 sycl::range range_nobj{obj_cnt};
458 using namespace shamrock;
459
460 sham::DeviceQueue &q = shamsys::instance::get_compute_scheduler().get_queue();
461
462 Tscal h_tolerance = solver_config.htol_up_coarse_cycle;
463
464 NamedStackEntry stack_loc1{"init cache"};
465
466 // start by counting number of leaf neighbours
467
468 sham::DeviceBuffer<u32> neigh_count_leaf(
469 leaf_cnt, shamsys::instance::get_compute_scheduler_ptr());
470
471 shamlog_debug_sycl_ln("Cache", "generate cache for Nleaf=", leaf_cnt);
472
474 q,
475 sham::MultiRef{tree_field_rint, leaf_it},
476 sham::MultiRef{neigh_count_leaf},
477 leaf_cnt,
478 [intnode_cnt](
479 u32 id_a,
480 const Tscal *__restrict rint_tree,
481 auto leaf_looper,
482 u32 *__restrict neigh_cnt) {
483 u32 offset_leaf = intnode_cnt;
484
485 Tscal leaf_a_rint = rint_tree[offset_leaf + id_a] * Kernel::Rkern;
486 Tvec leaf_a_bmin = leaf_looper.aabb_min[offset_leaf + id_a];
487 Tvec leaf_a_bmax = leaf_looper.aabb_max[offset_leaf + id_a];
488 Tvec leaf_a_bmin_ext = leaf_a_bmin - leaf_a_rint;
489 Tvec leaf_a_bmax_ext = leaf_a_bmax + leaf_a_rint;
490
491 u32 cnt = 0;
492
493 leaf_looper.rtree_for(
494 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
495 Tscal int_r_max_cell = rint_tree[node_id] * Kernel::Rkern;
496
497 Tvec ext_bmin = node_aabb.lower - int_r_max_cell;
498 Tvec ext_bmax = node_aabb.upper + int_r_max_cell;
499
500 return BBAA::cella_neigh_b(leaf_a_bmin, leaf_a_bmax, ext_bmin, ext_bmax)
501 || BBAA::cella_neigh_b(
502 leaf_a_bmin_ext,
503 leaf_a_bmax_ext,
504 node_aabb.lower,
505 node_aabb.upper);
506 },
507 [&](u32 leaf_b) {
508 cnt++;
509 });
510
511 neigh_cnt[id_a] = cnt;
512 });
513
514 //{
515 // u32 offset_leaf = intnode_cnt;
516 // sycl::host_accessor neigh_cnt{neigh_count_leaf};
517 // sycl::host_accessor pos_min_cell
518 // {shambase::get_check_ref(tree.tree_cell_ranges.buf_pos_min_cell_flt)};
519 // sycl::host_accessor pos_max_cell
520 // {shambase::get_check_ref(tree.tree_cell_ranges.buf_pos_max_cell_flt)};
521 //
522 // for (u32 i = 0; i < 1000; i++) {
523 // if(neigh_cnt[i] > 30){
524 // logger::raw_ln(i, neigh_cnt[i], pos_max_cell[i+offset_leaf] -
525 // pos_min_cell[i+offset_leaf]);
526 // }
527 // }
528 //}
529
530 tree::ObjectCache pleaf_cache
531 = tree::prepare_object_cache(std::move(neigh_count_leaf), leaf_cnt);
532
533 // fill ids of leaf neighbours
534
535 NamedStackEntry stack_loc2{"fill cache"};
536
538 q,
539 sham::MultiRef{tree_field_rint, pleaf_cache.scanned_cnt, leaf_it},
540 sham::MultiRef{pleaf_cache.index_neigh_map},
541 leaf_cnt,
542 [intnode_cnt](
543 u32 id_a,
544 const Tscal *__restrict rint_tree,
545 const u32 *__restrict scanned_neigh_cnt,
546 auto leaf_looper,
547 u32 *__restrict neigh) {
548 u32 offset_leaf = intnode_cnt;
549
550 Tscal leaf_a_rint = rint_tree[offset_leaf + id_a] * Kernel::Rkern;
551 Tvec leaf_a_bmin = leaf_looper.aabb_min[offset_leaf + id_a];
552 Tvec leaf_a_bmax = leaf_looper.aabb_max[offset_leaf + id_a];
553 Tvec leaf_a_bmin_ext = leaf_a_bmin - leaf_a_rint;
554 Tvec leaf_a_bmax_ext = leaf_a_bmax + leaf_a_rint;
555
556 u32 cnt = scanned_neigh_cnt[id_a];
557
558 leaf_looper.rtree_for(
559 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
560 Tscal int_r_max_cell = rint_tree[node_id] * Kernel::Rkern;
561
562 Tvec ext_bmin = node_aabb.lower - int_r_max_cell;
563 Tvec ext_bmax = node_aabb.upper + int_r_max_cell;
564
565 return BBAA::cella_neigh_b(leaf_a_bmin, leaf_a_bmax, ext_bmin, ext_bmax)
566 || BBAA::cella_neigh_b(
567 leaf_a_bmin_ext,
568 leaf_a_bmax_ext,
569 node_aabb.lower,
570 node_aabb.upper);
571 },
572 [&](u32 leaf_b) {
573 neigh[cnt] = leaf_b;
574 cnt++;
575 });
576 });
577
578 // search in which leaf each parts are
579 sham::DeviceBuffer<u32> leaf_part_id(
580 obj_cnt, shamsys::instance::get_compute_scheduler_ptr());
581
583 q,
584 sham::MultiRef{buf_xyz, leaf_it},
585 sham::MultiRef{leaf_part_id},
586 obj_cnt,
587 [intnode_cnt](
588 u32 id_a, const Tvec *__restrict xyz, auto leaf_looper, u32 *__restrict found_id) {
589 u32 offset_leaf = intnode_cnt;
590
591 Tvec r_a = xyz[id_a];
592
593 u32 found_id_ = i32_max; // to ensure a crash because of out of bound
594 // access if not found
595
596 leaf_looper.rtree_for(
597 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
598 return BBAA::is_coord_in_range_incl_max(
599 r_a, node_aabb.lower, node_aabb.upper);
600 },
601 [&](u32 leaf_b) {
602 found_id_ = leaf_b - offset_leaf;
603 });
604
605 SHAM_ASSERT(found_id_ < offset_leaf + 1);
606
607 found_id[id_a] = found_id_;
608 });
609
610 //{
611 // sycl::host_accessor xyz{buf_xyz};
612 // sycl::host_accessor acc {leaf_part_id};
613 //
614 // for(u32 i = 0; i < obj_cnt; i++){
615 // u32 leaf_id = acc[i];
616 // if(leaf_id >= leaf_cnt){
617 // logger::raw_ln("error : i=",i,"r=",xyz[i],"leaf_id=",leaf_id);
618 // }
619 // }
620 //}
621
622 sham::DeviceBuffer<u32> neigh_count(
623 obj_cnt, shamsys::instance::get_compute_scheduler_ptr());
624
625 shamlog_debug_sycl_ln("Cache", "generate cache for N=", obj_cnt);
626
628 q,
629 sham::MultiRef{buf_xyz, buf_hpart, pleaf_cache, obj_it.cell_iterator, leaf_part_id},
630 sham::MultiRef{neigh_count},
631 obj_cnt,
632 [intnode_cnt, h_tolerance](
633 u32 id_a,
634 const Tvec *__restrict xyz,
635 const Tscal *__restrict hpart,
636 auto acc_neigh_leaf_looper,
637 auto particle_looper,
638 const u32 *__restrict leaf_owner,
639 u32 *__restrict neigh_cnt) {
640 tree::ObjectCacheIterator neigh_leaf_looper(acc_neigh_leaf_looper);
641
642 u32 offset_leaf = intnode_cnt;
643
644 constexpr Tscal Rker2 = Kernel::Rkern * Kernel::Rkern;
645
646 Tscal rint_a = hpart[id_a] * h_tolerance;
647
648 Tvec xyz_a = xyz[id_a];
649
650 u32 cnt = 0;
651
652 u32 leaf_own_a = leaf_owner[id_a];
653
654 neigh_leaf_looper.for_each_object(leaf_own_a, [&](u32 leaf_b) {
655 SHAM_ASSERT(leaf_b >= offset_leaf);
656
657 particle_looper.for_each_in_leaf_cell(leaf_b - offset_leaf, [&](u32 id_b) {
658 Tvec dr = xyz_a - xyz[id_b];
659 Tscal rab2 = sycl::dot(dr, dr);
660 Tscal rint_b = hpart[id_b] * h_tolerance;
661
662 bool no_interact
663 = rab2 > rint_a * rint_a * Rker2 && rab2 > rint_b * rint_b * Rker2;
664
665 cnt += (no_interact) ? 0 : 1;
666 });
667 });
668
669 neigh_cnt[id_a] = cnt;
670 });
671
672 tree::ObjectCache pcache = tree::prepare_object_cache(std::move(neigh_count), obj_cnt);
673
674 NamedStackEntry stack_loc3{"fill cache"};
675
677 q,
679 buf_xyz,
680 buf_hpart,
681 pleaf_cache,
682 pcache.scanned_cnt,
683 obj_it.cell_iterator,
684 leaf_part_id},
685 sham::MultiRef{pcache.index_neigh_map},
686 obj_cnt,
687 [intnode_cnt, h_tolerance](
688 u32 id_a,
689 const Tvec *__restrict xyz,
690 const Tscal *__restrict hpart,
691 auto acc_neigh_leaf_looper,
692 const u32 *__restrict scanned_neigh_cnt,
693 auto particle_looper,
694 const u32 *__restrict leaf_owner,
695 u32 *__restrict neigh) {
696 tree::ObjectCacheIterator neigh_leaf_looper(acc_neigh_leaf_looper);
697
698 u32 offset_leaf = intnode_cnt;
699
700 constexpr Tscal Rker2 = Kernel::Rkern * Kernel::Rkern;
701
702 Tscal rint_a = hpart[id_a] * h_tolerance;
703
704 Tvec xyz_a = xyz[id_a];
705
706 u32 cnt = scanned_neigh_cnt[id_a];
707
708 u32 leaf_own_a = leaf_owner[id_a];
709
710 neigh_leaf_looper.for_each_object(leaf_own_a, [&](u32 leaf_b) {
711 SHAM_ASSERT(leaf_b >= offset_leaf);
712
713 particle_looper.for_each_in_leaf_cell(leaf_b - offset_leaf, [&](u32 id_b) {
714 Tvec dr = xyz_a - xyz[id_b];
715 Tscal rab2 = sycl::dot(dr, dr);
716 Tscal rint_b = hpart[id_b] * h_tolerance;
717
718 bool no_interact
719 = rab2 > rint_a * rint_a * Rker2 && rab2 > rint_b * rint_b * Rker2;
720
721 if (!no_interact) {
722 neigh[cnt] = id_b;
723 }
724 cnt += (no_interact) ? 0 : 1;
725 });
726 });
727 });
728 return pcache;
729 };
730
731 shambase::get_check_ref(storage.neigh_cache).free_alloc();
732
733 using namespace shamrock::patch;
734 scheduler().for_each_patchdata_nonempty([&](Patch cur_p, PatchDataLayer &pdat) {
735 auto &ncache = shambase::get_check_ref(storage.neigh_cache);
736 ncache.neigh_cache.add_obj(cur_p.id_patch, build_neigh_cache(cur_p.id_patch));
737 });
738
739 time_neigh.stop();
740 storage.timings_details.neighbors += time_neigh.elapsed_sec();
741}
742
743template<class Tvec, class Tmorton, template<class> class SPHKernel>
746
747 // interface_control
748 using GhostHandle = sph::BasicSPHGhostHandler<Tvec>;
749 using GhostHandleCache = typename GhostHandle::CacheMap;
751
752 shambase::Timer time_neigh;
753 time_neigh.start();
754
755 StackEntry stack_loc{};
756
757 // do cache
758 auto build_neigh_cache = [&](u64 patch_id) {
759 shamlog_debug_ln("BasicSPH", "build particle cache id =", patch_id);
760
761 NamedStackEntry cache_build_stack_loc{"build cache"};
762
763 auto &mfield = storage.merged_xyzh.get().get(patch_id);
764
765 sham::DeviceBuffer<Tvec> &buf_xyz = mfield.template get_field_buf_ref<Tvec>(0);
766 sham::DeviceBuffer<Tscal> &buf_hpart = mfield.template get_field_buf_ref<Tscal>(1);
767
768 sham::DeviceBuffer<Tscal> &tree_field_rint
769 = storage.rtree_rint_field.get().get(patch_id).buf_field;
770
771 RTree &tree = storage.merged_pos_trees.get().get(patch_id);
772 auto obj_it = tree.get_object_iterator();
773 auto leaf_it = tree.get_traverser();
774
775 // a depth first traversal holds at most depth + 1 entries in its stack
776 u32 tree_depth = tree.get_exact_tree_depth();
777 u32 stack_size = tree_depth + 1;
778
779 // shamlog_info_ln("Cache", "patch", patch_id, "tree depth =", tree_depth);
780
781 u32 leaf_cnt = tree.get_leaf_cell_count();
782 u32 intnode_cnt = tree.get_internal_cell_count();
783
784 u32 obj_cnt = shambase::get_check_ref(storage.part_counts).indexes.get(patch_id);
785
786 sycl::range range_nleaf{leaf_cnt};
787 sycl::range range_nobj{obj_cnt};
788 using namespace shamrock;
789
790 sham::DeviceQueue &q = shamsys::instance::get_compute_scheduler().get_queue();
791
792 Tscal h_tolerance = solver_config.htol_up_coarse_cycle;
793
794 NamedStackEntry stack_loc1{"init cache"};
795
796 // start by counting number of leaf neighbours
797
798 sham::DeviceBuffer<u32> neigh_count_leaf(
799 leaf_cnt, shamsys::instance::get_compute_scheduler_ptr());
800
801 shamlog_debug_sycl_ln("Cache", "generate cache for Nleaf=", leaf_cnt);
802
803 sham::kernel_call_hndl(
804 q,
805 sham::MultiRef{tree_field_rint, leaf_it},
806 sham::MultiRef{neigh_count_leaf},
807 leaf_cnt,
808 [intnode_cnt, stack_size](
809 u32 n,
810 const Tscal *__restrict rint_tree,
811 auto leaf_looper,
812 u32 *__restrict neigh_cnt) {
813 return [=](sycl::handler &cgh) {
814 u32 offset_leaf = intnode_cnt;
815 constexpr u32 group_size = 256;
816
817 sycl::local_accessor<u32, 1> stack_local(stack_size * group_size, cgh);
818
819 cgh.parallel_for(sham::make_ndrange(group_size, n), [=](sycl::nd_item<1> item) {
820 u32 id_a = (u32) item.get_global_linear_id();
821
822 if (id_a >= n) {
823 return;
824 }
825
826 u32 group_id = (u32) item.get_local_id(0);
827 u32 *stack_ptr = &stack_local[group_id * stack_size];
828 auto stack_id = [&stack_ptr](u32 id) -> u32 &{
829 return stack_ptr[id];
830 };
831
832 Tscal leaf_a_rint = rint_tree[offset_leaf + id_a] * Kernel::Rkern;
833 Tvec leaf_a_bmin = leaf_looper.aabb_min[offset_leaf + id_a];
834 Tvec leaf_a_bmax = leaf_looper.aabb_max[offset_leaf + id_a];
835 Tvec leaf_a_bmin_ext = leaf_a_bmin - leaf_a_rint;
836 Tvec leaf_a_bmax_ext = leaf_a_bmax + leaf_a_rint;
837
838 u32 cnt = 0;
839
840 leaf_looper.rtree_for(
841 stack_id,
842 stack_size,
843 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
844 Tscal int_r_max_cell = rint_tree[node_id] * Kernel::Rkern;
845
846 Tvec ext_bmin = node_aabb.lower - int_r_max_cell;
847 Tvec ext_bmax = node_aabb.upper + int_r_max_cell;
848
849 return BBAA::cella_neigh_b(
850 leaf_a_bmin, leaf_a_bmax, ext_bmin, ext_bmax)
851 || BBAA::cella_neigh_b(
852 leaf_a_bmin_ext,
853 leaf_a_bmax_ext,
854 node_aabb.lower,
855 node_aabb.upper);
856 },
857 [&](u32 leaf_b) {
858 cnt++;
859 });
860
861 neigh_cnt[id_a] = cnt;
862 });
863 };
864 });
865
866 tree::ObjectCache pleaf_cache
867 = tree::prepare_object_cache(std::move(neigh_count_leaf), leaf_cnt);
868
869 // fill ids of leaf neighbours
870
871 NamedStackEntry stack_loc2{"fill cache"};
872
873 sham::kernel_call_hndl(
874 q,
875 sham::MultiRef{tree_field_rint, pleaf_cache.scanned_cnt, leaf_it},
876 sham::MultiRef{pleaf_cache.index_neigh_map},
877 leaf_cnt,
878 [intnode_cnt, stack_size](
879 u32 n,
880 const Tscal *__restrict rint_tree,
881 const u32 *__restrict scanned_neigh_cnt,
882 auto leaf_looper,
883 u32 *__restrict neigh) {
884 return [=](sycl::handler &cgh) {
885 u32 offset_leaf = intnode_cnt;
886 constexpr u32 group_size = 256;
887
888 sycl::local_accessor<u32, 1> stack_local(stack_size * group_size, cgh);
889
890 cgh.parallel_for(sham::make_ndrange(group_size, n), [=](sycl::nd_item<1> item) {
891 u32 id_a = (u32) item.get_global_linear_id();
892
893 if (id_a >= n) {
894 return;
895 }
896
897 u32 group_id = (u32) item.get_local_id(0);
898 u32 *stack_ptr = &stack_local[group_id * stack_size];
899 auto stack_id = [&stack_ptr](u32 id) -> u32 &{
900 return stack_ptr[id];
901 };
902
903 Tscal leaf_a_rint = rint_tree[offset_leaf + id_a] * Kernel::Rkern;
904 Tvec leaf_a_bmin = leaf_looper.aabb_min[offset_leaf + id_a];
905 Tvec leaf_a_bmax = leaf_looper.aabb_max[offset_leaf + id_a];
906 Tvec leaf_a_bmin_ext = leaf_a_bmin - leaf_a_rint;
907 Tvec leaf_a_bmax_ext = leaf_a_bmax + leaf_a_rint;
908
909 u32 cnt = scanned_neigh_cnt[id_a];
910
911 leaf_looper.rtree_for(
912 stack_id,
913 stack_size,
914 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
915 Tscal int_r_max_cell = rint_tree[node_id] * Kernel::Rkern;
916
917 Tvec ext_bmin = node_aabb.lower - int_r_max_cell;
918 Tvec ext_bmax = node_aabb.upper + int_r_max_cell;
919
920 return BBAA::cella_neigh_b(
921 leaf_a_bmin, leaf_a_bmax, ext_bmin, ext_bmax)
922 || BBAA::cella_neigh_b(
923 leaf_a_bmin_ext,
924 leaf_a_bmax_ext,
925 node_aabb.lower,
926 node_aabb.upper);
927 },
928 [&](u32 leaf_b) {
929 neigh[cnt] = leaf_b;
930 cnt++;
931 });
932 });
933 };
934 });
935
936 // search in which leaf each parts are
937 sham::DeviceBuffer<u32> leaf_part_id(
938 obj_cnt, shamsys::instance::get_compute_scheduler_ptr());
939
940 sham::kernel_call_hndl(
941 q,
942 sham::MultiRef{buf_xyz, leaf_it},
943 sham::MultiRef{leaf_part_id},
944 obj_cnt,
945 [intnode_cnt, stack_size](
946 u32 n, const Tvec *__restrict xyz, auto leaf_looper, u32 *__restrict found_id) {
947 return [=](sycl::handler &cgh) {
948 u32 offset_leaf = intnode_cnt;
949 constexpr u32 group_size = 256;
950
951 sycl::local_accessor<u32, 1> stack_local(stack_size * group_size, cgh);
952
953 cgh.parallel_for(sham::make_ndrange(group_size, n), [=](sycl::nd_item<1> item) {
954 u32 id_a = (u32) item.get_global_linear_id();
955
956 if (id_a >= n) {
957 return;
958 }
959
960 u32 group_id = (u32) item.get_local_id(0);
961 u32 *stack_ptr = &stack_local[group_id * stack_size];
962 auto stack_id = [&stack_ptr](u32 id) -> u32 &{
963 return stack_ptr[id];
964 };
965
966 Tvec r_a = xyz[id_a];
967
968 u32 found_id_ = i32_max; // to ensure a crash because of out of
969 // bound access if not found
970
971 leaf_looper.rtree_for(
972 stack_id,
973 stack_size,
974 [&](u32 node_id, shammath::AABB<Tvec> node_aabb) -> bool {
975 return BBAA::is_coord_in_range_incl_max(
976 r_a, node_aabb.lower, node_aabb.upper);
977 },
978 [&](u32 leaf_b) {
979 found_id_ = leaf_b - offset_leaf;
980 });
981
982 SHAM_ASSERT(found_id_ < offset_leaf + 1);
983
984 found_id[id_a] = found_id_;
985 });
986 };
987 });
988
989 sham::DeviceBuffer<u32> neigh_count(
990 obj_cnt, shamsys::instance::get_compute_scheduler_ptr());
991
992 shamlog_debug_sycl_ln("Cache", "generate cache for N=", obj_cnt);
993
995 q,
996 sham::MultiRef{buf_xyz, buf_hpart, pleaf_cache, obj_it.cell_iterator, leaf_part_id},
997 sham::MultiRef{neigh_count},
998 obj_cnt,
999 [intnode_cnt, h_tolerance](
1000 u32 id_a,
1001 const Tvec *__restrict xyz,
1002 const Tscal *__restrict hpart,
1003 auto acc_neigh_leaf_looper,
1004 auto particle_looper,
1005 const u32 *__restrict leaf_owner,
1006 u32 *__restrict neigh_cnt) {
1007 tree::ObjectCacheIterator neigh_leaf_looper(acc_neigh_leaf_looper);
1008
1009 u32 offset_leaf = intnode_cnt;
1010
1011 constexpr Tscal Rker2 = Kernel::Rkern * Kernel::Rkern;
1012
1013 Tscal rint_a = hpart[id_a] * h_tolerance;
1014
1015 Tvec xyz_a = xyz[id_a];
1016
1017 u32 cnt = 0;
1018
1019 u32 leaf_own_a = leaf_owner[id_a];
1020
1021 neigh_leaf_looper.for_each_object(leaf_own_a, [&](u32 leaf_b) {
1022 SHAM_ASSERT(leaf_b >= offset_leaf);
1023
1024 particle_looper.for_each_in_leaf_cell(leaf_b - offset_leaf, [&](u32 id_b) {
1025 Tvec dr = xyz_a - xyz[id_b];
1026 Tscal rab2 = sycl::dot(dr, dr);
1027 Tscal rint_b = hpart[id_b] * h_tolerance;
1028
1029 bool no_interact
1030 = rab2 > rint_a * rint_a * Rker2 && rab2 > rint_b * rint_b * Rker2;
1031
1032 cnt += (no_interact) ? 0 : 1;
1033 });
1034 });
1035
1036 neigh_cnt[id_a] = cnt;
1037 });
1038
1039 tree::ObjectCache pcache = tree::prepare_object_cache(std::move(neigh_count), obj_cnt);
1040
1041 NamedStackEntry stack_loc3{"fill cache"};
1042
1044 q,
1046 buf_xyz,
1047 buf_hpart,
1048 pleaf_cache,
1049 pcache.scanned_cnt,
1050 obj_it.cell_iterator,
1051 leaf_part_id},
1052 sham::MultiRef{pcache.index_neigh_map},
1053 obj_cnt,
1054 [intnode_cnt, h_tolerance](
1055 u32 id_a,
1056 const Tvec *__restrict xyz,
1057 const Tscal *__restrict hpart,
1058 auto acc_neigh_leaf_looper,
1059 const u32 *__restrict scanned_neigh_cnt,
1060 auto particle_looper,
1061 const u32 *__restrict leaf_owner,
1062 u32 *__restrict neigh) {
1063 tree::ObjectCacheIterator neigh_leaf_looper(acc_neigh_leaf_looper);
1064
1065 u32 offset_leaf = intnode_cnt;
1066
1067 constexpr Tscal Rker2 = Kernel::Rkern * Kernel::Rkern;
1068
1069 Tscal rint_a = hpart[id_a] * h_tolerance;
1070
1071 Tvec xyz_a = xyz[id_a];
1072
1073 u32 cnt = scanned_neigh_cnt[id_a];
1074
1075 u32 leaf_own_a = leaf_owner[id_a];
1076
1077 neigh_leaf_looper.for_each_object(leaf_own_a, [&](u32 leaf_b) {
1078 SHAM_ASSERT(leaf_b >= offset_leaf);
1079
1080 particle_looper.for_each_in_leaf_cell(leaf_b - offset_leaf, [&](u32 id_b) {
1081 Tvec dr = xyz_a - xyz[id_b];
1082 Tscal rab2 = sycl::dot(dr, dr);
1083 Tscal rint_b = hpart[id_b] * h_tolerance;
1084
1085 bool no_interact
1086 = rab2 > rint_a * rint_a * Rker2 && rab2 > rint_b * rint_b * Rker2;
1087
1088 if (!no_interact) {
1089 neigh[cnt] = id_b;
1090 }
1091 cnt += (no_interact) ? 0 : 1;
1092 });
1093 });
1094 });
1095 return pcache;
1096 };
1097
1098 shambase::get_check_ref(storage.neigh_cache).free_alloc();
1099
1100 using namespace shamrock::patch;
1101 scheduler().for_each_patchdata_nonempty([&](Patch cur_p, PatchDataLayer &pdat) {
1102 auto &ncache = shambase::get_check_ref(storage.neigh_cache);
1103 ncache.neigh_cache.add_obj(cur_p.id_patch, build_neigh_cache(cur_p.id_patch));
1104 });
1105
1106 time_neigh.stop();
1107 storage.timings_details.neighbors += time_neigh.elapsed_sec();
1108}
1109
1110using namespace shammath;
1114
constexpr const char * xyz
Position field (3D coordinates).
constexpr const char * hpart
Smoothing length field.
std::uint32_t u32
32 bit unsigned integer
std::uint64_t u64
64 bit unsigned integer
Shamrock assertion utility.
#define SHAM_ASSERT(x)
Shorthand for SHAM_ASSERT_NAMED without a message.
Definition assert.hpp:67
A buffer allocated in USM (Unified Shared Memory).
A SYCL queue associated with a device and a context.
Class Timer measures the time elapsed since the timer was started.
Definition Timer.hpp:36
f64 elapsed_sec() const
Converts the stored nanosecond time to a floating point representation in seconds.
Definition Timer.hpp:88
void start()
Starts the timer.
Definition Timer.hpp:51
void stop()
Stops the timer and stores the elapsed time in nanoseconds.
Definition Timer.hpp:65
PatchDataLayer container class, the layout is described in patchdata_layout.
A Compressed Leaf Bounding Volume Hierarchy (CLBVH) for neighborhood queries.
sycl::nd_range< 1 > make_ndrange(u32 wg_size, u32 nthread)
void kernel_call(sham::DeviceQueue &q, RefIn in, RefOut in_out, u32 n, Functor &&func, SourceLocation &&callsite=SourceLocation{})
Submit a kernel to a SYCL queue.
T & get_check_ref(const std::unique_ptr< T > &ptr, SourceLocation loc=SourceLocation())
Takes a std::unique_ptr and returns a reference to the object it holds. It throws a std::runtime_erro...
Definition memory.hpp:112
namespace for math utility
Definition AABB.hpp:26
namespace for the main framework
Definition __init__.py:1
constexpr i32 i32_max
i32 max value
sph kernels
shambase::details::NamedBasicStackEntry NamedStackEntry
Alias for shambase::details::NamedBasicStackEntry.
shambase::details::BasicStackEntry StackEntry
Alias for shambase::details::BasicStackEntry.
A class that references multiple buffers or similar objects.
Definition MultiRef.hpp:33
Axis-Aligned bounding box.
Definition AABB.hpp:99
T lower
Lower bound of the AABB.
Definition AABB.hpp:104
T upper
Upper bound of the AABB.
Definition AABB.hpp:105
Patch object that contain generic patch information.
Definition Patch.hpp:33
u64 id_patch
unique key that identify the patch
Definition Patch.hpp:86