39 class BasicSPHGhostHandler {
44 using flt = shambase::VecComponent<vec>;
45 static constexpr u32 dim = shambase::VectorProperties<vec>::dimension;
46 using per_index = sycl::vec<i32, dim>;
51 per_index periodicity_index;
62 std::shared_ptr<shamrock::patch::PatchDataLayerLayout> &xyzh_ghost_layout;
64 std::shared_ptr<shamrock::solvergraph::RankGetter> patch_rank_owner;
68 std::shared_ptr<shamrock::solvergraph::RankGetter> patch_rank_owner,
69 std::shared_ptr<shamrock::patch::PatchDataLayerLayout> &xyzh_ghost_layout)
70 : sched(sched), patch_rank_owner(std::move(patch_rank_owner)),
71 xyzh_ghost_layout(xyzh_ghost_layout) {}
100 return builder.template map<T>([&](
u64 sender,
u64 receiver, InterfaceIdTable &build_table) {
101 if (build_table.ids_interf.
get_size() == 0) {
103 "their is an empty id table in the interface, it should have been removed");
109 build_table.build_infos,
110 build_table.ids_interf,
118 void modify_interface_native(
128 InterfaceIdTable &build_table;
131 std::vector<Args> vecarg;
134 builder.
for_each([&](
u64 sender,
u64 receiver, InterfaceIdTable &build_table) {
135 if (build_table.ids_interf.get_size() == 0) {
137 "their is an empty id table in the interface, it should have been removed");
140 vecarg.push_back({.sender = sender, .receiver = receiver, .build_table = build_table});
150 build_table.build_infos,
151 build_table.ids_interf,
152 build_table.ids_interf.get_size(),
174 std::function<T(
u64,
u64, InterfaceBuildInfos, sycl::buffer<u32> &,
u32)> gen_1,
175 std::function<
void(
u64,
u64, InterfaceBuildInfos, sycl::buffer<u32> &,
u32, T &)>
183 InterfaceIdTable &build_table;
186 std::vector<Args> vecarg;
189 T>([&](
u64 sender,
u64 receiver, InterfaceIdTable &build_table) {
190 if (!
bool(build_table.ids_interf)) {
192 "their is an empty id table in the interface, it should have been removed");
196 {.sender = sender, .receiver = receiver, .build_table = build_table});
201 build_table.build_infos,
202 *build_table.ids_interf,
203 build_table.ids_interf->size());
208 InterfaceIdTable &build_table = vecarg[i].build_table;
213 build_table.build_infos,
214 *build_table.ids_interf,
215 build_table.ids_interf->size(),
232 const u32 ihpart = sched.pdl_old().template get_field_idx<flt>(
"hpart");
238 InterfaceBuildInfos binfo,
241 using namespace shamrock::patch;
247 sender_pdat.get_field<vec>(0).append_subset_to(
248 buf_idx, cnt, ret.get_field<vec>(0));
249 sender_pdat.get_field<flt>(ihpart).append_subset_to(
250 buf_idx, cnt, ret.get_field<flt>(1));
252 ret.get_field<vec>(0).apply_offset(binfo.offset);
258 inline shambase::DistributedDataShared<shamrock::patch::PatchDataLayer> communicate_pdat(
259 const std::shared_ptr<shamrock::patch::PatchDataLayerLayout> &pdl_ptr,
260 shambase::DistributedDataShared<shamrock::patch::PatchDataLayer> &&interf,
261 std::shared_ptr<shamrock::solvergraph::ExchangeGhostLayer> exchange_gz_node,
262 bool show_debug_infos) {
267 std::shared_ptr<shamrock::solvergraph::PatchDataLayerDDShared> exchange_gz_edge
268 = std::make_shared<shamrock::solvergraph::PatchDataLayerDDShared>(
"",
"");
270 exchange_gz_edge->patchdatas
271 = std::forward<shambase::DistributedDataShared<shamrock::patch::PatchDataLayer>>(
274 if (show_debug_infos) {
275 std::shared_ptr<shamrock::solvergraph::ScalarsEdge<u64>> object_counts
276 = std::make_shared<shamrock::solvergraph::ScalarsEdge<u64>>(
277 "object_counts",
"object_counts");
278 sched.for_each_patchdata_nonempty(
279 [&](
const shamrock::patch::Patch p, shamrock::patch::PatchDataLayer &pdat) {
282 auto exchange_gz_node_debug
283 = std::make_shared<shamrock::solvergraph::ExchangeGhostLayerDebugDotGraph>();
284 exchange_gz_node_debug->set_edges(object_counts, exchange_gz_edge);
285 exchange_gz_node_debug->evaluate();
288 exchange_gz_node->set_edges(this->patch_rank_owner, exchange_gz_edge);
290 exchange_gz_node->evaluate();
294 shambase::DistributedDataShared<shamrock::patch::PatchDataLayer> recv_dat;
297 shamalgs::collective::serialize_sparse_comm<shamrock::patch::PatchDataLayer>(
298 shamsys::instance::get_compute_scheduler_ptr(),
299 std::forward<shambase::DistributedDataShared<shamrock::patch::PatchDataLayer>>(
303 return sched.get_patch_rank_owner(
id);
305 [](shamrock::patch::PatchDataLayer &pdat) {
306 shamalgs::SerializeHelper ser(shamsys::instance::get_compute_scheduler_ptr());
307 ser.allocate(pdat.serialize_buf_byte_size());
308 pdat.serialize_buf(ser);
309 return ser.finalize();
311 [&](sham::DeviceBuffer<u8> &&buf) {
313 shamalgs::SerializeHelper ser(
314 shamsys::instance::get_compute_scheduler_ptr(),
315 std::forward<sham::DeviceBuffer<u8>>(buf));
316 return shamrock::patch::PatchDataLayer::deserialize_buf(ser, pdl_ptr);
319 recv_dat = std::move(exchange_gz_edge->patchdatas);
326 inline shambase::DistributedDataShared<PatchDataField<T>> communicate_pdatfield(
327 shambase::DistributedDataShared<PatchDataField<T>> &&interf,
329 std::shared_ptr<shamrock::solvergraph::ExchangeGhostField<T>> exchange_gz_node) {
334 std::shared_ptr<shamrock::solvergraph::PatchDataFieldDDShared<T>> exchange_gz_edge
335 = std::make_shared<shamrock::solvergraph::PatchDataFieldDDShared<T>>(
"",
"");
337 exchange_gz_edge->patchdata_fields
338 = std::forward<shambase::DistributedDataShared<PatchDataField<T>>>(interf);
340 exchange_gz_node->set_edges(this->patch_rank_owner, exchange_gz_edge);
342 exchange_gz_node->evaluate();
346 shambase::DistributedDataShared<PatchDataField<T>> recv_dat;
349 shamalgs::collective::serialize_sparse_comm<PatchDataField<T>>(
350 shamsys::instance::get_compute_scheduler_ptr(),
351 std::forward<shambase::DistributedDataShared<PatchDataField<T>>>(interf),
354 return sched.get_patch_rank_owner(
id);
356 [](PatchDataField<T> &pdat) {
357 shamalgs::SerializeHelper ser(shamsys::instance::get_compute_scheduler_ptr());
358 ser.allocate(pdat.serialize_full_byte_size());
359 pdat.serialize_full(ser);
360 return ser.finalize();
362 [&](sham::DeviceBuffer<u8> &&buf) {
364 shamalgs::SerializeHelper ser(
365 shamsys::instance::get_compute_scheduler_ptr(),
366 std::forward<sham::DeviceBuffer<u8>>(buf));
370 recv_dat = std::move(exchange_gz_edge->patchdata_fields);
376 inline shambase::DistributedDataShared<shamrock::patch::PatchDataLayer>
377 build_communicate_positions(
378 shambase::DistributedDataShared<InterfaceIdTable> &builder,
379 std::shared_ptr<shamrock::solvergraph::ExchangeGhostLayer> &exchange_gz_positions,
380 bool show_debug_infos) {
381 auto pos_interf = build_position_interf_field(builder);
382 return communicate_pdat(
383 xyzh_ghost_layout, std::move(pos_interf), exchange_gz_positions, show_debug_infos);
386 template<
class T,
class Tmerged>
387 inline shambase::DistributedData<Tmerged> merge_native(
388 shambase::DistributedDataShared<T> &&interfs,
390 Tmerged(
const shamrock::patch::Patch, shamrock::patch::PatchDataLayer &pdat)> init,
391 std::function<
void(Tmerged &, T &)> appender) {
395 shambase::DistributedData<Tmerged> merge_f;
397 sched.for_each_patchdata_nonempty(
398 [&](
const shamrock::patch::Patch p, shamrock::patch::PatchDataLayer &pdat) {
399 Tmerged tmp_merge = init(p, pdat);
401 interfs.for_each([&](
u64 sender,
u64 receiver, T &interface) {
403 appender(tmp_merge, interface);
413 inline shambase::DistributedData<shamrock::patch::PatchDataLayer> merge_position_buf(
414 shambase::DistributedDataShared<shamrock::patch::PatchDataLayer> &&positioninterfs) {
417 const u32 ihpart = sched.pdl_old().template get_field_idx<flt>(
"hpart");
419 return merge_native<shamrock::patch::PatchDataLayer, shamrock::patch::PatchDataLayer>(
420 std::forward<shambase::DistributedDataShared<shamrock::patch::PatchDataLayer>>(
422 [=,
this](
const shamrock::patch::Patch p, shamrock::patch::PatchDataLayer &pdat) {
423 PatchDataField<vec> &pos = pdat.get_field<vec>(0);
424 PatchDataField<flt> &
hpart = pdat.get_field<flt>(ihpart);
426 shamrock::patch::PatchDataLayer ret(xyzh_ghost_layout);
428 ret.get_field<vec>(0).insert(pos);
429 ret.get_field<flt>(1).insert(hpart);
430 ret.check_field_obj_cnt_match();
434 [](shamrock::patch::PatchDataLayer &merged, shamrock::patch::PatchDataLayer &pint) {
435 merged.insert_elements(pint);
441 inline shambase::DistributedData<shamrock::patch::PatchDataLayer>
442 build_comm_merge_positions(
443 shambase::DistributedDataShared<InterfaceIdTable> &builder,
444 std::shared_ptr<shamrock::solvergraph::ExchangeGhostLayer> &exchange_gz_positions,
445 bool show_debug_infos) {
446 auto pos_interf = build_position_interf_field(builder);
447 return merge_position_buf(communicate_pdat(
448 xyzh_ghost_layout, std::move(pos_interf), exchange_gz_positions, show_debug_infos));