Shamrock 2025.10.0
Astrophysical Code
Loading...
Searching...
No Matches
BasicSPHGhosts.hpp
Go to the documentation of this file.
1// -------------------------------------------------------//
2//
3// SHAMROCK code for hydrodynamics
4// Copyright (c) 2021-2026 Timothée David--Cléris <tim.shamrock@proton.me>
5// SPDX-License-Identifier: CeCILL Free Software License Agreement v2.1
6// Shamrock is licensed under the CeCILL 2.1 License, see LICENSE for more information
7//
8// -------------------------------------------------------//
9
10#pragma once
11
18
21#include "shambackends/vec.hpp"
34#include <utility>
35
36namespace shammodels::sph {
37
38 template<class vec>
39 class BasicSPHGhostHandler {
40
41 PatchScheduler &sched;
42
43 public:
44 using flt = shambase::VecComponent<vec>;
45 static constexpr u32 dim = shambase::VectorProperties<vec>::dimension;
46 using per_index = sycl::vec<i32, dim>;
47
49 vec offset;
50 vec offset_speed;
51 per_index periodicity_index;
53 flt volume_ratio;
54 };
55
57 InterfaceBuildInfos build_infos;
58 sham::DeviceBuffer<u32> ids_interf;
59 f64 part_cnt_ratio;
60 };
61
62 std::shared_ptr<shamrock::patch::PatchDataLayerLayout> &xyzh_ghost_layout;
63
64 std::shared_ptr<shamrock::solvergraph::RankGetter> patch_rank_owner;
65
66 BasicSPHGhostHandler(
67 PatchScheduler &sched,
68 std::shared_ptr<shamrock::solvergraph::RankGetter> patch_rank_owner,
69 std::shared_ptr<shamrock::patch::PatchDataLayerLayout> &xyzh_ghost_layout)
70 : sched(sched), patch_rank_owner(std::move(patch_rank_owner)),
71 xyzh_ghost_layout(xyzh_ghost_layout) {}
72
73 void gen_debug_patch_ghost(shambase::DistributedDataShared<InterfaceIdTable> &interf_info);
74
76
93 template<class T>
96 std::function<T(u64, u64, InterfaceBuildInfos, sham::DeviceBuffer<u32> &, u32)> fct) {
97 StackEntry stack_loc{};
98
99 // clang-format off
100 return builder.template map<T>([&](u64 sender, u64 receiver, InterfaceIdTable &build_table) {
101 if (build_table.ids_interf.get_size() == 0) {
103 "their is an empty id table in the interface, it should have been removed");
104 }
105
106 return fct(
107 sender,
108 receiver,
109 build_table.build_infos,
110 build_table.ids_interf,
111 build_table.ids_interf.get_size());
112
113 });
114 // clang-format on
115 }
116
117 template<class T>
118 void modify_interface_native(
121 std::function<void(u64, u64, InterfaceBuildInfos, sham::DeviceBuffer<u32> &, u32, T &)>
122 fct) {
123 StackEntry stack_loc{};
124
125 struct Args {
126 u64 sender;
127 u64 receiver;
128 InterfaceIdTable &build_table;
129 };
130
131 std::vector<Args> vecarg;
132
133 // clang-format off
134 builder.for_each([&](u64 sender, u64 receiver, InterfaceIdTable &build_table) {
135 if (build_table.ids_interf.get_size() == 0) {
137 "their is an empty id table in the interface, it should have been removed");
138 }
139
140 vecarg.push_back({.sender = sender, .receiver = receiver, .build_table = build_table});
141 });
142 // clang-format on
143
144 u32 i = 0;
145 mod.for_each([&](u64 sender, u64 receiver, T &ref) {
146 InterfaceIdTable &build_table = vecarg[i].build_table;
147
148 fct(sender,
149 receiver,
150 build_table.build_infos,
151 build_table.ids_interf,
152 build_table.ids_interf.get_size(),
153 ref);
154
155 i++;
156 });
157 }
158
171 template<class T>
174 std::function<T(u64, u64, InterfaceBuildInfos, sycl::buffer<u32> &, u32)> gen_1,
175 std::function<void(u64, u64, InterfaceBuildInfos, sycl::buffer<u32> &, u32, T &)>
176 modif) {
177
178 StackEntry stack_loc{};
179
180 struct Args {
181 u64 sender;
182 u64 receiver;
183 InterfaceIdTable &build_table;
184 };
185
186 std::vector<Args> vecarg;
187
188 shambase::DistributedDataShared<T> ret = builder.template map<
189 T>([&](u64 sender, u64 receiver, InterfaceIdTable &build_table) {
190 if (!bool(build_table.ids_interf)) {
192 "their is an empty id table in the interface, it should have been removed");
193 }
194
195 vecarg.push_back(
196 {.sender = sender, .receiver = receiver, .build_table = build_table});
197
198 return gen_1(
199 sender,
200 receiver,
201 build_table.build_infos,
202 *build_table.ids_interf,
203 build_table.ids_interf->size());
204 });
205
206 u32 i = 0;
207 ret.for_each([&](u64 sender, u64 receiver, T &ref) {
208 InterfaceIdTable &build_table = vecarg[i].build_table;
209
210 modif(
211 sender,
212 receiver,
213 build_table.build_infos,
214 *build_table.ids_interf,
215 build_table.ids_interf->size(),
216 ref);
217
218 i++;
219 });
220
221 return ret;
222 }
223
225 // interface generation/communication utility //////////////////////////////////////////////
227
229 build_position_interf_field(shambase::DistributedDataShared<InterfaceIdTable> &builder) {
230 StackEntry stack_loc{};
231
232 const u32 ihpart = sched.pdl_old().template get_field_idx<flt>("hpart");
233
235 builder,
236 [&](u64 sender,
237 u64 /*receiver*/,
238 InterfaceBuildInfos binfo,
240 u32 cnt) {
241 using namespace shamrock::patch;
242
243 PatchDataLayer &sender_pdat = sched.patch_data.get_pdat(sender);
244
245 shamrock::patch::PatchDataLayer ret(xyzh_ghost_layout);
246
247 sender_pdat.get_field<vec>(0).append_subset_to(
248 buf_idx, cnt, ret.get_field<vec>(0));
249 sender_pdat.get_field<flt>(ihpart).append_subset_to(
250 buf_idx, cnt, ret.get_field<flt>(1));
251
252 ret.get_field<vec>(0).apply_offset(binfo.offset);
253
254 return ret;
255 });
256 }
257
258 inline shambase::DistributedDataShared<shamrock::patch::PatchDataLayer> communicate_pdat(
259 const std::shared_ptr<shamrock::patch::PatchDataLayerLayout> &pdl_ptr,
260 shambase::DistributedDataShared<shamrock::patch::PatchDataLayer> &&interf,
261 std::shared_ptr<shamrock::solvergraph::ExchangeGhostLayer> exchange_gz_node,
262 bool show_debug_infos) {
263 StackEntry stack_loc{};
264
265 // ----------------------------------------------------------------------------------------
266 // temporary wrapper to slowly migrate to the new solvergraph
267 std::shared_ptr<shamrock::solvergraph::PatchDataLayerDDShared> exchange_gz_edge
268 = std::make_shared<shamrock::solvergraph::PatchDataLayerDDShared>("", "");
269
270 exchange_gz_edge->patchdatas
271 = std::forward<shambase::DistributedDataShared<shamrock::patch::PatchDataLayer>>(
272 interf);
273
274 if (show_debug_infos) {
275 std::shared_ptr<shamrock::solvergraph::ScalarsEdge<u64>> object_counts
276 = std::make_shared<shamrock::solvergraph::ScalarsEdge<u64>>(
277 "object_counts", "object_counts");
278 sched.for_each_patchdata_nonempty(
279 [&](const shamrock::patch::Patch p, shamrock::patch::PatchDataLayer &pdat) {
280 object_counts->values.add_obj(p.id_patch, pdat.get_obj_cnt());
281 });
282 auto exchange_gz_node_debug
283 = std::make_shared<shamrock::solvergraph::ExchangeGhostLayerDebugDotGraph>();
284 exchange_gz_node_debug->set_edges(object_counts, exchange_gz_edge);
285 exchange_gz_node_debug->evaluate();
286 }
287
288 exchange_gz_node->set_edges(this->patch_rank_owner, exchange_gz_edge);
289
290 exchange_gz_node->evaluate();
291
292 // ----------------------------------------------------------------------------------------
293
294 shambase::DistributedDataShared<shamrock::patch::PatchDataLayer> recv_dat;
295
296#if false
297 shamalgs::collective::serialize_sparse_comm<shamrock::patch::PatchDataLayer>(
298 shamsys::instance::get_compute_scheduler_ptr(),
299 std::forward<shambase::DistributedDataShared<shamrock::patch::PatchDataLayer>>(
300 interf),
301 recv_dat,
302 [&](u64 id) {
303 return sched.get_patch_rank_owner(id);
304 },
305 [](shamrock::patch::PatchDataLayer &pdat) {
306 shamalgs::SerializeHelper ser(shamsys::instance::get_compute_scheduler_ptr());
307 ser.allocate(pdat.serialize_buf_byte_size());
308 pdat.serialize_buf(ser);
309 return ser.finalize();
310 },
311 [&](sham::DeviceBuffer<u8> &&buf) {
312 // exchange the buffer held by the distrib data and give it to the serializer
313 shamalgs::SerializeHelper ser(
314 shamsys::instance::get_compute_scheduler_ptr(),
315 std::forward<sham::DeviceBuffer<u8>>(buf));
316 return shamrock::patch::PatchDataLayer::deserialize_buf(ser, pdl_ptr);
317 });
318#else
319 recv_dat = std::move(exchange_gz_edge->patchdatas);
320#endif
321
322 return recv_dat;
323 }
324
325 template<class T>
326 inline shambase::DistributedDataShared<PatchDataField<T>> communicate_pdatfield(
327 shambase::DistributedDataShared<PatchDataField<T>> &&interf,
328 u32 nvar,
329 std::shared_ptr<shamrock::solvergraph::ExchangeGhostField<T>> exchange_gz_node) {
330 StackEntry stack_loc{};
331
332 // ----------------------------------------------------------------------------------------
333 // temporary wrapper to slowly migrate to the new solvergraph
334 std::shared_ptr<shamrock::solvergraph::PatchDataFieldDDShared<T>> exchange_gz_edge
335 = std::make_shared<shamrock::solvergraph::PatchDataFieldDDShared<T>>("", "");
336
337 exchange_gz_edge->patchdata_fields
338 = std::forward<shambase::DistributedDataShared<PatchDataField<T>>>(interf);
339
340 exchange_gz_node->set_edges(this->patch_rank_owner, exchange_gz_edge);
341
342 exchange_gz_node->evaluate();
343
344 // ----------------------------------------------------------------------------------------
345
346 shambase::DistributedDataShared<PatchDataField<T>> recv_dat;
347
348#if false
349 shamalgs::collective::serialize_sparse_comm<PatchDataField<T>>(
350 shamsys::instance::get_compute_scheduler_ptr(),
351 std::forward<shambase::DistributedDataShared<PatchDataField<T>>>(interf),
352 recv_dat,
353 [&](u64 id) {
354 return sched.get_patch_rank_owner(id);
355 },
356 [](PatchDataField<T> &pdat) {
357 shamalgs::SerializeHelper ser(shamsys::instance::get_compute_scheduler_ptr());
358 ser.allocate(pdat.serialize_full_byte_size());
359 pdat.serialize_full(ser);
360 return ser.finalize();
361 },
362 [&](sham::DeviceBuffer<u8> &&buf) {
363 // exchange the buffer held by the distrib data and give it to the serializer
364 shamalgs::SerializeHelper ser(
365 shamsys::instance::get_compute_scheduler_ptr(),
366 std::forward<sham::DeviceBuffer<u8>>(buf));
368 });
369#else
370 recv_dat = std::move(exchange_gz_edge->patchdata_fields);
371#endif
372
373 return recv_dat;
374 }
375
376 inline shambase::DistributedDataShared<shamrock::patch::PatchDataLayer>
377 build_communicate_positions(
378 shambase::DistributedDataShared<InterfaceIdTable> &builder,
379 std::shared_ptr<shamrock::solvergraph::ExchangeGhostLayer> &exchange_gz_positions,
380 bool show_debug_infos) {
381 auto pos_interf = build_position_interf_field(builder);
382 return communicate_pdat(
383 xyzh_ghost_layout, std::move(pos_interf), exchange_gz_positions, show_debug_infos);
384 }
385
386 template<class T, class Tmerged>
387 inline shambase::DistributedData<Tmerged> merge_native(
388 shambase::DistributedDataShared<T> &&interfs,
389 std::function<
390 Tmerged(const shamrock::patch::Patch, shamrock::patch::PatchDataLayer &pdat)> init,
391 std::function<void(Tmerged &, T &)> appender) {
392
393 StackEntry stack_loc{};
394
395 shambase::DistributedData<Tmerged> merge_f;
396
397 sched.for_each_patchdata_nonempty(
398 [&](const shamrock::patch::Patch p, shamrock::patch::PatchDataLayer &pdat) {
399 Tmerged tmp_merge = init(p, pdat);
400
401 interfs.for_each([&](u64 sender, u64 receiver, T &interface) {
402 if (receiver == p.id_patch) {
403 appender(tmp_merge, interface);
404 }
405 });
406
407 merge_f.add_obj(p.id_patch, std::move(tmp_merge));
408 });
409
410 return merge_f;
411 }
412
413 inline shambase::DistributedData<shamrock::patch::PatchDataLayer> merge_position_buf(
414 shambase::DistributedDataShared<shamrock::patch::PatchDataLayer> &&positioninterfs) {
415 StackEntry stack_loc{};
416
417 const u32 ihpart = sched.pdl_old().template get_field_idx<flt>("hpart");
418
419 return merge_native<shamrock::patch::PatchDataLayer, shamrock::patch::PatchDataLayer>(
420 std::forward<shambase::DistributedDataShared<shamrock::patch::PatchDataLayer>>(
421 positioninterfs),
422 [=, this](const shamrock::patch::Patch p, shamrock::patch::PatchDataLayer &pdat) {
423 PatchDataField<vec> &pos = pdat.get_field<vec>(0);
424 PatchDataField<flt> &hpart = pdat.get_field<flt>(ihpart);
425
426 shamrock::patch::PatchDataLayer ret(xyzh_ghost_layout);
427
428 ret.get_field<vec>(0).insert(pos);
429 ret.get_field<flt>(1).insert(hpart);
430 ret.check_field_obj_cnt_match();
431
432 return ret;
433 },
434 [](shamrock::patch::PatchDataLayer &merged, shamrock::patch::PatchDataLayer &pint) {
435 merged.insert_elements(pint);
436
438 });
439 }
440
441 inline shambase::DistributedData<shamrock::patch::PatchDataLayer>
442 build_comm_merge_positions(
443 shambase::DistributedDataShared<InterfaceIdTable> &builder,
444 std::shared_ptr<shamrock::solvergraph::ExchangeGhostLayer> &exchange_gz_positions,
445 bool show_debug_infos) {
446 auto pos_interf = build_position_interf_field(builder);
447 return merge_position_buf(communicate_pdat(
448 xyzh_ghost_layout, std::move(pos_interf), exchange_gz_positions, show_debug_infos));
449 }
450 };
451
452} // namespace shammodels::sph
Solver graph node for exchanging ghost field data between distributed processes.
Solver graph node for exchanging ghost layer data between distributed processes.
Solver graph node for exchanging ghost layer data between distributed processes.
constexpr const char * hpart
Smoothing length field.
Header file describing a Node Instance.
Shared distributed data layer for patch data management in solver graphs.
MPI scheduler.
double f64
Alias for double.
std::uint32_t u32
32 bit unsigned integer
std::uint64_t u64
64 bit unsigned integer
static PatchDataField deserialize_full(shamalgs::SerializeHelper &serializer)
deserialize a field inverse of serialize_full
The MPI scheduler.
SchedulerPatchData patch_data
handle the data of the patches of the scheduler
A buffer allocated in USM (Unified Shared Memory).
size_t get_size() const
Gets the number of elements in the buffer.
Container for objects shared between two distributed data elements.
void for_each(std::function< void(u64, u64, T &)> &&f)
Apply a function to all stored objects.
iterator add_obj(u64 id, T &&obj)
Adds a new object to the collection.
shambase::DistributedDataShared< T > build_interface_native_stagged(shambase::DistributedDataShared< InterfaceIdTable > &builder, std::function< T(u64, u64, InterfaceBuildInfos, sycl::buffer< u32 > &, u32)> gen_1, std::function< void(u64, u64, InterfaceBuildInfos, sycl::buffer< u32 > &, u32, T &)> modif)
native handle to generate interfaces generate interfaces of type T (template arg) based on the provid...
shambase::DistributedDataShared< T > build_interface_native(shambase::DistributedDataShared< InterfaceIdTable > &builder, std::function< T(u64, u64, InterfaceBuildInfos, sham::DeviceBuffer< u32 > &, u32)> fct)
native handle to generate interfaces generate interfaces of type T (template arg) based on the provid...
PatchDataLayer container class, the layout is described in patchdata_layout.
void check_field_obj_cnt_match()
check that all contained field have the same obj cnt
u32 get_obj_cnt() const
get the number of objects (particles) stored in this layer
ExcptTypes make_except_with_loc(std::string message, SourceLocation loc=SourceLocation{})
Create an exception with a message and a location.
namespace for the sph model
This file contains the definition for the stacktrace related functionality.
shambase::details::BasicStackEntry StackEntry
Alias for shambase::details::BasicStackEntry.
u64 id_patch
unique key that identify the patch
Definition Patch.hpp:86