Note
Go to the end to download the full example code.
Weak scale test for SPH#
This example tests the weak scalability performance of the SPH solver. It is based on a Sedov blast wave simulation, the number of particles is scaled with the number of processes. Run it only if you have enough memory available.
10 from statistics import mean, stdev
11
12 import shamrock
13 from shamrock import NeighCacheStrategy
14
15 result_text = ""
16
17 for N_target_base in [32e6]:
18 shamrock.backends.reset_mem_info_max()
19
20 gamma = 5.0 / 3.0
21 rho_g = 1
22 target_tot_u = 1
23
24 bmin = (-0.6, -0.6, -0.6)
25 bmax = (0.6, 0.6, 0.6)
26
27 compute_multiplier = shamrock.sys.world_size()
28 # compute_multiplier = 12
29 scheduler_split_val = int(2e7)
30 scheduler_merge_val = 1
31
32 N_target = N_target_base * compute_multiplier
33 xm, ym, zm = bmin
34 xM, yM, zM = bmax
35 vol_b = (xM - xm) * (yM - ym) * (zM - zm)
36
37 if shamrock.sys.world_rank() == 0:
38 print("N_target_base", N_target_base)
39 print("compute_multiplier", compute_multiplier)
40 print("scheduler_split_val", scheduler_split_val)
41 print("scheduler_merge_val", scheduler_merge_val)
42 print("N_target", N_target)
43 print("vol_b", vol_b)
44
45 part_vol = vol_b / N_target
46
47 # lattice volume
48 part_vol_lattice = 0.74 * part_vol
49
50 dr = (part_vol_lattice / ((4.0 / 3.0) * 3.1416)) ** (1.0 / 3.0)
51
52 pmass = -1
53
54 ctx = shamrock.Context()
55 ctx.pdata_layout_new()
56
57 model = shamrock.get_Model_SPH(context=ctx, vector_type="f64_3", sph_kernel="M4")
58
59 cfg = model.gen_default_config()
60 # cfg.set_artif_viscosity_Constant(alpha_u = 1, alpha_AV = 1, beta_AV = 2)
61 # cfg.set_artif_viscosity_VaryingMM97(alpha_min = 0.1,alpha_max = 1,sigma_decay = 0.1, alpha_u = 1, beta_AV = 2)
62 cfg.set_artif_viscosity_VaryingCD10(
63 alpha_min=0.0, alpha_max=1, sigma_decay=0.1, alpha_u=1, beta_AV=2
64 )
65 cfg.set_neigh_cache_strategy(NeighCacheStrategy.TwoStageSharedOffload)
66 cfg.set_boundary_periodic()
67 cfg.set_eos_adiabatic(gamma)
68 cfg.set_filter_empty_patch_gz(False)
69 cfg.print_status()
70 model.set_solver_config(cfg)
71 model.init_scheduler(scheduler_split_val, scheduler_merge_val)
72
73 bmin, bmax = shamrock.math.get_ideal_hcp_box(dr, bmin, bmax)
74 xm, ym, zm = bmin
75 xM, yM, zM = bmax
76
77 model.resize_simulation_box(bmin, bmax)
78
79 setup = model.get_setup()
80 gen = setup.make_generator_lattice_hcp(
81 dr, bmin, bmax, discontinuous=False, init_h_factor=2.1402
82 )
83
84 # Kind of optimized for Aurora
85 setup.apply_setup(
86 gen,
87 gen_step=int(scheduler_split_val / 8),
88 insert_step=int(scheduler_split_val * 2),
89 msg_count_limit=1024,
90 rank_comm_size_limit=int(scheduler_split_val) * 2,
91 max_msg_size=int(scheduler_split_val / 8),
92 do_setup_log=False,
93 speculative_balancing=True,
94 )
95
96 xc, yc, zc = model.get_closest_part_to((0, 0, 0))
97
98 if shamrock.sys.world_rank() == 0:
99 print("closest part to (0,0,0) is in :", xc, yc, zc)
100
101 vol_b = (xM - xm) * (yM - ym) * (zM - zm)
102
103 totmass = rho_g * vol_b
104 # print("Total mass :", totmass)
105
106 pmass = model.total_mass_to_part_mass(totmass)
107
108 model.set_value_in_a_box("uint", "f64", 0, bmin, bmax)
109
110 rinj = 16 * dr
111 u_inj = 1
112 model.add_kernel_value("uint", "f64", u_inj, (0, 0, 0), rinj)
113
114 tot_u = pmass * model.get_sum("uint", "f64")
115 if shamrock.sys.world_rank() == 0:
116 print("total u :", tot_u)
117
118 # print("Current part mass :", pmass)
119 model.set_particle_mass(pmass)
120
121 model.set_cfl_cour(0.1)
122 model.set_cfl_force(0.1)
123
124 shamrock.backends.reset_mem_info_max()
125
126 # converge smoothing length and compute initial dt
127 model.timestep()
128
129 # Now run the actual benchmark for 5 steps
130 res_rates = []
131 res_cnts = []
132 res_system_metrics = []
133 res_mpi_timers = []
134
135 """
136 Here we insert callbacks to measure solver MPI usage by fetching the timers twice at the begining and end of the step
137 """
138 before_mpi_timers, after_mpi_timers = None, None
139
140 def callback_before_mpi_timer():
141 global before_mpi_timers
142 # print(shamrock.sys.world_rank(), "register before_mpi_timers")
143 before_mpi_timers = shamrock.comm.get_timers()
144
145 def callback_after_mpi_timer():
146 global after_mpi_timers
147 # print(shamrock.sys.world_rank(), "register after_mpi_timers")
148 after_mpi_timers = shamrock.comm.get_timers()
149
150 model.add_timestep_callback(
151 step_begin=callback_before_mpi_timer, step_end=callback_after_mpi_timer
152 )
153
154 for i in range(10):
155 if shamrock.sys.world_rank() == 0:
156 print("running step ", i + 1, "/", 10, " ...")
157
158 shamrock.sys.mpi_barrier()
159
160 # To replay the same step
161 model.set_next_dt(0.0)
162 model.timestep()
163
164 if shamrock.sys.world_rank() == 0:
165 print("collecting results ...")
166
167 tmp_res_rate, tmp_res_cnt, tmp_system_metrics = (
168 model.solver_logs_last_rate(),
169 model.solver_logs_last_obj_count(),
170 model.solver_logs_last_system_metrics(),
171 )
172 res_rates.append(tmp_res_rate)
173 res_cnts.append(tmp_res_cnt)
174 res_system_metrics.append(tmp_system_metrics)
175 res_mpi_timers.append(shamrock.comm.mpi_timers_delta(before_mpi_timers, after_mpi_timers))
176
177 if shamrock.sys.world_rank() == 0:
178 print("sleeping 1 second ...")
179
180 import time
181
182 time.sleep(1)
183
184 if shamrock.sys.world_rank() == 0:
185 print("done sleeping 1 second ...")
186
187 # result is the best rate of the 5 steps
188 res_rate, res_cnt = max(res_rates), res_cnts[0]
189
190 # index of the max rate
191 max_rate_index = res_rates.index(max(res_rates))
192 max_rate_system_metrics = res_system_metrics[max_rate_index]
193 max_mpi_timers = res_mpi_timers[max_rate_index]
194 step_time = res_cnt / res_rate
195
196 if shamrock.sys.world_rank() == 0:
197 result_text += f"--- final score for N_target_base={N_target_base} ---"
198 result_text += f"world size : {shamrock.sys.world_size()}\n"
199 result_text += f"result rate : {res_rate}\n"
200 result_text += f"result cnt : {res_cnt}\n"
201 result_text += f"cnt/rank : {res_cnt / shamrock.sys.world_size()}\n"
202 result_text += f"result rate per rank : {res_rate / shamrock.sys.world_size()}\n"
203 result_text += f"rates infos : max={max(res_rates)}, min={min(res_rates)}, mean={mean(res_rates)}, stddev={stdev(res_rates)}\n"
204 result_text += f"res_rates = {res_rates}\n"
205 result_text += f"res_cnts = {res_cnts}\n"
206 result_text += f"step time = {step_time}\n"
207
208 dic_out = {
209 "world_size": shamrock.sys.world_size(),
210 "rate": res_rate,
211 "cnt": res_cnt,
212 "step_time": step_time,
213 "mpi_timers": max_mpi_timers,
214 }
215
216 # print the system metrics
217 metrics_duration = max_rate_system_metrics["duration"]
218 result_text += "system metrics:\n"
219 for key, value in max_rate_system_metrics.items():
220 if not key == "duration":
221 result_text += f"{key}: {value} J\n"
222 dic_out[key] = value
223
224 for key, value in max_rate_system_metrics.items():
225 if not key == "duration":
226 result_text += f"avg power {key} / step time : {value / metrics_duration} W\n"
227 dic_out[f"power_{key}"] = value / metrics_duration
228
229 dic_out["system_metric_duration"] = metrics_duration
230
231 result_text += f"dic_out = {dic_out}\n"
232
233 print("current results:")
234 print(result_text)
235
236 if shamrock.sys.world_rank() == 0:
237 print(result_text)
Estimated memory usage: 0 MB