Weak scale test for SPH#

This example tests the weak scalability performance of the SPH solver. It is based on a Sedov blast wave simulation, the number of particles is scaled with the number of processes. Run it only if you have enough memory available.

 10 from statistics import mean, stdev
 11
 12 import shamrock
 13 from shamrock import NeighCacheStrategy
 14
 15 result_text = ""
 16
 17 for N_target_base in [32e6]:
 18     shamrock.backends.reset_mem_info_max()
 19
 20     gamma = 5.0 / 3.0
 21     rho_g = 1
 22     target_tot_u = 1
 23
 24     bmin = (-0.6, -0.6, -0.6)
 25     bmax = (0.6, 0.6, 0.6)
 26
 27     compute_multiplier = shamrock.sys.world_size()
 28     # compute_multiplier = 12
 29     scheduler_split_val = int(2e7)
 30     scheduler_merge_val = 1
 31
 32     N_target = N_target_base * compute_multiplier
 33     xm, ym, zm = bmin
 34     xM, yM, zM = bmax
 35     vol_b = (xM - xm) * (yM - ym) * (zM - zm)
 36
 37     if shamrock.sys.world_rank() == 0:
 38         print("N_target_base", N_target_base)
 39         print("compute_multiplier", compute_multiplier)
 40         print("scheduler_split_val", scheduler_split_val)
 41         print("scheduler_merge_val", scheduler_merge_val)
 42         print("N_target", N_target)
 43         print("vol_b", vol_b)
 44
 45     part_vol = vol_b / N_target
 46
 47     # lattice volume
 48     part_vol_lattice = 0.74 * part_vol
 49
 50     dr = (part_vol_lattice / ((4.0 / 3.0) * 3.1416)) ** (1.0 / 3.0)
 51
 52     pmass = -1
 53
 54     ctx = shamrock.Context()
 55     ctx.pdata_layout_new()
 56
 57     model = shamrock.get_Model_SPH(context=ctx, vector_type="f64_3", sph_kernel="M4")
 58
 59     cfg = model.gen_default_config()
 60     # cfg.set_artif_viscosity_Constant(alpha_u = 1, alpha_AV = 1, beta_AV = 2)
 61     # cfg.set_artif_viscosity_VaryingMM97(alpha_min = 0.1,alpha_max = 1,sigma_decay = 0.1, alpha_u = 1, beta_AV = 2)
 62     cfg.set_artif_viscosity_VaryingCD10(
 63         alpha_min=0.0, alpha_max=1, sigma_decay=0.1, alpha_u=1, beta_AV=2
 64     )
 65     cfg.set_neigh_cache_strategy(NeighCacheStrategy.TwoStageSharedOffload)
 66     cfg.set_boundary_periodic()
 67     cfg.set_eos_adiabatic(gamma)
 68     cfg.set_filter_empty_patch_gz(False)
 69     cfg.print_status()
 70     model.set_solver_config(cfg)
 71     model.init_scheduler(scheduler_split_val, scheduler_merge_val)
 72
 73     bmin, bmax = shamrock.math.get_ideal_hcp_box(dr, bmin, bmax)
 74     xm, ym, zm = bmin
 75     xM, yM, zM = bmax
 76
 77     model.resize_simulation_box(bmin, bmax)
 78
 79     setup = model.get_setup()
 80     gen = setup.make_generator_lattice_hcp(
 81         dr, bmin, bmax, discontinuous=False, init_h_factor=2.1402
 82     )
 83
 84     # Kind of optimized for Aurora
 85     setup.apply_setup(
 86         gen,
 87         gen_step=int(scheduler_split_val / 8),
 88         insert_step=int(scheduler_split_val * 2),
 89         msg_count_limit=1024,
 90         rank_comm_size_limit=int(scheduler_split_val) * 2,
 91         max_msg_size=int(scheduler_split_val / 8),
 92         do_setup_log=False,
 93         speculative_balancing=True,
 94     )
 95
 96     xc, yc, zc = model.get_closest_part_to((0, 0, 0))
 97
 98     if shamrock.sys.world_rank() == 0:
 99         print("closest part to (0,0,0) is in :", xc, yc, zc)
100
101     vol_b = (xM - xm) * (yM - ym) * (zM - zm)
102
103     totmass = rho_g * vol_b
104     # print("Total mass :", totmass)
105
106     pmass = model.total_mass_to_part_mass(totmass)
107
108     model.set_value_in_a_box("uint", "f64", 0, bmin, bmax)
109
110     rinj = 16 * dr
111     u_inj = 1
112     model.add_kernel_value("uint", "f64", u_inj, (0, 0, 0), rinj)
113
114     tot_u = pmass * model.get_sum("uint", "f64")
115     if shamrock.sys.world_rank() == 0:
116         print("total u :", tot_u)
117
118     # print("Current part mass :", pmass)
119     model.set_particle_mass(pmass)
120
121     model.set_cfl_cour(0.1)
122     model.set_cfl_force(0.1)
123
124     shamrock.backends.reset_mem_info_max()
125
126     # converge smoothing length and compute initial dt
127     model.timestep()
128
129     # Now run the actual benchmark for 5 steps
130     res_rates = []
131     res_cnts = []
132     res_system_metrics = []
133     res_mpi_timers = []
134
135     """
136     Here we insert callbacks to measure solver MPI usage by fetching the timers twice at the begining and end of the step
137     """
138     before_mpi_timers, after_mpi_timers = None, None
139
140     def callback_before_mpi_timer():
141         global before_mpi_timers
142         # print(shamrock.sys.world_rank(), "register before_mpi_timers")
143         before_mpi_timers = shamrock.comm.get_timers()
144
145     def callback_after_mpi_timer():
146         global after_mpi_timers
147         # print(shamrock.sys.world_rank(), "register after_mpi_timers")
148         after_mpi_timers = shamrock.comm.get_timers()
149
150     model.add_timestep_callback(
151         step_begin=callback_before_mpi_timer, step_end=callback_after_mpi_timer
152     )
153
154     for i in range(10):
155         if shamrock.sys.world_rank() == 0:
156             print("running step ", i + 1, "/", 10, " ...")
157
158         shamrock.sys.mpi_barrier()
159
160         # To replay the same step
161         model.set_next_dt(0.0)
162         model.timestep()
163
164         if shamrock.sys.world_rank() == 0:
165             print("collecting results ...")
166
167         tmp_res_rate, tmp_res_cnt, tmp_system_metrics = (
168             model.solver_logs_last_rate(),
169             model.solver_logs_last_obj_count(),
170             model.solver_logs_last_system_metrics(),
171         )
172         res_rates.append(tmp_res_rate)
173         res_cnts.append(tmp_res_cnt)
174         res_system_metrics.append(tmp_system_metrics)
175         res_mpi_timers.append(shamrock.comm.mpi_timers_delta(before_mpi_timers, after_mpi_timers))
176
177         if shamrock.sys.world_rank() == 0:
178             print("sleeping 1 second ...")
179
180         import time
181
182         time.sleep(1)
183
184         if shamrock.sys.world_rank() == 0:
185             print("done sleeping 1 second ...")
186
187     # result is the best rate of the 5 steps
188     res_rate, res_cnt = max(res_rates), res_cnts[0]
189
190     # index of the max rate
191     max_rate_index = res_rates.index(max(res_rates))
192     max_rate_system_metrics = res_system_metrics[max_rate_index]
193     max_mpi_timers = res_mpi_timers[max_rate_index]
194     step_time = res_cnt / res_rate
195
196     if shamrock.sys.world_rank() == 0:
197         result_text += f"--- final score for N_target_base={N_target_base} ---"
198         result_text += f"world size  : {shamrock.sys.world_size()}\n"
199         result_text += f"result rate : {res_rate}\n"
200         result_text += f"result cnt  : {res_cnt}\n"
201         result_text += f"cnt/rank    : {res_cnt / shamrock.sys.world_size()}\n"
202         result_text += f"result rate per rank : {res_rate / shamrock.sys.world_size()}\n"
203         result_text += f"rates infos : max={max(res_rates)}, min={min(res_rates)}, mean={mean(res_rates)}, stddev={stdev(res_rates)}\n"
204         result_text += f"res_rates = {res_rates}\n"
205         result_text += f"res_cnts = {res_cnts}\n"
206         result_text += f"step time = {step_time}\n"
207
208         dic_out = {
209             "world_size": shamrock.sys.world_size(),
210             "rate": res_rate,
211             "cnt": res_cnt,
212             "step_time": step_time,
213             "mpi_timers": max_mpi_timers,
214         }
215
216         # print the system metrics
217         metrics_duration = max_rate_system_metrics["duration"]
218         result_text += "system metrics:\n"
219         for key, value in max_rate_system_metrics.items():
220             if not key == "duration":
221                 result_text += f"{key}: {value} J\n"
222                 dic_out[key] = value
223
224         for key, value in max_rate_system_metrics.items():
225             if not key == "duration":
226                 result_text += f"avg power {key} / step time : {value / metrics_duration} W\n"
227                 dic_out[f"power_{key}"] = value / metrics_duration
228
229         dic_out["system_metric_duration"] = metrics_duration
230
231         result_text += f"dic_out = {dic_out}\n"
232
233         print("current results:")
234         print(result_text)
235
236 if shamrock.sys.world_rank() == 0:
237     print(result_text)

Estimated memory usage: 0 MB

Gallery generated by Sphinx-Gallery