Note

Go to the end to download the full example code.

SPH benchmark for homogeneous density box#

This example tests the the performance of the SPH solver for a homogeneous density box, the resolution is automatically adapted to the available memory and number of processes.

  9 import datetime
 10 import json
 11 import math
 12 from statistics import mean, stdev
 13
 14 import shamrock
 15
 16 device_properties = shamrock.sys.get_compute_device_properties()
 17
 18 microbench_results = shamrock.sys.get_microbench_results()
 19 if len(microbench_results) == 0:
 20     print("no microbench results, please run with --benchmark-mpi")
 21     raise ValueError("no microbench results")
 22
 23 memory_gb = device_properties["global_mem_size"] / (1e9)
 24
 25 N_target_base = 2 ** int(math.log2(memory_gb * 1e6 / 1.5))
 26 print(f"N_target_base = {N_target_base}")
 27 print(f"memory_gb = {memory_gb}")
 28 print(f"device_properties = {device_properties}")
 29
 30 N_target_base = min(N_target_base, 2**25)
 31
 32 if device_properties["type"] == "CPU":
 33     N_target_base = min(N_target_base, 2**23)
 34
 35 shamrock.backends.reset_mem_info_max()
 36
 37 gamma = 5.0 / 3.0
 38 rho_g = 1
 39 target_tot_u = 1
 40
 41 bmin = (-0.6, -0.6, -0.6)
 42 bmax = (0.6, 0.6, 0.6)
 43
 44 compute_multiplier = shamrock.sys.world_size()
 45 # compute_multiplier = 12
 46 scheduler_split_val = int(2e7)
 47 scheduler_merge_val = 1
 48
 49 N_target = N_target_base * compute_multiplier
 50 xm, ym, zm = bmin
 51 xM, yM, zM = bmax
 52 vol_b = (xM - xm) * (yM - ym) * (zM - zm)
 53
 54 if shamrock.sys.world_rank() == 0:
 55     print("N_target_base", N_target_base)
 56     print("compute_multiplier", compute_multiplier)
 57     print("scheduler_split_val", scheduler_split_val)
 58     print("scheduler_merge_val", scheduler_merge_val)
 59     print("N_target", N_target)
 60     print("vol_b", vol_b)
 61
 62 part_vol = vol_b / N_target
 63
 64 # lattice volume
 65 part_vol_lattice = 0.74 * part_vol
 66
 67 dr = (part_vol_lattice / ((4.0 / 3.0) * 3.1416)) ** (1.0 / 3.0)
 68
 69 pmass = -1
 70
 71 ctx = shamrock.Context()
 72 ctx.pdata_layout_new()
 73
 74 model = shamrock.get_Model_SPH(context=ctx, vector_type="f64_3", sph_kernel="M4")
 75
 76 cfg = model.gen_default_config()
 77 # cfg.set_artif_viscosity_Constant(alpha_u = 1, alpha_AV = 1, beta_AV = 2)
 78 # cfg.set_artif_viscosity_VaryingMM97(alpha_min = 0.1,alpha_max = 1,sigma_decay = 0.1, alpha_u = 1, beta_AV = 2)
 79 cfg.set_artif_viscosity_VaryingCD10(
 80     alpha_min=0.0, alpha_max=1, sigma_decay=0.1, alpha_u=1, beta_AV=2
 81 )
 82 cfg.set_boundary_periodic()
 83 cfg.set_eos_adiabatic(gamma)
 84 cfg.print_status()
 85 model.set_solver_config(cfg)
 86 model.init_scheduler(scheduler_split_val, scheduler_merge_val)
 87
 88 bmin, bmax = shamrock.math.get_ideal_hcp_box(dr, bmin, bmax)
 89 xm, ym, zm = bmin
 90 xM, yM, zM = bmax
 91
 92 model.resize_simulation_box(bmin, bmax)
 93
 94 setup = model.get_setup()
 95 gen = setup.make_generator_lattice_hcp(dr, bmin, bmax)
 96
 97 # Kind of optimized for Aurora
 98 setup.apply_setup(
 99     gen,
100     gen_step=int(scheduler_split_val / 8),
101     insert_step=int(scheduler_split_val * 2),
102     msg_count_limit=1024,
103     rank_comm_size_limit=int(scheduler_split_val) * 2,
104     max_msg_size=int(scheduler_split_val / 8),
105     do_setup_log=False,
106 )
107
108 xc, yc, zc = model.get_closest_part_to((0, 0, 0))
109
110 if shamrock.sys.world_rank() == 0:
111     print("closest part to (0,0,0) is in :", xc, yc, zc)
112
113 vol_b = (xM - xm) * (yM - ym) * (zM - zm)
114
115 totmass = rho_g * vol_b
116 # print("Total mass :", totmass)
117
118 pmass = model.total_mass_to_part_mass(totmass)
119
120 model.set_value_in_a_box("uint", "f64", 0, bmin, bmax)
121
122 rinj = 16 * dr
123 u_inj = 1
124 model.add_kernel_value("uint", "f64", u_inj, (0, 0, 0), rinj)
125
126 tot_u = pmass * model.get_sum("uint", "f64")
127 if shamrock.sys.world_rank() == 0:
128     print("total u :", tot_u)
129
130 # print("Current part mass :", pmass)
131 model.set_particle_mass(pmass)
132
133 model.set_cfl_cour(0.1)
134 model.set_cfl_force(0.1)
135
136 shamrock.backends.reset_mem_info_max()
137
138 # converge smoothing length and compute initial dt
139 model.timestep()
140
141 # Now run the actual benchmark for 5 steps
142 res_rates = []
143 res_cnts = []
144 res_system_metrics = []
145 res_mpi_timers = []
146
147 """
148 Here we insert callbacks to measure solver MPI usage by fetching the timers twice at the begining and end of the step
149 """
150 before_mpi_timers, after_mpi_timers = None, None
151
152
153 def callback_before_mpi_timer():
154     global before_mpi_timers
155     # print(shamrock.sys.world_rank(), "register before_mpi_timers")
156     before_mpi_timers = shamrock.comm.get_timers()
157
158
159 def callback_after_mpi_timer():
160     global after_mpi_timers
161     # print(shamrock.sys.world_rank(), "register after_mpi_timers")
162     after_mpi_timers = shamrock.comm.get_timers()
163
164
165 model.add_timestep_callback(step_begin=callback_before_mpi_timer, step_end=callback_after_mpi_timer)
166
167 for i in range(10):
168     if shamrock.sys.world_rank() == 0:
169         print("running step ", i + 1, "/", 10, " ...")
170
171     shamrock.sys.mpi_barrier()
172
173     # To replay the same step
174     model.set_next_dt(0.0)
175     model.timestep()
176
177     if shamrock.sys.world_rank() == 0:
178         print("collecting results ...")
179
180     tmp_res_rate, tmp_res_cnt, tmp_system_metrics = (
181         model.solver_logs_last_rate(),
182         model.solver_logs_last_obj_count(),
183         model.solver_logs_last_system_metrics(),
184     )
185     res_rates.append(tmp_res_rate)
186     res_cnts.append(tmp_res_cnt)
187     res_system_metrics.append(tmp_system_metrics)
188     res_mpi_timers.append(shamrock.comm.mpi_timers_delta(before_mpi_timers, after_mpi_timers))
189
190     if shamrock.sys.world_rank() == 0:
191         print("sleeping 1 second ...")
192
193     import time
194
195     time.sleep(1)
196
197     if shamrock.sys.world_rank() == 0:
198         print("done sleeping 1 second ...")
199
200 # result is the best rate of the 5 steps
201 res_rate, res_cnt = max(res_rates), res_cnts[0]
202
203 # index of the max rate
204 max_rate_index = res_rates.index(max(res_rates))
205 max_rate_system_metrics = res_system_metrics[max_rate_index]
206 max_mpi_timers = res_mpi_timers[max_rate_index]
207 step_time = res_cnt / res_rate
208
209 if shamrock.sys.world_rank() == 0:
210     result_text = ""
211     result_text += f"--- final score for N_target_base={N_target_base} ---"
212     result_text += f"world size  : {shamrock.sys.world_size()}\n"
213     result_text += f"result rate : {res_rate}\n"
214     result_text += f"result cnt  : {res_cnt}\n"
215     result_text += f"cnt/rank    : {res_cnt / shamrock.sys.world_size()}\n"
216     result_text += f"result rate per rank : {res_rate / shamrock.sys.world_size()}\n"
217     result_text += f"rates infos : max={max(res_rates)}, min={min(res_rates)}, mean={mean(res_rates)}, stddev={stdev(res_rates)}\n"
218     result_text += f"res_rates = {res_rates}\n"
219     result_text += f"res_cnts = {res_cnts}\n"
220     result_text += f"step time = {step_time}\n"
221
222     dic_out = {
223         "device_properties": device_properties,
224         "microbench_results": shamrock.sys.get_microbench_results(),
225         "shamrock_version": shamrock.version_string(),
226         "shamrock_compiler_id_string": shamrock.get_compiler_id_string(),
227         "shamrock_compile_flags": shamrock.get_compile_arg(),
228         "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
229         "world_size": shamrock.sys.world_size(),
230         "rate": res_rate,
231         "cnt": res_cnt,
232         "step_time": step_time,
233         "mpi_timers": max_mpi_timers,
234     }
235
236     # print the system metrics
237     metrics_duration = max_rate_system_metrics["duration"]
238     result_text += "system metrics:\n"
239     for key, value in max_rate_system_metrics.items():
240         if not key == "duration":
241             result_text += f"{key}: {value} J\n"
242             dic_out[key] = value
243
244     for key, value in max_rate_system_metrics.items():
245         if not key == "duration":
246             result_text += f"avg power {key} / step time : {value / metrics_duration} W\n"
247             dic_out[f"power_{key}"] = value / metrics_duration
248
249     dic_out["system_metric_duration"] = metrics_duration
250
251     result_text += "---------submit this result--------\n"
252     result_text += f"{json.dumps(dic_out, indent=4)}\n"
253     result_text += "-----------------------------------\n"
254
255     print("current results:")
256     print(result_text)

Estimated memory usage: 0 MB

Gallery generated by Sphinx-Gallery