AMR benchmark for homogeneous density box#

This example tests the the performance of the AMR solver for a homogeneous density box, the resolution is automatically adapted to the available memory and number of processes.

  9 import datetime
 10 import json
 11 import math
 12 from statistics import mean, stdev
 13
 14 from shamrock.model_ramses import HLLC_PVRS
 15
 16 import shamrock
 17
 18 device_properties = shamrock.sys.get_compute_device_properties()
 19
 20 microbench_results = shamrock.sys.get_microbench_results(allow_run=True)
 21 if len(microbench_results) == 0:
 22     print("no microbench results, please run with --benchmark-mpi")
 23     raise ValueError("no microbench results")
 24
 25 memory_gb = device_properties["global_mem_size"] / (1e9)
 26
 27 sz = 1 << 1
 28
 29 N_target_blocks = 2 ** (math.floor(math.log2(memory_gb * 1e6 / (1.5 * 8))))
 30
 31
 32 print(f"N_target_blocks = {N_target_blocks}")
 33 print(f"memory_gb = {memory_gb}")
 34 print(f"device_properties = {device_properties}")
 35
 36 N_target_blocks = min(N_target_blocks, 2**22)
 37
 38 if device_properties["type"] == "CPU":
 39     N_target_blocks = min(N_target_blocks, 2**20)
 40
 41 # make_base_grid takes a per-axis block count, not a total, so convert the
 42 # memory-based total block budget above into a per-axis size (kept as a power
 43 # of two so the base grid splits evenly across the 3 axes).
 44 N_per_axis = 2 ** (int(math.log2(N_target_blocks)) // 3)
 45
 46 shamrock.backends.reset_mem_info_max()
 47
 48
 49 compute_multiplier = shamrock.sys.world_size()
 50 # compute_multiplier = 12
 51 scheduler_split_val = int(2e7)
 52 scheduler_merge_val = 1
 53
 54
 55 if shamrock.sys.world_rank() == 0:
 56     print("N_target_block", N_target_blocks)
 57     print("N_per_axis", N_per_axis)
 58     print("scheduler_split_val", scheduler_split_val)
 59     print("scheduler_merge_val", scheduler_merge_val)
 60     print("N_target", N_per_axis**3 * 8)
 61
 62
 63 ctx = shamrock.Context()
 64 ctx.pdata_layout_new()
 65
 66 model = shamrock.get_Model_Ramses(context=ctx, vector_type="f64_3", grid_repr="i64_3")
 67
 68 multx = 1
 69 multy = 1
 70 multz = 1
 71
 72
 73 cfg = model.gen_default_config()
 74 scale_fact = 1 / (sz * N_per_axis * multx)
 75 cfg.set_scale_factor(scale_fact)
 76 cfg.set_riemann_solver(HLLC_PVRS)
 77 cfg.set_eos_gamma(1.66667)
 78 cfg.set_slope_lim_vanleer_sym()
 79 cfg.set_face_time_interpolation(True)
 80 model.init_scheduler(scheduler_split_val, scheduler_merge_val)
 81 model.make_base_grid(
 82     (0, 0, 0),
 83     (sz, sz, sz),
 84     (N_per_axis * multx, N_per_axis * multy, N_per_axis * multz),
 85 )
 86
 87
 88 def rho_map(rmin, rmax):
 89     x, y, z = rmin
 90     if x > 0.25 and x < 0.75:
 91         return 2
 92     return 1.0
 93
 94
 95 def rhoe_map(rmin, rmax):
 96     rho = rho_map(rmin, rmax)
 97     return 1.0 * rho
 98
 99
100 def rhovel_map(rmin, rmax):
101     rho = rho_map(rmin, rmax)
102     return (1 * rho, 0 * rho, 0 * rho)
103
104
105 model.set_field_value_lambda_f64("rho", rho_map)
106 model.set_field_value_lambda_f64("rhoetot", rhoe_map)
107 model.set_field_value_lambda_f64_3("rhovel", rhovel_map)
108
109
110 # Now run the actual benchmark for 5 steps
111 res_rates = []
112 res_cnts = []
113 res_system_metrics = []
114 res_mpi_timers = []
115
116 """
117 Here we insert callbacks to measure solver MPI usage by fetching the timers twice at the begining and end of the step
118 """
119 before_mpi_timers, after_mpi_timers = None, None
120
121
122 def callback_before_mpi_timer():
123     global before_mpi_timers
124     # print(shamrock.sys.world_rank(), "register before_mpi_timers")
125     before_mpi_timers = shamrock.comm.get_timers()
126
127
128 def callback_after_mpi_timer():
129     global after_mpi_timers
130     # print(shamrock.sys.world_rank(), "register after_mpi_timers")
131     after_mpi_timers = shamrock.comm.get_timers()
132
133
134 model.add_timestep_callback(step_begin=callback_before_mpi_timer, step_end=callback_after_mpi_timer)
135
136 for i in range(10):
137     if shamrock.sys.world_rank() == 0:
138         print("running step ", i + 1, "/", 10, " ...")
139
140     shamrock.sys.mpi_barrier()
141
142     # To replay the same step
143     model.set_next_dt(0.0)
144     model.timestep()
145
146     if shamrock.sys.world_rank() == 0:
147         print("collecting results ...")
148
149     tmp_res_rate, tmp_res_cnt, tmp_system_metrics = (
150         model.solver_logs_last_rate(),
151         model.solver_logs_last_obj_count(),
152         model.solver_logs_last_system_metrics(),
153     )
154     res_rates.append(tmp_res_rate)
155     res_cnts.append(tmp_res_cnt)
156     res_system_metrics.append(tmp_system_metrics)
157     res_mpi_timers.append(shamrock.comm.mpi_timers_delta(before_mpi_timers, after_mpi_timers))
158
159     if shamrock.sys.world_rank() == 0:
160         print("sleeping 1 second ...")
161
162     import time
163
164     time.sleep(1)
165
166     if shamrock.sys.world_rank() == 0:
167         print("done sleeping 1 second ...")
168
169 # result is the best rate of the 5 steps
170 res_rate, res_cnt = max(res_rates), res_cnts[0]
171
172 # index of the max rate
173 max_rate_index = res_rates.index(max(res_rates))
174 max_rate_system_metrics = res_system_metrics[max_rate_index]
175 max_mpi_timers = res_mpi_timers[max_rate_index]
176 step_time = res_cnt / res_rate
177
178 if shamrock.sys.world_rank() == 0:
179     result_text = ""
180     result_text += f"--- final score for N_target_block={N_target_blocks} ---"
181     result_text += f"world size  : {shamrock.sys.world_size()}\n"
182     result_text += f"result rate : {res_rate}\n"
183     result_text += f"result cnt  : {res_cnt}\n"
184     result_text += f"cnt/rank    : {res_cnt / shamrock.sys.world_size()}\n"
185     result_text += f"result rate per rank : {res_rate / shamrock.sys.world_size()}\n"
186     result_text += f"rates infos : max={max(res_rates)}, min={min(res_rates)}, mean={mean(res_rates)}, stddev={stdev(res_rates)}\n"
187     result_text += f"res_rates = {res_rates}\n"
188     result_text += f"res_cnts = {res_cnts}\n"
189     result_text += f"step time = {step_time}\n"
190
191     dic_out = {
192         "device_properties": device_properties,
193         "microbench_results": shamrock.sys.get_microbench_results(),
194         "shamrock_version": shamrock.version_string(),
195         "shamrock_compiler_id_string": shamrock.get_compiler_id_string(),
196         "shamrock_compile_flags": shamrock.get_compile_arg(),
197         "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
198         "world_size": shamrock.sys.world_size(),
199         "rate": res_rate,
200         "cnt": res_cnt,
201         "step_time": step_time,
202         "mpi_timers": max_mpi_timers,
203     }
204
205     # print the system metrics
206     metrics_duration = max_rate_system_metrics["duration"]
207     result_text += "system metrics:\n"
208     for key, value in max_rate_system_metrics.items():
209         if not key == "duration":
210             result_text += f"{key}: {value} J\n"
211             dic_out[key] = value
212
213     for key, value in max_rate_system_metrics.items():
214         if not key == "duration":
215             result_text += f"avg power {key} / step time : {value / metrics_duration} W\n"
216             dic_out[f"power_{key}"] = value / metrics_duration
217
218     dic_out["system_metric_duration"] = metrics_duration
219
220     result_text += "---------submit this result--------\n"
221     result_text += f"{json.dumps(dic_out, indent=4)}\n"
222     result_text += "-----------------------------------\n"
223
224     print("current results:")
225     print(result_text)

Estimated memory usage: 0 MB

Gallery generated by Sphinx-Gallery