Note
Go to the end to download the full example code.
AMR benchmark for homogeneous density box#
This example tests the the performance of the AMR solver for a homogeneous density box, the resolution is automatically adapted to the available memory and number of processes.
9 import datetime
10 import json
11 import math
12 from statistics import mean, stdev
13
14 from shamrock.model_ramses import HLLC_PVRS
15
16 import shamrock
17
18 device_properties = shamrock.sys.get_compute_device_properties()
19
20 microbench_results = shamrock.sys.get_microbench_results(allow_run=True)
21 if len(microbench_results) == 0:
22 print("no microbench results, please run with --benchmark-mpi")
23 raise ValueError("no microbench results")
24
25 memory_gb = device_properties["global_mem_size"] / (1e9)
26
27 sz = 1 << 1
28
29 N_target_blocks = 2 ** (math.floor(math.log2(memory_gb * 1e6 / (1.5 * 8))))
30
31
32 print(f"N_target_blocks = {N_target_blocks}")
33 print(f"memory_gb = {memory_gb}")
34 print(f"device_properties = {device_properties}")
35
36 N_target_blocks = min(N_target_blocks, 2**22)
37
38 if device_properties["type"] == "CPU":
39 N_target_blocks = min(N_target_blocks, 2**20)
40
41 # make_base_grid takes a per-axis block count, not a total, so convert the
42 # memory-based total block budget above into a per-axis size (kept as a power
43 # of two so the base grid splits evenly across the 3 axes).
44 N_per_axis = 2 ** (int(math.log2(N_target_blocks)) // 3)
45
46 shamrock.backends.reset_mem_info_max()
47
48
49 compute_multiplier = shamrock.sys.world_size()
50 # compute_multiplier = 12
51 scheduler_split_val = int(2e7)
52 scheduler_merge_val = 1
53
54
55 if shamrock.sys.world_rank() == 0:
56 print("N_target_block", N_target_blocks)
57 print("N_per_axis", N_per_axis)
58 print("scheduler_split_val", scheduler_split_val)
59 print("scheduler_merge_val", scheduler_merge_val)
60 print("N_target", N_per_axis**3 * 8)
61
62
63 ctx = shamrock.Context()
64 ctx.pdata_layout_new()
65
66 model = shamrock.get_Model_Ramses(context=ctx, vector_type="f64_3", grid_repr="i64_3")
67
68 multx = 1
69 multy = 1
70 multz = 1
71
72
73 cfg = model.gen_default_config()
74 scale_fact = 1 / (sz * N_per_axis * multx)
75 cfg.set_scale_factor(scale_fact)
76 cfg.set_riemann_solver(HLLC_PVRS)
77 cfg.set_eos_gamma(1.66667)
78 cfg.set_slope_lim_vanleer_sym()
79 cfg.set_face_time_interpolation(True)
80 model.init_scheduler(scheduler_split_val, scheduler_merge_val)
81 model.make_base_grid(
82 (0, 0, 0),
83 (sz, sz, sz),
84 (N_per_axis * multx, N_per_axis * multy, N_per_axis * multz),
85 )
86
87
88 def rho_map(rmin, rmax):
89 x, y, z = rmin
90 if x > 0.25 and x < 0.75:
91 return 2
92 return 1.0
93
94
95 def rhoe_map(rmin, rmax):
96 rho = rho_map(rmin, rmax)
97 return 1.0 * rho
98
99
100 def rhovel_map(rmin, rmax):
101 rho = rho_map(rmin, rmax)
102 return (1 * rho, 0 * rho, 0 * rho)
103
104
105 model.set_field_value_lambda_f64("rho", rho_map)
106 model.set_field_value_lambda_f64("rhoetot", rhoe_map)
107 model.set_field_value_lambda_f64_3("rhovel", rhovel_map)
108
109
110 # Now run the actual benchmark for 5 steps
111 res_rates = []
112 res_cnts = []
113 res_system_metrics = []
114 res_mpi_timers = []
115
116 """
117 Here we insert callbacks to measure solver MPI usage by fetching the timers twice at the begining and end of the step
118 """
119 before_mpi_timers, after_mpi_timers = None, None
120
121
122 def callback_before_mpi_timer():
123 global before_mpi_timers
124 # print(shamrock.sys.world_rank(), "register before_mpi_timers")
125 before_mpi_timers = shamrock.comm.get_timers()
126
127
128 def callback_after_mpi_timer():
129 global after_mpi_timers
130 # print(shamrock.sys.world_rank(), "register after_mpi_timers")
131 after_mpi_timers = shamrock.comm.get_timers()
132
133
134 model.add_timestep_callback(step_begin=callback_before_mpi_timer, step_end=callback_after_mpi_timer)
135
136 for i in range(10):
137 if shamrock.sys.world_rank() == 0:
138 print("running step ", i + 1, "/", 10, " ...")
139
140 shamrock.sys.mpi_barrier()
141
142 # To replay the same step
143 model.set_next_dt(0.0)
144 model.timestep()
145
146 if shamrock.sys.world_rank() == 0:
147 print("collecting results ...")
148
149 tmp_res_rate, tmp_res_cnt, tmp_system_metrics = (
150 model.solver_logs_last_rate(),
151 model.solver_logs_last_obj_count(),
152 model.solver_logs_last_system_metrics(),
153 )
154 res_rates.append(tmp_res_rate)
155 res_cnts.append(tmp_res_cnt)
156 res_system_metrics.append(tmp_system_metrics)
157 res_mpi_timers.append(shamrock.comm.mpi_timers_delta(before_mpi_timers, after_mpi_timers))
158
159 if shamrock.sys.world_rank() == 0:
160 print("sleeping 1 second ...")
161
162 import time
163
164 time.sleep(1)
165
166 if shamrock.sys.world_rank() == 0:
167 print("done sleeping 1 second ...")
168
169 # result is the best rate of the 5 steps
170 res_rate, res_cnt = max(res_rates), res_cnts[0]
171
172 # index of the max rate
173 max_rate_index = res_rates.index(max(res_rates))
174 max_rate_system_metrics = res_system_metrics[max_rate_index]
175 max_mpi_timers = res_mpi_timers[max_rate_index]
176 step_time = res_cnt / res_rate
177
178 if shamrock.sys.world_rank() == 0:
179 result_text = ""
180 result_text += f"--- final score for N_target_block={N_target_blocks} ---"
181 result_text += f"world size : {shamrock.sys.world_size()}\n"
182 result_text += f"result rate : {res_rate}\n"
183 result_text += f"result cnt : {res_cnt}\n"
184 result_text += f"cnt/rank : {res_cnt / shamrock.sys.world_size()}\n"
185 result_text += f"result rate per rank : {res_rate / shamrock.sys.world_size()}\n"
186 result_text += f"rates infos : max={max(res_rates)}, min={min(res_rates)}, mean={mean(res_rates)}, stddev={stdev(res_rates)}\n"
187 result_text += f"res_rates = {res_rates}\n"
188 result_text += f"res_cnts = {res_cnts}\n"
189 result_text += f"step time = {step_time}\n"
190
191 dic_out = {
192 "device_properties": device_properties,
193 "microbench_results": shamrock.sys.get_microbench_results(),
194 "shamrock_version": shamrock.version_string(),
195 "shamrock_compiler_id_string": shamrock.get_compiler_id_string(),
196 "shamrock_compile_flags": shamrock.get_compile_arg(),
197 "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
198 "world_size": shamrock.sys.world_size(),
199 "rate": res_rate,
200 "cnt": res_cnt,
201 "step_time": step_time,
202 "mpi_timers": max_mpi_timers,
203 }
204
205 # print the system metrics
206 metrics_duration = max_rate_system_metrics["duration"]
207 result_text += "system metrics:\n"
208 for key, value in max_rate_system_metrics.items():
209 if not key == "duration":
210 result_text += f"{key}: {value} J\n"
211 dic_out[key] = value
212
213 for key, value in max_rate_system_metrics.items():
214 if not key == "duration":
215 result_text += f"avg power {key} / step time : {value / metrics_duration} W\n"
216 dic_out[f"power_{key}"] = value / metrics_duration
217
218 dic_out["system_metric_duration"] = metrics_duration
219
220 result_text += "---------submit this result--------\n"
221 result_text += f"{json.dumps(dic_out, indent=4)}\n"
222 result_text += "-----------------------------------\n"
223
224 print("current results:")
225 print(result_text)
Estimated memory usage: 0 MB