DTT performance benchmarks#

This example benchmarks the DTT performance for the different algorithms available in Shamrock

 import json
 import random
 import time

 import matplotlib.pyplot as plt
 import numpy as np
 from matplotlib import colors

 import shamrock

 # If we use the shamrock executable to run this script instead of the python interpreter,
 # we should not initialize the system as the shamrock executable needs to handle specific MPI logic
 if not shamrock.sys.is_initialized():
     shamrock.change_loglevel(1)
     shamrock.sys.init("0:0")

Use shamrock documentation style for matplotlib

 shamrock.matplotlib.set_shamrock_mpl_style()

Main benchmark functions

 bounding_box = shamrock.math.AABB_f64_3((0.0, 0.0, 0.0), (1.0, 1.0, 1.0))


 def benchmark_dtt_core(N, theta_crit, compression_level, ordered_result, nb_repeat=10):
     times = []
     random.seed(111)
     max_mem_delta = 0
     for i in range(nb_repeat):
         positions = shamrock.algs.mock_buffer_f64_3(
             random.randint(0, 1000000), N, bounding_box.lower, bounding_box.upper
         )
         tree = shamrock.tree.CLBVH_u64_f64_3()
         tree.rebuild_from_positions(positions, bounding_box, compression_level)
         shamrock.backends.reset_mem_info_max()
         mem_info_before = shamrock.backends.get_mem_perf_info()
         times.append(
             shamrock.tree.benchmark_clbvh_dual_tree_traversal(tree, theta_crit, ordered_result)
             * 1000
         )
         mem_info_after = shamrock.backends.get_mem_perf_info()

         mem_delta = (
             mem_info_after.max_allocated_byte_device - mem_info_before.max_allocated_byte_device
         )
         max_mem_delta = max(max_mem_delta, mem_delta)
     return times, max_mem_delta


 def benchmark_dtt(N, theta_crit, compression_level, ordered_result, nb_repeat=10):
     times, max_mem_delta = benchmark_dtt_core(
         N, theta_crit, compression_level, ordered_result, nb_repeat
     )
     return min(times), max(times), sum(times) / nb_repeat, max_mem_delta

Run the performance test for all parameters

 def run_performance_sweep(compression_level, threshold_run, ordered_result):
     # Define parameter ranges
     # logspace as array
     particle_counts = np.logspace(2, 7, 10).astype(int).tolist()
     theta_crits = [0.1, 0.3, 0.5, 0.7, 0.9]

     # Initialize results matrix
     results_mean = np.zeros((len(theta_crits), len(particle_counts)))
     results_min = np.zeros((len(theta_crits), len(particle_counts)))
     results_max = np.zeros((len(theta_crits), len(particle_counts)))
     results_max_mem_delta = np.zeros((len(theta_crits), len(particle_counts)))

     print(f"Particle counts: {particle_counts}")
     print(f"Theta_crit values: {theta_crits}")
     print(f"Compression level: {compression_level}")

     total_runs = len(particle_counts) * len(theta_crits)
     current_run = 0

     for i, theta_crit in enumerate(theta_crits):
         exceed_mem = False
         for j, N in enumerate(particle_counts):
             current_run += 1

             if exceed_mem:
                 print(
                     f"[{current_run:2d}/{total_runs}] Skipping N={N:5d}, theta_crit={theta_crit:.1f}"
                 )
                 results_mean[i, j] = np.nan
                 results_min[i, j] = np.nan
                 results_max[i, j] = np.nan
                 continue

             print(
                 f"[{current_run:2d}/{total_runs}] Running N={N:5d}, theta_crit={theta_crit:.1f}...",
                 end=" ",
             )

             start_time = time.time()
             min_time, max_time, mean_time, max_mem_delta = benchmark_dtt(
                 N, theta_crit, compression_level, ordered_result
             )
             elapsed = time.time() - start_time

             results_mean[i, j] = mean_time
             results_min[i, j] = min_time
             results_max[i, j] = max_time
             results_max_mem_delta[i, j] = max_mem_delta

             print(f"mean={mean_time:.3f}ms (took {elapsed:.1f}s)")

             if max_mem_delta > threshold_run:
                 exceed_mem = True

     return (
         particle_counts,
         theta_crits,
         results_mean,
         results_min,
         results_max,
         results_max_mem_delta,
     )

Create checkerboard plot with execution times and relative performance to reference algorithm

 def create_checkerboard_plot(
     particle_counts,
     theta_crits,
     results_data,
     compression_level,
     algname,
     max_axis_value,
     reference_data,
     results_max_mem_delta,
 ):
     """Create checkerboard plot with execution times"""

     fig, ax = plt.subplots(figsize=(12, 8))

     # Calculate relative performance compared to reference algorithm
     # results_data / reference_data gives the ratio (>1 means slower, <1 means faster)
     relative_performance = results_data / reference_data

     # Create the heatmap with relative performance values
     # Create a masked array to handle NaN values (skipped benchmarks) as white
     masked_relative = np.ma.masked_invalid(relative_performance)

     # Use a diverging colormap: red for better performance (<1), green for worse (>1)
     # RdYlGn_r (reversed) has green for high values (worse) and red for low values (better)
     cmap = plt.cm.RdYlGn_r.copy()  # Green for >1 (slower), Red for <1 (faster)
     cmap.set_bad(color="white")  # Set NaN values to white

     # Set the color scale limits for relative performance
     vmin = 0.5
     vmax = 1.5

     im = ax.imshow(
         masked_relative, cmap=cmap, aspect="auto", interpolation="nearest", vmin=vmin, vmax=vmax
     )

     # Set ticks and labels
     ax.set_xticks(range(len(particle_counts)))
     ax.set_yticks(range(len(theta_crits)))
     ax.set_xticklabels([f"{N // 1000}k" if N >= 1000 else str(N) for N in particle_counts])
     ax.set_yticklabels([f"{theta:.1f}" for theta in theta_crits])

     # Add labels
     ax.set_xlabel("Particle Count")
     ax.set_ylabel("Theta Critical")
     ax.set_title(
         f"Dual Tree Traversal Performance\n(Colors: Relative to Reference, Text: Absolute Time in ms)\ncompression level = {compression_level} algorithm = {algname}",
         pad=20,
     )

     # Add text annotations showing the values
     for i in range(len(theta_crits)):
         for j in range(len(particle_counts)):
             value = results_data[i, j]

             if np.isnan(value):
                 # For skipped benchmarks, show "SKIPPED" in black on white background
                 # ax.text(j, i, 'SKIPPED', ha='center', va='center',
                 #       color='black', fontweight='bold', fontsize=8)
                 pass
             else:
                 perf = relative_performance[i, j]
                 mem_delta = results_max_mem_delta[i, j] / 1e6
                 text_color = "black"
                 ax.text(
                     j,
                     i,
                     f"{value:.2f}ms\n{perf:.2f}\n{mem_delta:.2f}MB",
                     ha="center",
                     va="center",
                     color=text_color,
                     fontweight="bold",
                     fontsize=10,
                 )

     # Add colorbar for relative performance
     cbar = plt.colorbar(im, ax=ax, shrink=0.8)
     cbar.set_label("Relative performance (time / reference time)")
     cbar.ax.tick_params(labelsize=10)

     # Add custom tick labels for better interpretation
     tick_positions = [0.1, 0.2, 0.5, 1.0, 2.0, 3.0]
     cbar.set_ticks([pos for pos in tick_positions if vmin <= pos <= vmax])

     # Improve layout
     plt.tight_layout()

     # Add grid for better readability
     ax.set_xticks(np.arange(len(particle_counts)) - 0.5, minor=True)
     ax.set_yticks(np.arange(len(theta_crits)) - 0.5, minor=True)
     ax.grid(which="minor", color="black", linestyle="-", linewidth=1, alpha=0.3)

     return fig, ax

List current implementation

 if not shamrock.tree.is_impl_set_clbvh_dual_tree_traversal():
     shamrock.tree.autoselect_impl_clbvh_dual_tree_traversal()

 current_impl = shamrock.tree.get_current_impl_clbvh_dual_tree_traversal_impl()

 print(current_impl)

Info: defaulting dtt implementation to impl : {"implementation":"scan_multipass","parameters":{}}  [tree][rank=0]
{"implementation":"scan_multipass","parameters":{}}

List all implementations available

 all_default_impls = shamrock.tree.get_default_impl_list_clbvh_dual_tree_traversal()

 print(all_default_impls)

['{"implementation":"reference","parameters":{}}', '{"implementation":"parallel_select","parameters":{}}', '{"implementation":"scan_multipass","parameters":{}}']

Run the performance benchmarks for all implementations

 results = {}


 for ordered_result in [True, False]:
     for default_impl in all_default_impls:
         shamrock.tree.set_impl_clbvh_dual_tree_traversal(default_impl)

         impl_name = json.loads(default_impl)["implementation"]
         n = impl_name + " ordered=" + str(ordered_result)

         print(f"Running DTT performance benchmarks for {n}...")

         compression_level = 4

         threshold_run = 5e6
         # Run the performance sweep
         (
             particle_counts,
             theta_crits,
             results_mean,
             results_min,
             results_max,
             results_max_mem_delta,
         ) = run_performance_sweep(compression_level, threshold_run, ordered_result)

         results[n] = {
             "particle_counts": particle_counts,
             "theta_crits": theta_crits,
             "results_mean": results_mean,
             "results_min": results_min,
             "results_max": results_max,
             "results_max_mem_delta": results_max_mem_delta,
             "name": n,
         }

Info: setting dtt implementation to impl : {"implementation":"reference","parameters":{}}  [tree][rank=0]
Running DTT performance benchmarks for reference ordered=True...
Particle counts: [100, 359, 1291, 4641, 16681, 59948, 215443, 774263, 2782559, 10000000]
Theta_crit values: [0.1, 0.3, 0.5, 0.7, 0.9]
Compression level: 4
[ 1/50] Running N=  100, theta_crit=0.1... Info: defaulting sort by key (pow2 len) implementation to impl : {"implementation":"bitonic_sort","parameters":{"stencil_size":16}}  [algs][rank=0]
mean=2.284ms (took 0.0s)
[ 2/50] Running N=  359, theta_crit=0.1... mean=2.334ms (took 0.0s)
[ 3/50] Running N= 1291, theta_crit=0.1... mean=2.590ms (took 0.1s)
[ 4/50] Running N= 4641, theta_crit=0.1... mean=6.233ms (took 0.1s)
[ 5/50] Running N=16681, theta_crit=0.1... mean=48.855ms (took 0.6s)
[ 6/50] Skipping N=59948, theta_crit=0.1
[ 7/50] Skipping N=215443, theta_crit=0.1
[ 8/50] Skipping N=774263, theta_crit=0.1
[ 9/50] Skipping N=2782559, theta_crit=0.1
[10/50] Skipping N=10000000, theta_crit=0.1
[11/50] Running N=  100, theta_crit=0.3... mean=2.139ms (took 0.0s)
[12/50] Running N=  359, theta_crit=0.3... mean=2.383ms (took 0.0s)
[13/50] Running N= 1291, theta_crit=0.3... mean=3.034ms (took 0.1s)
[14/50] Running N= 4641, theta_crit=0.3... mean=5.005ms (took 0.1s)
[15/50] Running N=16681, theta_crit=0.3... mean=15.378ms (took 0.3s)
[16/50] Running N=59948, theta_crit=0.3... mean=91.509ms (took 1.3s)
[17/50] Skipping N=215443, theta_crit=0.3
[18/50] Skipping N=774263, theta_crit=0.3
[19/50] Skipping N=2782559, theta_crit=0.3
[20/50] Skipping N=10000000, theta_crit=0.3
[21/50] Running N=  100, theta_crit=0.5... mean=2.278ms (took 0.0s)
[22/50] Running N=  359, theta_crit=0.5... mean=2.800ms (took 0.1s)
[23/50] Running N= 1291, theta_crit=0.5... mean=2.980ms (took 0.1s)
[24/50] Running N= 4641, theta_crit=0.5... mean=3.556ms (took 0.1s)
[25/50] Running N=16681, theta_crit=0.5... mean=6.511ms (took 0.2s)
[26/50] Running N=59948, theta_crit=0.5... mean=25.027ms (took 0.7s)
[27/50] Skipping N=215443, theta_crit=0.5
[28/50] Skipping N=774263, theta_crit=0.5
[29/50] Skipping N=2782559, theta_crit=0.5
[30/50] Skipping N=10000000, theta_crit=0.5
[31/50] Running N=  100, theta_crit=0.7... mean=2.187ms (took 0.0s)
[32/50] Running N=  359, theta_crit=0.7... mean=2.316ms (took 0.0s)
[33/50] Running N= 1291, theta_crit=0.7... mean=2.489ms (took 0.1s)
[34/50] Running N= 4641, theta_crit=0.7... mean=2.885ms (took 0.1s)
[35/50] Running N=16681, theta_crit=0.7... mean=4.645ms (took 0.2s)
[36/50] Running N=59948, theta_crit=0.7... mean=14.969ms (took 0.6s)
[37/50] Running N=215443, theta_crit=0.7... mean=37.716ms (took 1.9s)
[38/50] Skipping N=774263, theta_crit=0.7
[39/50] Skipping N=2782559, theta_crit=0.7
[40/50] Skipping N=10000000, theta_crit=0.7
[41/50] Running N=  100, theta_crit=0.9... mean=2.424ms (took 0.0s)
[42/50] Running N=  359, theta_crit=0.9... mean=2.616ms (took 0.0s)
[43/50] Running N= 1291, theta_crit=0.9... mean=2.655ms (took 0.1s)
[44/50] Running N= 4641, theta_crit=0.9... mean=2.998ms (took 0.1s)
[45/50] Running N=16681, theta_crit=0.9... mean=3.994ms (took 0.2s)
[46/50] Running N=59948, theta_crit=0.9... mean=8.286ms (took 0.4s)
[47/50] Running N=215443, theta_crit=0.9... mean=22.401ms (took 1.6s)
[48/50] Running N=774263, theta_crit=0.9... mean=76.388ms (took 5.6s)
[49/50] Skipping N=2782559, theta_crit=0.9
[50/50] Skipping N=10000000, theta_crit=0.9
Info: setting dtt implementation to impl : {"implementation":"parallel_select","parameters":{}}  [tree][rank=0]
Running DTT performance benchmarks for parallel_select ordered=True...
Particle counts: [100, 359, 1291, 4641, 16681, 59948, 215443, 774263, 2782559, 10000000]
Theta_crit values: [0.1, 0.3, 0.5, 0.7, 0.9]
Compression level: 4
[ 1/50] Running N=  100, theta_crit=0.1... mean=1.712ms (took 0.0s)
[ 2/50] Running N=  359, theta_crit=0.1... mean=1.770ms (took 0.0s)
[ 3/50] Running N= 1291, theta_crit=0.1... mean=2.278ms (took 0.1s)
[ 4/50] Running N= 4641, theta_crit=0.1... mean=8.814ms (took 0.1s)
[ 5/50] Running N=16681, theta_crit=0.1... mean=96.695ms (took 1.1s)
[ 6/50] Skipping N=59948, theta_crit=0.1
[ 7/50] Skipping N=215443, theta_crit=0.1
[ 8/50] Skipping N=774263, theta_crit=0.1
[ 9/50] Skipping N=2782559, theta_crit=0.1
[10/50] Skipping N=10000000, theta_crit=0.1
[11/50] Running N=  100, theta_crit=0.3... mean=1.669ms (took 0.0s)
[12/50] Running N=  359, theta_crit=0.3... mean=1.317ms (took 0.0s)
[13/50] Running N= 1291, theta_crit=0.3... mean=1.920ms (took 0.0s)
[14/50] Running N= 4641, theta_crit=0.3... mean=8.606ms (took 0.1s)
[15/50] Running N=16681, theta_crit=0.3... mean=54.902ms (took 0.7s)
[16/50] Running N=59948, theta_crit=0.3... mean=362.670ms (took 4.0s)
[17/50] Skipping N=215443, theta_crit=0.3
[18/50] Skipping N=774263, theta_crit=0.3
[19/50] Skipping N=2782559, theta_crit=0.3
[20/50] Skipping N=10000000, theta_crit=0.3
[21/50] Running N=  100, theta_crit=0.5... mean=1.531ms (took 0.0s)
[22/50] Running N=  359, theta_crit=0.5... mean=1.521ms (took 0.0s)
[23/50] Running N= 1291, theta_crit=0.5... mean=1.869ms (took 0.0s)
[24/50] Running N= 4641, theta_crit=0.5... mean=4.490ms (took 0.1s)
[25/50] Running N=16681, theta_crit=0.5... mean=22.030ms (took 0.3s)
[26/50] Running N=59948, theta_crit=0.5... mean=113.688ms (took 1.5s)
[27/50] Running N=215443, theta_crit=0.5... mean=495.765ms (took 6.3s)
[28/50] Skipping N=774263, theta_crit=0.5
[29/50] Skipping N=2782559, theta_crit=0.5
[30/50] Skipping N=10000000, theta_crit=0.5
[31/50] Running N=  100, theta_crit=0.7... mean=1.167ms (took 0.0s)
[32/50] Running N=  359, theta_crit=0.7... mean=1.235ms (took 0.0s)
[33/50] Running N= 1291, theta_crit=0.7... mean=2.017ms (took 0.0s)
[34/50] Running N= 4641, theta_crit=0.7... mean=3.642ms (took 0.1s)
[35/50] Running N=16681, theta_crit=0.7... mean=12.971ms (took 0.3s)
[36/50] Running N=59948, theta_crit=0.7... mean=62.280ms (took 1.0s)
[37/50] Running N=215443, theta_crit=0.7... mean=282.161ms (took 4.2s)
[38/50] Skipping N=774263, theta_crit=0.7
[39/50] Skipping N=2782559, theta_crit=0.7
[40/50] Skipping N=10000000, theta_crit=0.7
[41/50] Running N=  100, theta_crit=0.9... mean=1.168ms (took 0.0s)
[42/50] Running N=  359, theta_crit=0.9... mean=1.220ms (took 0.0s)
[43/50] Running N= 1291, theta_crit=0.9... mean=1.470ms (took 0.0s)
[44/50] Running N= 4641, theta_crit=0.9... mean=2.450ms (took 0.1s)
[45/50] Running N=16681, theta_crit=0.9... mean=7.154ms (took 0.2s)
[46/50] Running N=59948, theta_crit=0.9... mean=29.760ms (took 0.6s)
[47/50] Running N=215443, theta_crit=0.9... mean=124.784ms (took 2.6s)
[48/50] Running N=774263, theta_crit=0.9... mean=544.365ms (took 10.5s)
[49/50] Skipping N=2782559, theta_crit=0.9
[50/50] Skipping N=10000000, theta_crit=0.9
Info: setting dtt implementation to impl : {"implementation":"scan_multipass","parameters":{}}  [tree][rank=0]
Running DTT performance benchmarks for scan_multipass ordered=True...
Particle counts: [100, 359, 1291, 4641, 16681, 59948, 215443, 774263, 2782559, 10000000]
Theta_crit values: [0.1, 0.3, 0.5, 0.7, 0.9]
Compression level: 4
[ 1/50] Running N=  100, theta_crit=0.1... mean=5.640ms (took 0.1s)
[ 2/50] Running N=  359, theta_crit=0.1... mean=7.979ms (took 0.1s)
[ 3/50] Running N= 1291, theta_crit=0.1... mean=10.508ms (took 0.1s)
[ 4/50] Running N= 4641, theta_crit=0.1... mean=14.645ms (took 0.2s)
[ 5/50] Skipping N=16681, theta_crit=0.1
[ 6/50] Skipping N=59948, theta_crit=0.1
[ 7/50] Skipping N=215443, theta_crit=0.1
[ 8/50] Skipping N=774263, theta_crit=0.1
[ 9/50] Skipping N=2782559, theta_crit=0.1
[10/50] Skipping N=10000000, theta_crit=0.1
[11/50] Running N=  100, theta_crit=0.3... mean=5.393ms (took 0.1s)
[12/50] Running N=  359, theta_crit=0.3... mean=7.384ms (took 0.1s)
[13/50] Running N= 1291, theta_crit=0.3... mean=10.022ms (took 0.1s)
[14/50] Running N= 4641, theta_crit=0.3... mean=13.796ms (took 0.2s)
[15/50] Running N=16681, theta_crit=0.3... mean=27.736ms (took 0.4s)
[16/50] Skipping N=59948, theta_crit=0.3
[17/50] Skipping N=215443, theta_crit=0.3
[18/50] Skipping N=774263, theta_crit=0.3
[19/50] Skipping N=2782559, theta_crit=0.3
[20/50] Skipping N=10000000, theta_crit=0.3
[21/50] Running N=  100, theta_crit=0.5... mean=5.404ms (took 0.1s)
[22/50] Running N=  359, theta_crit=0.5... mean=7.893ms (took 0.1s)
[23/50] Running N= 1291, theta_crit=0.5... mean=10.113ms (took 0.1s)
[24/50] Running N= 4641, theta_crit=0.5... mean=12.846ms (took 0.2s)
[25/50] Running N=16681, theta_crit=0.5... mean=17.362ms (took 0.3s)
[26/50] Skipping N=59948, theta_crit=0.5
[27/50] Skipping N=215443, theta_crit=0.5
[28/50] Skipping N=774263, theta_crit=0.5
[29/50] Skipping N=2782559, theta_crit=0.5
[30/50] Skipping N=10000000, theta_crit=0.5
[31/50] Running N=  100, theta_crit=0.7... mean=5.616ms (took 0.1s)
[32/50] Running N=  359, theta_crit=0.7... mean=7.826ms (took 0.1s)
[33/50] Running N= 1291, theta_crit=0.7... mean=10.131ms (took 0.1s)
[34/50] Running N= 4641, theta_crit=0.7... mean=12.319ms (took 0.2s)
[35/50] Running N=16681, theta_crit=0.7... mean=15.796ms (took 0.3s)
[36/50] Running N=59948, theta_crit=0.7... mean=23.334ms (took 0.6s)
[37/50] Skipping N=215443, theta_crit=0.7
[38/50] Skipping N=774263, theta_crit=0.7
[39/50] Skipping N=2782559, theta_crit=0.7
[40/50] Skipping N=10000000, theta_crit=0.7
[41/50] Running N=  100, theta_crit=0.9... mean=5.679ms (took 0.1s)
[42/50] Running N=  359, theta_crit=0.9... mean=7.731ms (took 0.1s)
[43/50] Running N= 1291, theta_crit=0.9... mean=9.963ms (took 0.1s)
[44/50] Running N= 4641, theta_crit=0.9... mean=12.128ms (took 0.2s)
[45/50] Running N=16681, theta_crit=0.9... mean=15.201ms (took 0.3s)
[46/50] Running N=59948, theta_crit=0.9... mean=20.616ms (took 0.6s)
[47/50] Skipping N=215443, theta_crit=0.9
[48/50] Skipping N=774263, theta_crit=0.9
[49/50] Skipping N=2782559, theta_crit=0.9
[50/50] Skipping N=10000000, theta_crit=0.9
Info: setting dtt implementation to impl : {"implementation":"reference","parameters":{}}  [tree][rank=0]
Running DTT performance benchmarks for reference ordered=False...
Particle counts: [100, 359, 1291, 4641, 16681, 59948, 215443, 774263, 2782559, 10000000]
Theta_crit values: [0.1, 0.3, 0.5, 0.7, 0.9]
Compression level: 4
[ 1/50] Running N=  100, theta_crit=0.1... mean=1.433ms (took 0.0s)
[ 2/50] Running N=  359, theta_crit=0.1... mean=1.462ms (took 0.0s)
[ 3/50] Running N= 1291, theta_crit=0.1... mean=1.612ms (took 0.0s)
[ 4/50] Running N= 4641, theta_crit=0.1... mean=3.142ms (took 0.1s)
[ 5/50] Running N=16681, theta_crit=0.1... mean=23.245ms (took 0.4s)
[ 6/50] Skipping N=59948, theta_crit=0.1
[ 7/50] Skipping N=215443, theta_crit=0.1
[ 8/50] Skipping N=774263, theta_crit=0.1
[ 9/50] Skipping N=2782559, theta_crit=0.1
[10/50] Skipping N=10000000, theta_crit=0.1
[11/50] Running N=  100, theta_crit=0.3... mean=0.883ms (took 0.0s)
[12/50] Running N=  359, theta_crit=0.3... mean=0.948ms (took 0.0s)
[13/50] Running N= 1291, theta_crit=0.3... mean=1.146ms (took 0.0s)
[14/50] Running N= 4641, theta_crit=0.3... mean=2.413ms (took 0.1s)
[15/50] Running N=16681, theta_crit=0.3... mean=8.879ms (took 0.2s)
[16/50] Running N=59948, theta_crit=0.3... mean=51.571ms (took 0.9s)
[17/50] Skipping N=215443, theta_crit=0.3
[18/50] Skipping N=774263, theta_crit=0.3
[19/50] Skipping N=2782559, theta_crit=0.3
[20/50] Skipping N=10000000, theta_crit=0.3
[21/50] Running N=  100, theta_crit=0.5... mean=0.980ms (took 0.0s)
[22/50] Running N=  359, theta_crit=0.5... mean=1.056ms (took 0.0s)
[23/50] Running N= 1291, theta_crit=0.5... mean=1.282ms (took 0.0s)
[24/50] Running N= 4641, theta_crit=0.5... mean=1.664ms (took 0.1s)
[25/50] Running N=16681, theta_crit=0.5... mean=3.610ms (took 0.2s)
[26/50] Running N=59948, theta_crit=0.5... mean=15.120ms (took 0.5s)
[27/50] Running N=215443, theta_crit=0.5... mean=45.666ms (took 1.8s)
[28/50] Skipping N=774263, theta_crit=0.5
[29/50] Skipping N=2782559, theta_crit=0.5
[30/50] Skipping N=10000000, theta_crit=0.5
[31/50] Running N=  100, theta_crit=0.7... mean=1.006ms (took 0.0s)
[32/50] Running N=  359, theta_crit=0.7... mean=1.033ms (took 0.0s)
[33/50] Running N= 1291, theta_crit=0.7... mean=1.074ms (took 0.0s)
[34/50] Running N= 4641, theta_crit=0.7... mean=1.405ms (took 0.1s)
[35/50] Running N=16681, theta_crit=0.7... mean=2.643ms (took 0.1s)
[36/50] Running N=59948, theta_crit=0.7... mean=8.578ms (took 0.4s)
[37/50] Running N=215443, theta_crit=0.7... mean=24.313ms (took 1.6s)
[38/50] Running N=774263, theta_crit=0.7... mean=99.174ms (took 5.8s)
[39/50] Skipping N=2782559, theta_crit=0.7
[40/50] Skipping N=10000000, theta_crit=0.7
[41/50] Running N=  100, theta_crit=0.9... mean=1.018ms (took 0.0s)
[42/50] Running N=  359, theta_crit=0.9... mean=1.047ms (took 0.0s)
[43/50] Running N= 1291, theta_crit=0.9... mean=1.118ms (took 0.0s)
[44/50] Running N= 4641, theta_crit=0.9... mean=1.349ms (took 0.1s)
[45/50] Running N=16681, theta_crit=0.9... mean=2.103ms (took 0.1s)
[46/50] Running N=59948, theta_crit=0.9... mean=5.797ms (took 0.4s)
[47/50] Running N=215443, theta_crit=0.9... mean=17.470ms (took 1.5s)
[48/50] Running N=774263, theta_crit=0.9... mean=59.869ms (took 5.5s)
[49/50] Skipping N=2782559, theta_crit=0.9
[50/50] Skipping N=10000000, theta_crit=0.9
Info: setting dtt implementation to impl : {"implementation":"parallel_select","parameters":{}}  [tree][rank=0]
Running DTT performance benchmarks for parallel_select ordered=False...
Particle counts: [100, 359, 1291, 4641, 16681, 59948, 215443, 774263, 2782559, 10000000]
Theta_crit values: [0.1, 0.3, 0.5, 0.7, 0.9]
Compression level: 4
[ 1/50] Running N=  100, theta_crit=0.1... mean=1.149ms (took 0.0s)
[ 2/50] Running N=  359, theta_crit=0.1... mean=1.198ms (took 0.0s)
[ 3/50] Running N= 1291, theta_crit=0.1... mean=1.778ms (took 0.0s)
[ 4/50] Running N= 4641, theta_crit=0.1... mean=8.896ms (took 0.1s)
[ 5/50] Running N=16681, theta_crit=0.1... mean=95.588ms (took 1.1s)
[ 6/50] Skipping N=59948, theta_crit=0.1
[ 7/50] Skipping N=215443, theta_crit=0.1
[ 8/50] Skipping N=774263, theta_crit=0.1
[ 9/50] Skipping N=2782559, theta_crit=0.1
[10/50] Skipping N=10000000, theta_crit=0.1
[11/50] Running N=  100, theta_crit=0.3... mean=1.206ms (took 0.0s)
[12/50] Running N=  359, theta_crit=0.3... mean=1.216ms (took 0.0s)
[13/50] Running N= 1291, theta_crit=0.3... mean=1.778ms (took 0.0s)
[14/50] Running N= 4641, theta_crit=0.3... mean=7.576ms (took 0.1s)
[15/50] Running N=16681, theta_crit=0.3... mean=51.720ms (took 0.6s)
[16/50] Running N=59948, theta_crit=0.3... mean=346.057ms (took 3.8s)
[17/50] Skipping N=215443, theta_crit=0.3
[18/50] Skipping N=774263, theta_crit=0.3
[19/50] Skipping N=2782559, theta_crit=0.3
[20/50] Skipping N=10000000, theta_crit=0.3
[21/50] Running N=  100, theta_crit=0.5... mean=1.204ms (took 0.0s)
[22/50] Running N=  359, theta_crit=0.5... mean=1.219ms (took 0.0s)
[23/50] Running N= 1291, theta_crit=0.5... mean=1.715ms (took 0.0s)
[24/50] Running N= 4641, theta_crit=0.5... mean=4.447ms (took 0.1s)
[25/50] Running N=16681, theta_crit=0.5... mean=20.904ms (took 0.3s)
[26/50] Running N=59948, theta_crit=0.5... mean=109.637ms (took 1.4s)
[27/50] Running N=215443, theta_crit=0.5... mean=489.248ms (took 6.3s)
[28/50] Skipping N=774263, theta_crit=0.5
[29/50] Skipping N=2782559, theta_crit=0.5
[30/50] Skipping N=10000000, theta_crit=0.5
[31/50] Running N=  100, theta_crit=0.7... mean=1.163ms (took 0.0s)
[32/50] Running N=  359, theta_crit=0.7... mean=1.268ms (took 0.0s)
[33/50] Running N= 1291, theta_crit=0.7... mean=1.544ms (took 0.0s)
[34/50] Running N= 4641, theta_crit=0.7... mean=3.235ms (took 0.1s)
[35/50] Running N=16681, theta_crit=0.7... mean=12.813ms (took 0.2s)
[36/50] Running N=59948, theta_crit=0.7... mean=61.304ms (took 1.0s)
[37/50] Running N=215443, theta_crit=0.7... mean=274.466ms (took 4.2s)
[38/50] Skipping N=774263, theta_crit=0.7
[39/50] Skipping N=2782559, theta_crit=0.7
[40/50] Skipping N=10000000, theta_crit=0.7
[41/50] Running N=  100, theta_crit=0.9... mean=1.172ms (took 0.0s)
[42/50] Running N=  359, theta_crit=0.9... mean=1.206ms (took 0.0s)
[43/50] Running N= 1291, theta_crit=0.9... mean=1.447ms (took 0.0s)
[44/50] Running N= 4641, theta_crit=0.9... mean=2.470ms (took 0.1s)
[45/50] Running N=16681, theta_crit=0.9... mean=7.281ms (took 0.2s)
[46/50] Running N=59948, theta_crit=0.9... mean=30.125ms (took 0.6s)
[47/50] Running N=215443, theta_crit=0.9... mean=124.589ms (took 2.6s)
[48/50] Running N=774263, theta_crit=0.9... mean=549.633ms (took 10.5s)
[49/50] Skipping N=2782559, theta_crit=0.9
[50/50] Skipping N=10000000, theta_crit=0.9
Info: setting dtt implementation to impl : {"implementation":"scan_multipass","parameters":{}}  [tree][rank=0]
Running DTT performance benchmarks for scan_multipass ordered=False...
Particle counts: [100, 359, 1291, 4641, 16681, 59948, 215443, 774263, 2782559, 10000000]
Theta_crit values: [0.1, 0.3, 0.5, 0.7, 0.9]
Compression level: 4
[ 1/50] Running N=  100, theta_crit=0.1... mean=4.185ms (took 0.1s)
[ 2/50] Running N=  359, theta_crit=0.1... mean=6.082ms (took 0.1s)
[ 3/50] Running N= 1291, theta_crit=0.1... mean=8.226ms (took 0.1s)
[ 4/50] Running N= 4641, theta_crit=0.1... mean=10.927ms (took 0.2s)
[ 5/50] Skipping N=16681, theta_crit=0.1
[ 6/50] Skipping N=59948, theta_crit=0.1
[ 7/50] Skipping N=215443, theta_crit=0.1
[ 8/50] Skipping N=774263, theta_crit=0.1
[ 9/50] Skipping N=2782559, theta_crit=0.1
[10/50] Skipping N=10000000, theta_crit=0.1
[11/50] Running N=  100, theta_crit=0.3... mean=4.181ms (took 0.1s)
[12/50] Running N=  359, theta_crit=0.3... mean=6.019ms (took 0.1s)
[13/50] Running N= 1291, theta_crit=0.3... mean=8.111ms (took 0.1s)
[14/50] Running N= 4641, theta_crit=0.3... mean=10.821ms (took 0.2s)
[15/50] Running N=16681, theta_crit=0.3... mean=17.444ms (took 0.3s)
[16/50] Skipping N=59948, theta_crit=0.3
[17/50] Skipping N=215443, theta_crit=0.3
[18/50] Skipping N=774263, theta_crit=0.3
[19/50] Skipping N=2782559, theta_crit=0.3
[20/50] Skipping N=10000000, theta_crit=0.3
[21/50] Running N=  100, theta_crit=0.5... mean=4.221ms (took 0.1s)
[22/50] Running N=  359, theta_crit=0.5... mean=6.343ms (took 0.1s)
[23/50] Running N= 1291, theta_crit=0.5... mean=8.156ms (took 0.1s)
[24/50] Running N= 4641, theta_crit=0.5... mean=9.873ms (took 0.1s)
[25/50] Running N=16681, theta_crit=0.5... mean=13.362ms (took 0.3s)
[26/50] Skipping N=59948, theta_crit=0.5
[27/50] Skipping N=215443, theta_crit=0.5
[28/50] Skipping N=774263, theta_crit=0.5
[29/50] Skipping N=2782559, theta_crit=0.5
[30/50] Skipping N=10000000, theta_crit=0.5
[31/50] Running N=  100, theta_crit=0.7... mean=4.164ms (took 0.1s)
[32/50] Running N=  359, theta_crit=0.7... mean=6.077ms (took 0.1s)
[33/50] Running N= 1291, theta_crit=0.7... mean=8.162ms (took 0.1s)
[34/50] Running N= 4641, theta_crit=0.7... mean=10.075ms (took 0.1s)
[35/50] Running N=16681, theta_crit=0.7... mean=12.873ms (took 0.2s)
[36/50] Running N=59948, theta_crit=0.7... mean=19.180ms (took 0.5s)
[37/50] Skipping N=215443, theta_crit=0.7
[38/50] Skipping N=774263, theta_crit=0.7
[39/50] Skipping N=2782559, theta_crit=0.7
[40/50] Skipping N=10000000, theta_crit=0.7
[41/50] Running N=  100, theta_crit=0.9... mean=4.221ms (took 0.1s)
[42/50] Running N=  359, theta_crit=0.9... mean=6.100ms (took 0.1s)
[43/50] Running N= 1291, theta_crit=0.9... mean=8.155ms (took 0.1s)
[44/50] Running N= 4641, theta_crit=0.9... mean=9.861ms (took 0.1s)
[45/50] Running N=16681, theta_crit=0.9... mean=12.869ms (took 0.3s)
[46/50] Running N=59948, theta_crit=0.9... mean=17.680ms (took 0.5s)
[47/50] Skipping N=215443, theta_crit=0.9
[48/50] Skipping N=774263, theta_crit=0.9
[49/50] Skipping N=2782559, theta_crit=0.9
[50/50] Skipping N=10000000, theta_crit=0.9

Plot the performance benchmarks for all implementations

 dump_folder = "_to_trash"

 import os

 # Create the dump directory if it does not exist
 if shamrock.sys.world_rank() == 0:
     os.makedirs(dump_folder, exist_ok=True)

 ref_key = "reference ordered=False"
 largest_refalg_value = np.nanmax(results[ref_key]["results_min"])

 i = 0
 # iterate over the results
 for k, v in results.items():
     # Get the results for this algorithm
     particle_counts = v["particle_counts"]
     theta_crits = v["theta_crits"]
     results_min = v["results_min"]
     results_max_mem_delta = v["results_max_mem_delta"]

     # Get reference algorithm results for comparison
     reference_min = results[ref_key]["results_min"]

     # Create and display the plot
     fig, ax = create_checkerboard_plot(
         particle_counts,
         theta_crits,
         results_min,
         compression_level,
         v["name"],
         largest_refalg_value,
         reference_min,
         results_max_mem_delta,
     )

     plt.savefig(f"{dump_folder}/benchmark-dtt-performance-{i}.pdf")
     i += 1

 plt.show()