From 233169d9ab5fb8a796d91a3694f8a522fa54d194 Mon Sep 17 00:00:00 2001 From: Sebastiano Tronto Date: Mon, 26 Jan 2026 22:52:15 +0100 Subject: Improve multithreading benchmarks (compare speedup factor, not with vcube) --- benchmarks/plot-benchmarks.py | 226 +++++++++++++++++++++++------------------- 1 file changed, 123 insertions(+), 103 deletions(-) (limited to 'benchmarks/plot-benchmarks.py') diff --git a/benchmarks/plot-benchmarks.py b/benchmarks/plot-benchmarks.py index 46031c6..027e0fe 100644 --- a/benchmarks/plot-benchmarks.py +++ b/benchmarks/plot-benchmarks.py @@ -1,6 +1,14 @@ import matplotlib.pyplot as plt import results_h48 import results_vcube +from pathlib import Path + +benchmarks_dir = Path("benchmarks") +benchmarks_img_dir = benchmarks_dir / "img" +benchmarks_single_thread = benchmarks_dir / "tables_1_thread.md" +benchmarks_4_threads = benchmarks_dir / "tables_4_threads.md" +benchmarks_16_threads = benchmarks_dir / "tables_16_threads.md" +benchmarks_all_solutions = benchmarks_dir / "tables_all_solutions.md" # Table sizes in bytes @@ -23,7 +31,7 @@ sizes_vcube = { # Printing tables in markdown format -def print_row(solver_name, solver_size, dict, mul_by_size, superflip_star): +def print_row(f, solver_name, solver_size, dict, mul_by_size, superflip_star): if dict is None: return solver_gib = solver_size / (2**30) @@ -36,104 +44,106 @@ def print_row(solver_name, solver_size, dict, mul_by_size, superflip_star): f"{dict["superflip"]*m:>9.2f}" if "superflip" in dict else s ] sep = "|" - print(sep + sep.join(cols) + sep) + f.write(sep + sep.join(cols) + sep + "\n") -def print_table(h48, vcube, ms, st): +def print_table(f, h48, vcube, ms, st): vcube = vcube or {} - print("| Solver | Size |17 moves|18 moves|19 moves|20 moves|Superflip|") - print("|:---------|:-------|-------:|-------:|-------:|-------:|--------:|") - print_row("vcube 212", sizes_vcube[212], vcube.get(212), ms, not st) - print_row("H48 h11", sizes_h48[11], h48[11], ms, False) - print_row("vcube 404", sizes_vcube[404], vcube.get(404), ms, not st) - print_row("H48 h10", sizes_h48[10], h48[10], ms, False) - print_row("vcube 308", sizes_vcube[308], vcube.get(308), ms, not st) - print_row("H48 h9", sizes_h48[9], h48[9], ms, False) - print_row("vcube 208", sizes_vcube[208], vcube.get(208), ms, not st) - print_row("H48 h8", sizes_h48[8], h48[8], ms, False) - print_row("H48 h7", sizes_h48[7], h48[7], ms, False) - print_row("vcube 112", sizes_vcube[112], vcube.get(112), ms, not st) - print_row("H48 h6", sizes_h48[6], h48[6], ms, False) - -print("
Single solution, single thread") -print() -print("Time per cube (in seconds, lower is better).") -print() -print_table(results_h48.h48_single_thread, results_vcube.vcube_single_thread, False, True) -print() -print("Time per cube adjusted for table size (in seconds \\* GiB, lower is better).") -print() -print_table(results_h48.h48_single_thread, results_vcube.vcube_single_thread, True, True) -print() -print( - '\n' - '\n' - '\n' - '' -) -print("
") - -print() -print("
Single solution, 4 threads") -print() -print("Time per cube (in seconds, lower is better).") -print() -print_table(results_h48.h48_4_threads, results_vcube.vcube_4_threads, False, False) -print() -print("Time per cube adjusted for table size (in seconds \\* GiB, lower is better).") -print() -print_table(results_h48.h48_4_threads, results_vcube.vcube_4_threads, True, False) -print() -print("(a) vcube cannot parallelize on a single scramble, the results for the") -print("superflip are going to be the same as in the single thread case.") -print() -print( - '\n' - '\n' - '\n' - '' -) -print("
") - -print() -print("
Single solution, 16 threads") -print() -print("Time per cube (in seconds, lower is better).") -print() -print_table(results_h48.h48_16_threads, results_vcube.vcube_16_threads, False, False) -print() -print("Time per cube adjusted for table size (in seconds \\* GiB, lower is better).") -print() -print_table(results_h48.h48_16_threads, results_vcube.vcube_16_threads, True, False) -print() -print("(a) vcube cannot parallelize on a single scramble, the results for the") -print("superflip are going to be the same as in the single thread case.") -print() -print( - '\n' - '\n' - '\n' - '' -) -print("
") - -print() -print("
All solutions, 16 threads") -print() -print("*Note: vcube does not have an option for finding multiple solutions.*") -print() -print("Time per cube (in seconds, lower is better).") -print() -print_table(results_h48.h48_all_solutions, None, False, False) -print() -print("Time per cube adjusted for table size (in seconds \\* GiB, lower is better).") -print() -print_table(results_h48.h48_all_solutions, None, True, False) -print() -print("
") + f.write("| Solver | Size |17 moves|18 moves|19 moves|20 moves|Superflip|\n") + f.write("|:---------|:-------|-------:|-------:|-------:|-------:|--------:|\n") + print_row(f, "vcube 212", sizes_vcube[212], vcube.get(212), ms, not st) + print_row(f, "H48 h11", sizes_h48[11], h48[11], ms, False) + print_row(f, "vcube 404", sizes_vcube[404], vcube.get(404), ms, not st) + print_row(f, "H48 h10", sizes_h48[10], h48[10], ms, False) + print_row(f, "vcube 308", sizes_vcube[308], vcube.get(308), ms, not st) + print_row(f, "H48 h9", sizes_h48[9], h48[9], ms, False) + print_row(f, "vcube 208", sizes_vcube[208], vcube.get(208), ms, not st) + print_row(f, "H48 h8", sizes_h48[8], h48[8], ms, False) + print_row(f, "H48 h7", sizes_h48[7], h48[7], ms, False) + print_row(f, "vcube 112", sizes_vcube[112], vcube.get(112), ms, not st) + print_row(f, "H48 h6", sizes_h48[6], h48[6], ms, False) + +def print_factor_table(f, slow, fast): + ratio = {} + for m in [6, 7, 8, 9, 10, 11]: + ratio[m] = {} + for k in slow[m]: + ratio[m][k] = (slow[m][k] / fast[m][k]) * (25 if k != "superflip" else 1) + f.write("| Solver | Size |17 moves|18 moves|19 moves|20 moves|Superflip|\n") + f.write("|:---------|:-------|-------:|-------:|-------:|-------:|--------:|\n") + print_row(f, "H48 h11", sizes_h48[11], ratio[11], False, False) + print_row(f, "H48 h10", sizes_h48[10], ratio[10], False, False) + print_row(f, "H48 h9", sizes_h48[9], ratio[9], False, False) + print_row(f, "H48 h8", sizes_h48[8], ratio[8], False, False) + print_row(f, "H48 h7", sizes_h48[7], ratio[7], False, False) + print_row(f, "H48 h6", sizes_h48[6], ratio[6], False, False) + +with open(benchmarks_single_thread, "w") as f: + f.write(f"\n") + f.write("
Results: Single solution, single thread\n") + f.write("\n") + f.write("Time per cube (in seconds, lower is better).\n") + f.write("\n") + print_table(f, results_h48.h48_single_thread, results_vcube.vcube_single_thread, False, True) + f.write("\n") + f.write("Time per cube adjusted for table size (in seconds \\* GiB, lower is better).\n") + f.write("\n") + print_table(f, results_h48.h48_single_thread, results_vcube.vcube_single_thread, True, True) + f.write("\n") + f.write( + '\n' + '\n' + '\n' + '\n' + ) + f.write("
\n") + +with open(benchmarks_4_threads, "w") as f: + f.write(f"\n") + f.write("
Results: Single solution, 4 threads\n") + f.write("\n") + f.write("Time per cube (in seconds, lower is better).\n") + f.write("\n") + print_table(f, results_h48.h48_4_threads, None, False, False) + f.write("\n") + f.write("Speed-up factor (higher is better).\n") + f.write("\n") + print_factor_table(f, results_h48.h48_single_thread, results_h48.h48_4_threads) + f.write('\n') + f.write("\n") + f.write("
\n") + +with open(benchmarks_16_threads, "w") as f: + f.write(f"\n") + f.write("
Results: Single solution, 16 threads\n") + f.write("\n") + f.write("Time per cube (in seconds, lower is better).\n") + f.write("\n") + print_table(f, results_h48.h48_16_threads, None, False, False) + f.write("\n") + f.write("Speed-up factor (higher is better).\n") + f.write("\n") + print_factor_table(f, results_h48.h48_single_thread, results_h48.h48_16_threads) + f.write('\n') + f.write("\n") + f.write("
\n") + +with open(benchmarks_all_solutions, "w") as f: + f.write(f"\n") + f.write("
Results: All solutions, 16 threads\n") + f.write("\n") + f.write("Time per cube (in seconds, lower is better).\n") + f.write("\n") + print_table(f, results_h48.h48_all_solutions, None, False, False) + f.write("\n") + f.write("Time per cube adjusted for table size (in seconds \\* GiB, lower is better).\n") + f.write("\n") + print_table(f, results_h48.h48_all_solutions, None, True, False) + f.write("\n") + f.write("
\n") # Plotting -def plot(title, hd, vd, key): +def plot_comparison(title, hd, vd, key): d = 1 if key == "superflip" else 25 h48x = [sizes_h48[m]/(2**30) for m in hd.keys() if key in hd[m]] vcubex = [sizes_vcube[m]/(2**30) for m in vd.keys() if key in vd[m]] @@ -148,17 +158,27 @@ def plot(title, hd, vd, key): plt.plot(vcubex, vcubey, "o--", label = "vcube") plt.legend(loc = "right") filename = title.replace(" ", "").replace(",", "") + ".png" - plt.savefig("benchmarks/img/" + filename, dpi=300) + plt.savefig(benchmarks_img_dir / filename, dpi=300) #plt.show() rh, rv = results_h48.h48_single_thread, results_vcube.vcube_single_thread for m in [17, 18, 19, 20]: - plot(f"{m} moves 1 thread", rh, rv, m) - -rh, rv = results_h48.h48_4_threads, results_vcube.vcube_4_threads -for m in [17, 18, 19, 20]: - plot(f"{m} moves 4 threads", rh, rv, m) + plot_comparison(f"{m} moves 1 thread", rh, rv, m) -rh, rv = results_h48.h48_16_threads, results_vcube.vcube_16_threads -for m in [17, 18, 19, 20]: - plot(f"{m} moves 16 threads", rh, rv, m) +def plot_multithread_scatter(title, slow, fast): + plt.clf() + plt.title(title) + plt.xlabel("Moves") + plt.ylabel("Speed-up factor") + x = [17, 18, 19] + plt.xticks(x) + for h in [11, 10, 9, 8, 7, 6]: + y = [slow[h][i] / fast[h][i] for i in x] + plt.scatter(x, y, label=f"H48 h{h}") + plt.legend(loc = "right") + filename = title.replace(" ", "").replace(",", "") + ".png" + plt.savefig(benchmarks_img_dir / filename, dpi=300) + plt.show() + +plot_multithread_scatter("4 threads speedup factor", results_h48.h48_single_thread, results_h48.h48_4_threads) +plot_multithread_scatter("16 threads speedup factor", results_h48.h48_single_thread, results_h48.h48_16_threads) -- cgit v1.3