plot-benchmarks.py (7146B)
1 import matplotlib.pyplot as plt 2 import results_h48 3 import results_vcube 4 from pathlib import Path 5 6 benchmarks_dir = Path("benchmarks") 7 benchmarks_img_dir = benchmarks_dir / "img" 8 benchmarks_single_thread = benchmarks_dir / "tables_1_thread.md" 9 benchmarks_4_threads = benchmarks_dir / "tables_4_threads.md" 10 benchmarks_16_threads = benchmarks_dir / "tables_16_threads.md" 11 benchmarks_all_solutions = benchmarks_dir / "tables_all_solutions.md" 12 13 # Table sizes in bytes 14 15 sizes_h48 = { 16 6: 1897951528, 17 7: 3793842344, 18 8: 7585624040, 19 9: 15169187432, 20 10: 30336314216, 21 11: 60670567784, 22 } 23 24 sizes_vcube = { 25 112: 2603089920, 26 208: 7809269760, 27 308: 22777036800, 28 404: 34165555200, 29 212: 62474158080, 30 } 31 32 # Printing tables in markdown format 33 34 def print_row(f, solver_name, solver_size, dict, mul_by_size, superflip_star): 35 if dict is None: 36 return 37 solver_gib = solver_size / (2**30) 38 m = solver_gib if mul_by_size else 1 39 s = " (a) " if superflip_star else " " 40 cols = [f"{solver_name: <10}", f"{solver_gib:>4.1f} GiB"] + [ 41 f"{dict[key]*m/25:>8.2f}" if key in dict else " " 42 for key in [17, 18, 19, 20] 43 ] + [ 44 f"{dict["superflip"]*m:>9.2f}" if "superflip" in dict else s 45 ] 46 sep = "|" 47 f.write(sep + sep.join(cols) + sep + "\n") 48 49 def print_table(f, h48, vcube, ms, st): 50 vcube = vcube or {} 51 f.write("| Solver | Size |17 moves|18 moves|19 moves|20 moves|Superflip|\n") 52 f.write("|:---------|:-------|-------:|-------:|-------:|-------:|--------:|\n") 53 print_row(f, "vcube 212", sizes_vcube[212], vcube.get(212), ms, not st) 54 print_row(f, "H48 h11", sizes_h48[11], h48[11], ms, False) 55 print_row(f, "vcube 404", sizes_vcube[404], vcube.get(404), ms, not st) 56 print_row(f, "H48 h10", sizes_h48[10], h48[10], ms, False) 57 print_row(f, "vcube 308", sizes_vcube[308], vcube.get(308), ms, not st) 58 print_row(f, "H48 h9", sizes_h48[9], h48[9], ms, False) 59 print_row(f, "vcube 208", sizes_vcube[208], vcube.get(208), ms, not st) 60 print_row(f, "H48 h8", sizes_h48[8], h48[8], ms, False) 61 print_row(f, "H48 h7", sizes_h48[7], h48[7], ms, False) 62 print_row(f, "vcube 112", sizes_vcube[112], vcube.get(112), ms, not st) 63 print_row(f, "H48 h6", sizes_h48[6], h48[6], ms, False) 64 65 def print_factor_table(f, slow, fast): 66 ratio = {} 67 for m in [6, 7, 8, 9, 10, 11]: 68 ratio[m] = {} 69 for k in slow[m]: 70 ratio[m][k] = (slow[m][k] / fast[m][k]) * (25 if k != "superflip" else 1) 71 f.write("| Solver | Size |17 moves|18 moves|19 moves|20 moves|Superflip|\n") 72 f.write("|:---------|:-------|-------:|-------:|-------:|-------:|--------:|\n") 73 print_row(f, "H48 h11", sizes_h48[11], ratio[11], False, False) 74 print_row(f, "H48 h10", sizes_h48[10], ratio[10], False, False) 75 print_row(f, "H48 h9", sizes_h48[9], ratio[9], False, False) 76 print_row(f, "H48 h8", sizes_h48[8], ratio[8], False, False) 77 print_row(f, "H48 h7", sizes_h48[7], ratio[7], False, False) 78 print_row(f, "H48 h6", sizes_h48[6], ratio[6], False, False) 79 80 with open(benchmarks_single_thread, "w") as f: 81 f.write(f"<!-- The following details block can be found in {benchmarks_single_thread} -->\n") 82 f.write("<details><summary>Results: Single solution, single thread</summary>\n") 83 f.write("\n") 84 f.write("Time per cube (in seconds, lower is better).\n") 85 f.write("\n") 86 print_table(f, results_h48.h48_single_thread, results_vcube.vcube_single_thread, False, True) 87 f.write("\n") 88 f.write("Time per cube adjusted for table size (in seconds \\* GiB, lower is better).\n") 89 f.write("\n") 90 print_table(f, results_h48.h48_single_thread, results_vcube.vcube_single_thread, True, True) 91 f.write("\n") 92 f.write( 93 '<img src="img/17moves1thread.png">\n' 94 '<img src="img/18moves1thread.png">\n' 95 '<img src="img/19moves1thread.png">\n' 96 '<img src="img/20moves1thread.png">\n' 97 ) 98 f.write("</details>\n") 99 100 with open(benchmarks_4_threads, "w") as f: 101 f.write(f"<!-- The following details block can be found in {benchmarks_4_threads} -->\n") 102 f.write("<details><summary>Results: Single solution, 4 threads</summary>\n") 103 f.write("\n") 104 f.write("Time per cube (in seconds, lower is better).\n") 105 f.write("\n") 106 print_table(f, results_h48.h48_4_threads, None, False, False) 107 f.write("\n") 108 f.write("Speed-up factor (higher is better).\n") 109 f.write("\n") 110 print_factor_table(f, results_h48.h48_single_thread, results_h48.h48_4_threads) 111 f.write("\n") 112 f.write('<img src="img/4threadsspeedupfactor.png">\n') 113 f.write("\n") 114 f.write("</details>\n") 115 116 with open(benchmarks_16_threads, "w") as f: 117 f.write(f"<!-- The following details block can be found in {benchmarks_16_threads} -->\n") 118 f.write("<details><summary>Results: Single solution, 16 threads</summary>\n") 119 f.write("\n") 120 f.write("Time per cube (in seconds, lower is better).\n") 121 f.write("\n") 122 print_table(f, results_h48.h48_16_threads, None, False, False) 123 f.write("\n") 124 f.write("Speed-up factor (higher is better).\n") 125 f.write("\n") 126 print_factor_table(f, results_h48.h48_single_thread, results_h48.h48_16_threads) 127 f.write("\n") 128 f.write('<img src="img/16threadsspeedupfactor.png">\n') 129 f.write("\n") 130 f.write("</details>\n") 131 132 with open(benchmarks_all_solutions, "w") as f: 133 f.write(f"<!-- The following details block can be found in {benchmarks_all_solutions} -->\n") 134 f.write("<details><summary>Results: All solutions, 16 threads</summary>\n") 135 f.write("\n") 136 f.write("Time per cube (in seconds, lower is better).\n") 137 f.write("\n") 138 print_table(f, results_h48.h48_all_solutions, None, False, False) 139 f.write("\n") 140 f.write("Time per cube adjusted for table size (in seconds \\* GiB, lower is better).\n") 141 f.write("\n") 142 print_table(f, results_h48.h48_all_solutions, None, True, False) 143 f.write("\n") 144 f.write("</details>\n") 145 146 # Plotting 147 148 def plot_comparison(title, hd, vd, key): 149 d = 1 if key == "superflip" else 25 150 h48x = [sizes_h48[m]/(2**30) for m in hd.keys() if key in hd[m]] 151 vcubex = [sizes_vcube[m]/(2**30) for m in vd.keys() if key in vd[m]] 152 h48y = [d[key] for _, d in hd.items() if key in d] 153 vcubey = [d[key] for _, d in vd.items() if key in d] 154 155 plt.clf() 156 plt.title(title) 157 plt.xlabel("Table size (GiB)") 158 plt.ylabel("Time to solve (s / cube)") 159 plt.plot(h48x, h48y, "o--", label = "H48") 160 plt.plot(vcubex, vcubey, "o--", label = "vcube") 161 plt.legend(loc = "right") 162 filename = title.replace(" ", "").replace(",", "") + ".png" 163 plt.savefig(benchmarks_img_dir / filename, dpi=300) 164 #plt.show() 165 166 rh, rv = results_h48.h48_single_thread, results_vcube.vcube_single_thread 167 for m in [17, 18, 19, 20]: 168 plot_comparison(f"{m} moves 1 thread", rh, rv, m) 169 170 def plot_multithread_scatter(title, slow, fast): 171 plt.clf() 172 plt.title(title) 173 plt.xlabel("Moves") 174 plt.ylabel("Speed-up factor") 175 x = [17, 18, 19] 176 plt.xticks(x) 177 for h in [11, 10, 9, 8, 7, 6]: 178 y = [slow[h][i] / fast[h][i] for i in x] 179 plt.scatter(x, y, label=f"H48 h{h}") 180 plt.legend(loc = "right") 181 filename = title.replace(" ", "").replace(",", "") + ".png" 182 plt.savefig(benchmarks_img_dir / filename, dpi=300) 183 #plt.show() 184 185 plot_multithread_scatter("4 threads speedup factor", results_h48.h48_single_thread, results_h48.h48_4_threads) 186 plot_multithread_scatter("16 threads speedup factor", results_h48.h48_single_thread, results_h48.h48_16_threads)