121 lines
6.1 KiB
Python
121 lines
6.1 KiB
Python
"""Compare the logarithmic-space grid against the PUBLISHED x-space grid, under the
|
|
identical protocol (both sweeps use sweep_full.evaluate, same XTOL/MAXFEV/ACC, failures
|
|
charged the full budget, the 2x bound counted per run).
|
|
|
|
python -u compare_log_vs_x.py sweep_full.json sweep_log.json
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
|
|
|
|
def best(rows, key, kind):
|
|
sel = [r for r in rows if (r["spec"][0] == "pure") == (kind != "base")]
|
|
if kind == "base":
|
|
sel = [r for r in rows if r["spec"][0] == "base"]
|
|
if not sel:
|
|
return None
|
|
return min(sel, key=lambda r: (r[key]["n"] - r[key]["solved"], r[key]["total"]))
|
|
|
|
|
|
def two_set(r):
|
|
return r["APS"]["total"] + r["CHA"]["total"], r["APS"]["solved"] + r["CHA"]["solved"]
|
|
|
|
|
|
def per_set_best(rows, setname):
|
|
"""The PAPER'S ranking rule, applied within one set: fewest unsolved, then fewest
|
|
evaluations. Ranking across the two sets by (solved, cost) is NOT the paper's rule
|
|
and picks a config that buys one extra solve for thousands of evaluations — that
|
|
mistake produced a misleading 'x-space best' on 2026-09-20."""
|
|
return min(rows, key=lambda r: (r[setname]["n"] - r[setname]["solved"], r[setname]["total"]))
|
|
|
|
|
|
def main(x_path, log_path):
|
|
xs = json.load(open(x_path, encoding="utf8"))
|
|
lg = json.load(open(log_path, encoding="utf8"))
|
|
print(f"{'grid':>10} {'set':>4} {'configuration':<34} {'solved':>9} {'total fev':>10} {'over2x':>7}")
|
|
for name, data in (("x-space", xs), ("log-space", lg)):
|
|
pure = [r for r in data if r["spec"][0] == "pure"]
|
|
if not pure:
|
|
continue
|
|
for setname in ("APS", "CHA"):
|
|
b = per_set_best(pure, setname)
|
|
print(f"{name:>10} {setname:>4} {b['label']:<34} "
|
|
f"{b[setname]['solved']}/{b[setname]['n']:>3} {b[setname]['total']:10d} "
|
|
f"{b[setname]['over']:7d}")
|
|
# the two-set figure the prereg's 6026 target refers to: the best config by total
|
|
# among those that solve the most problems in EACH set at the paper's own optimum
|
|
print()
|
|
rows = []
|
|
for name, data in (("x-space", xs), ("log-space", lg)):
|
|
pure = [r for r in data if r["spec"][0] == "pure"]
|
|
if not pure:
|
|
continue
|
|
b = min(pure, key=lambda r: (r["APS"]["n"] - r["APS"]["solved"]
|
|
+ r["CHA"]["n"] - r["CHA"]["solved"],
|
|
r["APS"]["total"] + r["CHA"]["total"]))
|
|
tot, solved = two_set(b)
|
|
over = b["APS"]["over"] + b["CHA"]["over"]
|
|
print(f"{name:>10} two-set best: {b['label']:<30} solved {solved}/199 "
|
|
f"APS {b['APS']['total']:6d} CHA {b['CHA']['total']:6d} total {tot:6d} over {over}")
|
|
rows.append((name, b, tot, solved, over))
|
|
# the baselines the targets are stated against
|
|
for nm in ("itp", "brent", "toms748", "chandrupatla", "bisection"):
|
|
cand = [r for r in xs if r["spec"][0] == "base" and r["label"].startswith(nm)]
|
|
if not cand:
|
|
continue
|
|
b = min(cand, key=lambda r: (399 - (r["APS"]["solved"] + r["CHA"]["solved"]),
|
|
r["APS"]["total"] + r["CHA"]["total"]))
|
|
tot, solved = two_set(b)
|
|
print(f"{'baseline':>10} {b['label']:<34} "
|
|
f"{b['APS']['solved']}/{b['APS']['n']} {b['APS']['total']:7d} "
|
|
f"{b['CHA']['solved']}/{b['CHA']['n']} {b['CHA']['total']:7d} "
|
|
f"{solved}/399 {tot:8d} {b['APS']['over'] + b['CHA']['over']:7d}")
|
|
|
|
# The prereg's 6026 target is stated WITH the guarantee intact, so the comparison must
|
|
# stay inside the guarantee-bearing guards; and each verdict compares LIKE WITH LIKE
|
|
# (the same set, the same guard family) — comparing two different configurations gave a
|
|
# false PASS on 2026-09-20.
|
|
GUAR = ("credit", "project", "safe")
|
|
print("\nguarantee-bearing guards only (credit/project/safe):")
|
|
bests = {}
|
|
for name, data in (("x-space", xs), ("log-space", lg)):
|
|
pure = [r for r in data if r["spec"][0] == "pure" and r["spec"][3] in GUAR]
|
|
if not pure:
|
|
continue
|
|
b2 = min(pure, key=lambda r: (r["APS"]["n"] - r["APS"]["solved"]
|
|
+ r["CHA"]["n"] - r["CHA"]["solved"],
|
|
r["APS"]["total"] + r["CHA"]["total"]))
|
|
tot, solved = two_set(b2)
|
|
bests[name] = (b2, tot, solved)
|
|
print(f" {name:>9}: {b2['label']:<32} solved {solved}/199 "
|
|
f"APS {b2['APS']['total']:6d} CHA {b2['CHA']['total']:6d} total {tot:6d} "
|
|
f"over {b2['APS']['over'] + b2['CHA']['over']}")
|
|
print(f" {'':>9} per-set APS best {per_set_best(pure, 'APS')['label']:<28} "
|
|
f"{per_set_best(pure, 'APS')['APS']['total']:6d} | "
|
|
f"CHA best {per_set_best(pure, 'CHA')['label']:<28} "
|
|
f"{per_set_best(pure, 'CHA')['CHA']['total']:6d}")
|
|
|
|
if len(bests) == 2:
|
|
bx, tx, _sx = bests["x-space"]
|
|
bl, tl, sl = bests["log-space"]
|
|
apx = per_set_best([r for r in xs if r["spec"][0] == "pure" and r["spec"][3] in GUAR], "APS")
|
|
apl = per_set_best([r for r in lg if r["spec"][0] == "pure" and r["spec"][3] in GUAR], "APS")
|
|
chl = per_set_best([r for r in lg if r["spec"][0] == "pure" and r["spec"][3] in GUAR], "CHA")
|
|
print("\nTARGETS (PREREG): two sets < 6026, CHA <= 2857, solved >= 193/199, guarantee intact")
|
|
print(f" VERDICT G2' (APS must not grow, per-set best vs per-set best): "
|
|
f"{'PASS' if apl['APS']['total'] <= apx['APS']['total'] else 'FAIL'} "
|
|
f"(log {apl['APS']['total']} vs x {apx['APS']['total']})")
|
|
print(f" VERDICT L3 (CHA <= 2857): {'PASS' if chl['CHA']['total'] <= 2857 else 'FAIL'} "
|
|
f"(log {chl['CHA']['total']})")
|
|
print(f" VERDICT two sets < 6026: {'PASS' if tl < 6026 else 'FAIL'} "
|
|
f"(log {tl}, x {tx})")
|
|
print(f" VERDICT solved >= 193: {'PASS' if sl >= 193 else 'FAIL'} ({sl}/199)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1] if len(sys.argv) > 1 else "sweep_full.json",
|
|
sys.argv[2] if len(sys.argv) > 2 else "sweep_log.json")
|