logarithmic march MEASURED and REFUTED: APS 6940 vs the published 2806 (2.5x worse), CHA 3115 vs the 2857 target, two sets 10414 vs <6026; the mechanism did work in the predicted direction (CHA 4516 -> 3115 = 1.45x) but the canonical set pays for it — the same trade that ended the four previous attempts, now measured for the transform too. Guarantee intact everywhere (over2x = 0)
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,120 @@
|
||||
"""Compare the logarithmic-space grid against the PUBLISHED x-space grid, under the
|
||||
identical protocol (both sweeps use sweep_full.evaluate, same XTOL/MAXFEV/ACC, failures
|
||||
charged the full budget, the 2x bound counted per run).
|
||||
|
||||
python -u compare_log_vs_x.py sweep_full.json sweep_log.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def best(rows, key, kind):
|
||||
sel = [r for r in rows if (r["spec"][0] == "pure") == (kind != "base")]
|
||||
if kind == "base":
|
||||
sel = [r for r in rows if r["spec"][0] == "base"]
|
||||
if not sel:
|
||||
return None
|
||||
return min(sel, key=lambda r: (r[key]["n"] - r[key]["solved"], r[key]["total"]))
|
||||
|
||||
|
||||
def two_set(r):
|
||||
return r["APS"]["total"] + r["CHA"]["total"], r["APS"]["solved"] + r["CHA"]["solved"]
|
||||
|
||||
|
||||
def per_set_best(rows, setname):
|
||||
"""The PAPER'S ranking rule, applied within one set: fewest unsolved, then fewest
|
||||
evaluations. Ranking across the two sets by (solved, cost) is NOT the paper's rule
|
||||
and picks a config that buys one extra solve for thousands of evaluations — that
|
||||
mistake produced a misleading 'x-space best' on 2026-09-20."""
|
||||
return min(rows, key=lambda r: (r[setname]["n"] - r[setname]["solved"], r[setname]["total"]))
|
||||
|
||||
|
||||
def main(x_path, log_path):
|
||||
xs = json.load(open(x_path, encoding="utf8"))
|
||||
lg = json.load(open(log_path, encoding="utf8"))
|
||||
print(f"{'grid':>10} {'set':>4} {'configuration':<34} {'solved':>9} {'total fev':>10} {'over2x':>7}")
|
||||
for name, data in (("x-space", xs), ("log-space", lg)):
|
||||
pure = [r for r in data if r["spec"][0] == "pure"]
|
||||
if not pure:
|
||||
continue
|
||||
for setname in ("APS", "CHA"):
|
||||
b = per_set_best(pure, setname)
|
||||
print(f"{name:>10} {setname:>4} {b['label']:<34} "
|
||||
f"{b[setname]['solved']}/{b[setname]['n']:>3} {b[setname]['total']:10d} "
|
||||
f"{b[setname]['over']:7d}")
|
||||
# the two-set figure the prereg's 6026 target refers to: the best config by total
|
||||
# among those that solve the most problems in EACH set at the paper's own optimum
|
||||
print()
|
||||
rows = []
|
||||
for name, data in (("x-space", xs), ("log-space", lg)):
|
||||
pure = [r for r in data if r["spec"][0] == "pure"]
|
||||
if not pure:
|
||||
continue
|
||||
b = min(pure, key=lambda r: (r["APS"]["n"] - r["APS"]["solved"]
|
||||
+ r["CHA"]["n"] - r["CHA"]["solved"],
|
||||
r["APS"]["total"] + r["CHA"]["total"]))
|
||||
tot, solved = two_set(b)
|
||||
over = b["APS"]["over"] + b["CHA"]["over"]
|
||||
print(f"{name:>10} two-set best: {b['label']:<30} solved {solved}/199 "
|
||||
f"APS {b['APS']['total']:6d} CHA {b['CHA']['total']:6d} total {tot:6d} over {over}")
|
||||
rows.append((name, b, tot, solved, over))
|
||||
# the baselines the targets are stated against
|
||||
for nm in ("itp", "brent", "toms748", "chandrupatla", "bisection"):
|
||||
cand = [r for r in xs if r["spec"][0] == "base" and r["label"].startswith(nm)]
|
||||
if not cand:
|
||||
continue
|
||||
b = min(cand, key=lambda r: (399 - (r["APS"]["solved"] + r["CHA"]["solved"]),
|
||||
r["APS"]["total"] + r["CHA"]["total"]))
|
||||
tot, solved = two_set(b)
|
||||
print(f"{'baseline':>10} {b['label']:<34} "
|
||||
f"{b['APS']['solved']}/{b['APS']['n']} {b['APS']['total']:7d} "
|
||||
f"{b['CHA']['solved']}/{b['CHA']['n']} {b['CHA']['total']:7d} "
|
||||
f"{solved}/399 {tot:8d} {b['APS']['over'] + b['CHA']['over']:7d}")
|
||||
|
||||
# The prereg's 6026 target is stated WITH the guarantee intact, so the comparison must
|
||||
# stay inside the guarantee-bearing guards; and each verdict compares LIKE WITH LIKE
|
||||
# (the same set, the same guard family) — comparing two different configurations gave a
|
||||
# false PASS on 2026-09-20.
|
||||
GUAR = ("credit", "project", "safe")
|
||||
print("\nguarantee-bearing guards only (credit/project/safe):")
|
||||
bests = {}
|
||||
for name, data in (("x-space", xs), ("log-space", lg)):
|
||||
pure = [r for r in data if r["spec"][0] == "pure" and r["spec"][3] in GUAR]
|
||||
if not pure:
|
||||
continue
|
||||
b2 = min(pure, key=lambda r: (r["APS"]["n"] - r["APS"]["solved"]
|
||||
+ r["CHA"]["n"] - r["CHA"]["solved"],
|
||||
r["APS"]["total"] + r["CHA"]["total"]))
|
||||
tot, solved = two_set(b2)
|
||||
bests[name] = (b2, tot, solved)
|
||||
print(f" {name:>9}: {b2['label']:<32} solved {solved}/199 "
|
||||
f"APS {b2['APS']['total']:6d} CHA {b2['CHA']['total']:6d} total {tot:6d} "
|
||||
f"over {b2['APS']['over'] + b2['CHA']['over']}")
|
||||
print(f" {'':>9} per-set APS best {per_set_best(pure, 'APS')['label']:<28} "
|
||||
f"{per_set_best(pure, 'APS')['APS']['total']:6d} | "
|
||||
f"CHA best {per_set_best(pure, 'CHA')['label']:<28} "
|
||||
f"{per_set_best(pure, 'CHA')['CHA']['total']:6d}")
|
||||
|
||||
if len(bests) == 2:
|
||||
bx, tx, _sx = bests["x-space"]
|
||||
bl, tl, sl = bests["log-space"]
|
||||
apx = per_set_best([r for r in xs if r["spec"][0] == "pure" and r["spec"][3] in GUAR], "APS")
|
||||
apl = per_set_best([r for r in lg if r["spec"][0] == "pure" and r["spec"][3] in GUAR], "APS")
|
||||
chl = per_set_best([r for r in lg if r["spec"][0] == "pure" and r["spec"][3] in GUAR], "CHA")
|
||||
print("\nTARGETS (PREREG): two sets < 6026, CHA <= 2857, solved >= 193/199, guarantee intact")
|
||||
print(f" VERDICT G2' (APS must not grow, per-set best vs per-set best): "
|
||||
f"{'PASS' if apl['APS']['total'] <= apx['APS']['total'] else 'FAIL'} "
|
||||
f"(log {apl['APS']['total']} vs x {apx['APS']['total']})")
|
||||
print(f" VERDICT L3 (CHA <= 2857): {'PASS' if chl['CHA']['total'] <= 2857 else 'FAIL'} "
|
||||
f"(log {chl['CHA']['total']})")
|
||||
print(f" VERDICT two sets < 6026: {'PASS' if tl < 6026 else 'FAIL'} "
|
||||
f"(log {tl}, x {tx})")
|
||||
print(f" VERDICT solved >= 193: {'PASS' if sl >= 193 else 'FAIL'} ({sl}/199)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1] if len(sys.argv) > 1 else "sweep_full.json",
|
||||
sys.argv[2] if len(sys.argv) > 2 else "sweep_log.json")
|
||||
Reference in New Issue
Block a user