baseline: the published sim-numerics artifact as it stands (490 tests, verify_article.py checks all 72 paper numbers against sweep_full.json)
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,212 @@
|
||||
"""Check every number the paper states against sweep_full.json.
|
||||
|
||||
A paper is a claim about data; transcribing its tables by hand is how a claim
|
||||
stops matching its data. A failure here means the paper is wrong, not the
|
||||
test.
|
||||
|
||||
The proposed method is now the CREDIT safeguard - the only configuration with
|
||||
no free constant. The projection variant is kept in the paper as the other
|
||||
end of a stated trade (better average, one constant), so both are checked.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
from report_sweep import PROPOSED, head_to_head, place, rank
|
||||
|
||||
FAILS = []
|
||||
PROJ = "pure.ratio.derived3.project"
|
||||
|
||||
|
||||
def check(label, got, want):
|
||||
ok = got == want
|
||||
print(f" {'OK ' if ok else 'BAD'} {label:58s} got {got!r:>16} "
|
||||
f"want {want!r}")
|
||||
if not ok:
|
||||
FAILS.append(label)
|
||||
|
||||
|
||||
def row(rows, setname, lbl):
|
||||
return next(r for r in rows if r["label"] == lbl)[setname]
|
||||
|
||||
|
||||
def main():
|
||||
with open("sweep_full.json", encoding="utf8") as fh:
|
||||
rows = json.load(fh)
|
||||
|
||||
# Guard against quoting places from a field that is missing a competitor.
|
||||
# It has already happened once: before Chandrupatla's own method was
|
||||
# added, every ranking assertion here passed only because the method that
|
||||
# beats us was absent.
|
||||
labels = {r["label"] for r in rows}
|
||||
missing = {"chandrupatla", "dekker", PROPOSED} - labels
|
||||
if missing:
|
||||
print(f" !! THE SWEEP IS STALE: missing {sorted(missing)}. "
|
||||
"Re-run sweep_full.py.\n")
|
||||
FAILS.append("sweep is stale")
|
||||
|
||||
check("number of configurations", len(rows), 264)
|
||||
|
||||
# ---- the proposed method: credit safeguard, no free constant ----------
|
||||
print("\nThe proposed method (credit safeguard):")
|
||||
check("its label", PROPOSED, "pure.ratio.derived3.credit")
|
||||
for setname, pl, tot, sol in (("APS", 29, 3568, 153),
|
||||
("CHA", 10, 3020, 40)):
|
||||
d = row(rows, setname, PROPOSED)
|
||||
check(f"{setname} place", place(rows, setname, PROPOSED), pl)
|
||||
check(f"{setname} total", d["total"], tot)
|
||||
check(f"{setname} solved", d["solved"], sol)
|
||||
check("both sets", row(rows, "APS", PROPOSED)["total"]
|
||||
+ row(rows, "CHA", PROPOSED)["total"], 6588)
|
||||
|
||||
print("\nHead to head of the proposed method:")
|
||||
for setname, opp, want in [
|
||||
("APS", "chandrupatla", (79, 3, 71)),
|
||||
("APS", "toms748", (77, 9, 67)),
|
||||
("APS", "brent", (78, 5, 70)),
|
||||
("APS", "itp", (103, 10, 40)),
|
||||
("APS", "bisection", (122, 1, 30)),
|
||||
("CHA", "chandrupatla", (5, 0, 35)),
|
||||
("CHA", "toms748", (24, 0, 16)),
|
||||
("CHA", "brent", (18, 0, 20)),
|
||||
("CHA", "itp", (12, 20, 8))]:
|
||||
w, t, l, _ = head_to_head(rows, PROPOSED, opp, setname)
|
||||
check(f"{setname} vs {opp}", (w, t, l), want)
|
||||
|
||||
# ---- the trade against the projection variant -------------------------
|
||||
print("\nThe stated trade: credit against the projection variant:")
|
||||
check("projection APS", row(rows, "APS", PROJ)["total"], 2806)
|
||||
check("projection CHA", row(rows, "CHA", PROJ)["total"], 3220)
|
||||
check("projection both", row(rows, "APS", PROJ)["total"]
|
||||
+ row(rows, "CHA", PROJ)["total"], 6026)
|
||||
check("credit costs more in total",
|
||||
(row(rows, "APS", PROPOSED)["total"]
|
||||
+ row(rows, "CHA", PROPOSED)["total"])
|
||||
> (row(rows, "APS", PROJ)["total"] + row(rows, "CHA", PROJ)["total"]),
|
||||
True)
|
||||
for setname, opp, want in [("APS", "brent", (102, 11, 40)),
|
||||
("APS", "toms748", (86, 16, 51))]:
|
||||
w, t, l, _ = head_to_head(rows, PROJ, opp, setname)
|
||||
check(f"projection {setname} vs {opp}", (w, t, l), want)
|
||||
|
||||
# ---- what the earlier, order-destroying guards cost --------------------
|
||||
print("\nThe guards that replace the step instead of projecting it:")
|
||||
check("APS unguarded", row(rows, "APS", "pure.ratio.derived3.none")["total"],
|
||||
2806)
|
||||
cha_none = row(rows, "CHA", "pure.ratio.derived3.none")
|
||||
check("CHA unguarded total", cha_none["total"], 4104)
|
||||
check("CHA unguarded solved", cha_none["solved"], 35)
|
||||
check("CHA unguarded bound violations", cha_none["over"], 10)
|
||||
check("APS bracket-guard", row(rows, "APS", "pure.ratio.derived3.bracket")["total"],
|
||||
7260)
|
||||
check("APS both-guard", row(rows, "APS", "pure.ratio.derived3.both")["total"],
|
||||
7239)
|
||||
|
||||
# ---- the field --------------------------------------------------------
|
||||
print("\nAlefeld-Potra-Shi:")
|
||||
check("APS winner", rank(rows, "APS")[0]["label"], "chandrupatla")
|
||||
for lbl, pl, tot in [("chandrupatla", 1, 2785), ("toms748", 4, 2820),
|
||||
("brent", 5, 2889), ("dekker", 12, 3158),
|
||||
("itp_oracle", 13, 3260), ("ridders", 20, 3313),
|
||||
("itp", 34, 3957), ("illinois", 73, 7045),
|
||||
("bisection", 79, 7530), ("regula_falsi", 244, 10560)]:
|
||||
check(f"APS {lbl} place", place(rows, "APS", lbl), pl)
|
||||
check(f"APS {lbl} total", row(rows, "APS", lbl)["total"], tot)
|
||||
check("APS regula falsi solved", row(rows, "APS", "regula_falsi")["solved"],
|
||||
127)
|
||||
|
||||
print("\nChandrupatla:")
|
||||
top = rank(rows, "CHA")[0]
|
||||
check("CHA rank 1 label", top["label"], "pure.sqrt.bracket.project")
|
||||
check("CHA rank 1 solves one more than anyone else", top["CHA"]["solved"], 41)
|
||||
for lbl, pl, tot in [("chandrupatla", 2, 2423), ("itp_oracle", 3, 2611),
|
||||
("itp", 6, 2857), ("bisection", 16, 3090),
|
||||
("toms748", 164, 4000), ("brent", 226, 4099)]:
|
||||
check(f"CHA {lbl} place", place(rows, "CHA", lbl), pl)
|
||||
check(f"CHA {lbl} total", row(rows, "CHA", lbl)["total"], tot)
|
||||
check("CHA brent solved", row(rows, "CHA", "brent")["solved"], 38)
|
||||
|
||||
ours_both = (row(rows, "APS", PROPOSED)["total"]
|
||||
+ row(rows, "CHA", PROPOSED)["total"])
|
||||
cha_both = (row(rows, "APS", "chandrupatla")["total"]
|
||||
+ row(rows, "CHA", "chandrupatla")["total"])
|
||||
check("chandrupatla, both sets", cha_both, 5208)
|
||||
check("Chandrupatla beats us on both sets", cha_both < ours_both, True)
|
||||
|
||||
# ---- the regime findings ---------------------------------------------
|
||||
print("\nNegative results, the regime claims:")
|
||||
check("CHA cbrt over the whole set",
|
||||
row(rows, "CHA", "pure.cbrt.derived3.project")["total"], 4099)
|
||||
switched = (row(rows, "APS", PROJ)["total"]
|
||||
+ row(rows, "CHA", "pure.cbrt.derived3.project")["total"])
|
||||
single = row(rows, "APS", PROJ)["total"] + row(rows, "CHA", PROJ)["total"]
|
||||
check("single fixed choice, both sets", single, 6026)
|
||||
check("switch by benchmark set, both sets", switched, 6905)
|
||||
check("switching is worse than not switching", switched > single, True)
|
||||
for pid, ours_n, cb_n in (("fun4.1", 89, 32), ("fun4.2", 93, 32),
|
||||
("fun4.3", 113, 40), ("fun4.4", 125, 47),
|
||||
("fun4.5", 153, 59)):
|
||||
o = row(rows, "CHA", PROJ)["per"][pid]
|
||||
c = row(rows, "CHA", "pure.cbrt.derived3.project")["per"][pid]
|
||||
check(f"{pid} projection variant", o[0], ours_n)
|
||||
check(f"{pid} cbrt", c[0], cb_n)
|
||||
|
||||
# ---- the law across the grid ------------------------------------------
|
||||
print("\nThe law across the grid:")
|
||||
lin = ("ratio", "rational", "exp_abs")
|
||||
quad = ("rat_sq", "exp_sq", "cosine")
|
||||
want = {("derived3", "lin"): 6809, ("derived3", "quad"): 22294,
|
||||
("derived", "lin"): 6906, ("derived", "quad"): 22167,
|
||||
("bracket", "lin"): 10263, ("bracket", "quad"): 19137,
|
||||
("start", "lin"): 16102, ("start", "quad"): 15345}
|
||||
for scale in ("derived3", "derived", "bracket", "start"):
|
||||
for group, gname in ((lin, "lin"), (quad, "quad")):
|
||||
tot = cnt = 0
|
||||
for r in rows:
|
||||
if r["spec"][0] != "pure":
|
||||
continue
|
||||
_, sim, sc, gu = r["spec"]
|
||||
if sc != scale or gu != "step" or sim not in group:
|
||||
continue
|
||||
cnt += 1
|
||||
tot += r["APS"]["total"] + r["CHA"]["total"]
|
||||
check(f"mean total, scale={scale}, {gname}",
|
||||
round(tot / cnt), want[(scale, gname)])
|
||||
|
||||
best_by_sim, best_by_scale = {}, {}
|
||||
for r in rows:
|
||||
if r["spec"][0] != "pure" or r["spec"][3] != "step":
|
||||
continue
|
||||
t = r["APS"]["total"] + r["CHA"]["total"]
|
||||
best_by_sim[r["spec"][1]] = min(best_by_sim.get(r["spec"][1], 10 ** 9), t)
|
||||
best_by_scale[r["spec"][2]] = min(best_by_scale.get(r["spec"][2], 10 ** 9), t)
|
||||
check("every c=1 similarity beats every c=0 similarity",
|
||||
max(best_by_sim[s] for s in lin) < min(best_by_sim[s] for s in quad),
|
||||
True)
|
||||
check("similarity ranking",
|
||||
[s for s, _ in sorted(best_by_sim.items(), key=lambda kv: kv[1])],
|
||||
["ratio", "exp_abs", "rational", "ratio_soft", "sqrt", "cbrt",
|
||||
"exp_sq", "rat_sq", "cosine"])
|
||||
for sim, v in [("ratio", 6496), ("exp_abs", 6822), ("rational", 7038),
|
||||
("ratio_soft", 9502), ("sqrt", 10497), ("cbrt", 11941),
|
||||
("exp_sq", 14314), ("rat_sq", 15102), ("cosine", 16394)]:
|
||||
check(f"similarity {sim} best", best_by_sim[sim], v)
|
||||
check("scale ranking",
|
||||
[s for s, _ in sorted(best_by_scale.items(), key=lambda kv: kv[1])],
|
||||
["derived3", "derived", "bracket", "start"])
|
||||
for sc, v in [("derived3", 6496), ("derived", 6628), ("bracket", 9081),
|
||||
("start", 14540)]:
|
||||
check(f"scale {sc} best", best_by_scale[sc], v)
|
||||
|
||||
print()
|
||||
if FAILS:
|
||||
print(f"{len(FAILS)} STATED NUMBERS DO NOT MATCH THE DATA:")
|
||||
for f in FAILS:
|
||||
print(f" {f}")
|
||||
sys.exit(1)
|
||||
print("every number stated in the paper matches the sweep data.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user