baseline: the published sim-numerics artifact as it stands (490 tests, verify_article.py checks all 72 paper numbers against sweep_full.json)
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
+165
@@ -0,0 +1,165 @@
|
||||
"""Read sweep_full.json and answer the questions the paper needs answered.
|
||||
|
||||
Everything printed here comes from the single server run; no number is
|
||||
recomputed on a different machine. (The test functions use libm, whose last
|
||||
digit differs between Windows and Linux, so mixing platforms would move
|
||||
evaluation counts by a unit or two - measured: scipy's toms748 scores 2819 on
|
||||
Windows and 2820 on Linux for the same set.)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
BASE = {"bisection", "regula_falsi", "illinois", "pegasus", "anderson_bjorck",
|
||||
"ridders", "brent", "toms748", "itp", "itp_oracle"}
|
||||
|
||||
# The configuration the paper proposes. It is not the winner of a search over
|
||||
# the grid - it is what the theory of section 2 dictates, fixed before the
|
||||
# ranking is looked at:
|
||||
# similarity a nonzero slope of (1 - S) at 0 -> the ratio form 1 - |z|
|
||||
# scale the finest data-derived one -> the three-point difference
|
||||
# safeguard the one that carries a bound without destroying the order
|
||||
# AND without a free constant -> the credit projection, whose
|
||||
# endowment is a licence of one unconstrained step rather than
|
||||
# a number (see simroot.pure.project_credit)
|
||||
# The grid is evidence for the theory, not the method's selection procedure.
|
||||
PROPOSED = "pure.ratio.derived3.credit"
|
||||
TUNED = {"illinois", "pegasus", "anderson_bjorck", "brent", "toms748",
|
||||
"itp", "itp_oracle"}
|
||||
|
||||
|
||||
def load(path="sweep_full.json"):
|
||||
with open(path, encoding="utf8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def rank(rows, setname):
|
||||
return sorted(rows, key=lambda r: (r[setname]["n"] - r[setname]["solved"],
|
||||
r[setname]["total"]))
|
||||
|
||||
|
||||
def place(rows, setname, lbl):
|
||||
for i, r in enumerate(rank(rows, setname), 1):
|
||||
if r["label"] == lbl:
|
||||
return i
|
||||
raise KeyError(f"no configuration labelled {lbl!r} in the sweep; "
|
||||
"a mistyped name must not be reported as a missing place")
|
||||
|
||||
|
||||
def best_pure(rows, setname):
|
||||
"""The best configuration OF OURS. The overall winner of a set may be a
|
||||
baseline - itp_oracle tops Chandrupatla - and comparing that against
|
||||
itself prints nothing of interest."""
|
||||
for r in rank(rows, setname):
|
||||
if r["label"].startswith("pure."):
|
||||
return r["label"]
|
||||
raise KeyError("no pure configuration in the sweep")
|
||||
|
||||
|
||||
def head_to_head(rows, a, b, setname):
|
||||
ra = next(r for r in rows if r["label"] == a)[setname]
|
||||
rb = next(r for r in rows if r["label"] == b)[setname]
|
||||
win = tie = loss = both = 0
|
||||
for pid, (na, oka) in ra["per"].items():
|
||||
nb, okb = rb["per"][pid]
|
||||
if not (oka and okb):
|
||||
continue
|
||||
both += 1
|
||||
if na < nb:
|
||||
win += 1
|
||||
elif na > nb:
|
||||
loss += 1
|
||||
else:
|
||||
tie += 1
|
||||
return win, tie, loss, both
|
||||
|
||||
|
||||
def main():
|
||||
rows = load(sys.argv[1] if len(sys.argv) > 1 else "sweep_full.json")
|
||||
print(f"{len(rows)} configurations\n")
|
||||
|
||||
for setname in ("APS", "CHA"):
|
||||
r = rank(rows, setname)
|
||||
n = r[0][setname]["n"]
|
||||
print("=" * 90)
|
||||
print(f"{setname} ({n} problems) top 12, with the best baseline "
|
||||
f"positions marked")
|
||||
print("=" * 90)
|
||||
print(f"{'#':>3s} {'configuration':38s}{'tuning':>8s}{'solved':>9s}"
|
||||
f"{'total':>9s}{'over 2x':>9s}")
|
||||
print("-" * 90)
|
||||
for i, row in enumerate(r[:12], 1):
|
||||
d = row[setname]
|
||||
tuned = "yes" if row["label"] in TUNED else "no"
|
||||
star = " <-" if row["label"] not in BASE else ""
|
||||
print(f"{i:>3d} {row['label']:38s}{tuned:>8s}"
|
||||
f"{d['solved']:>6d}/{d['n']:<3d}{d['total']:>9d}"
|
||||
f"{d['over']:>9d}{star}")
|
||||
print()
|
||||
for b in ("brent", "toms748", "itp", "itp_oracle", "bisection"):
|
||||
p = place(rows, setname, b)
|
||||
d = next(x for x in rows if x["label"] == b)[setname]
|
||||
print(f" {b:38s}place {p:>3d} total {d['total']}")
|
||||
print()
|
||||
|
||||
print("=" * 90)
|
||||
print(f"THE PROPOSED CONFIGURATION {PROPOSED}")
|
||||
print(" (dictated by the theory, not selected from the ranking)")
|
||||
print("=" * 90)
|
||||
for setname in ("APS", "CHA"):
|
||||
d = next(r for r in rows if r["label"] == PROPOSED)[setname]
|
||||
print(f" {setname}: place {place(rows, setname, PROPOSED):>3d} of "
|
||||
f"{len(rows)} solved {d['solved']}/{d['n']} "
|
||||
f"total {d['total']} over 2x bound {d['over']}")
|
||||
for opp in ("brent", "toms748", "itp", "itp_oracle", "bisection"):
|
||||
w, t, l, both = head_to_head(rows, PROPOSED, opp, setname)
|
||||
print(f" vs {opp:12s}: {w} wins / {t} ties / {l} losses "
|
||||
f"(of {both})")
|
||||
print()
|
||||
|
||||
print("=" * 90)
|
||||
print("HEAD TO HEAD, best pure configuration of each set")
|
||||
print("=" * 90)
|
||||
for setname in ("APS", "CHA"):
|
||||
best = best_pure(rows, setname)
|
||||
for opp in ("brent", "toms748", "itp", "itp_oracle"):
|
||||
w, t, l, both = head_to_head(rows, best, opp, setname)
|
||||
print(f"{setname} {best} vs {opp:12s}: "
|
||||
f"{w} wins / {t} ties / {l} losses (of {both})")
|
||||
print()
|
||||
|
||||
print("=" * 90)
|
||||
print("BEST CONFIGURATION THAT CARRIES THE PROVEN 2x BOUND")
|
||||
print("=" * 90)
|
||||
for setname in ("APS", "CHA"):
|
||||
sound = [x for x in rank(rows, setname)
|
||||
if x["label"].startswith("pure.")
|
||||
and x["label"].rsplit(".", 1)[1] in ("bracket", "both")]
|
||||
b = sound[0]
|
||||
print(f"{setname}: {b['label']:38s} total {b[setname]['total']:>7d}"
|
||||
f" place {place(rows, setname, b['label'])}")
|
||||
|
||||
print("\n" + "=" * 90)
|
||||
print("WHICH SIMILARITY, WHICH SCALE (guard = step, both sets summed)")
|
||||
print("=" * 90)
|
||||
sims, scales = {}, {}
|
||||
for row in rows:
|
||||
if not row["label"].startswith("pure."):
|
||||
continue
|
||||
_, sim, scale, guard = row["label"].split(".")
|
||||
if guard != "step":
|
||||
continue
|
||||
tot = row["APS"]["total"] + row["CHA"]["total"]
|
||||
sims.setdefault(sim, []).append(tot)
|
||||
scales.setdefault(scale, []).append(tot)
|
||||
print("similarity (best over scales):")
|
||||
for sim, v in sorted(sims.items(), key=lambda kv: min(kv[1])):
|
||||
print(f" {sim:12s} best {min(v):>7d} worst {max(v):>7d}")
|
||||
print("scale (best over similarities):")
|
||||
for sc, v in sorted(scales.items(), key=lambda kv: min(kv[1])):
|
||||
print(f" {sc:12s} best {min(v):>7d} worst {max(v):>7d}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user