baseline: the published sim-numerics artifact as it stands (490 tests, verify_article.py checks all 72 paper numbers against sweep_full.json)
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
+241
@@ -0,0 +1,241 @@
|
||||
"""The full sweep: every similarity function x every scale x every safeguard,
|
||||
over both benchmark sets, against every baseline.
|
||||
|
||||
Purpose. Every claim in the paper so far rests on one configuration
|
||||
(similarity = ratio) measured on eight hand-picked problems, with only two
|
||||
configurations carried to the 199-problem benchmark. The convergence law of
|
||||
stage 0c predicts the behaviour of ALL nine similarity functions; a law
|
||||
confirmed on eight problems and one configuration is an anecdote. This sweep
|
||||
is what turns it into evidence, and what would refute it if it is wrong.
|
||||
|
||||
Grid
|
||||
similarity 9 (ratio sqrt cbrt rational ratio_soft exp_abs
|
||||
rat_sq exp_sq cosine)
|
||||
scale 4 (start bracket derived derived3)
|
||||
safeguard 4 (none step bracket both)
|
||||
= 144 configurations, plus 10 baselines, over 199 problems.
|
||||
|
||||
Cost model and reporting rules are the same as bench_full.py: evaluations of
|
||||
f; failures charged the full budget; the 2x-bisection bound checked only on
|
||||
solved runs; ITP reported both at its recommended constants and as a
|
||||
per-problem oracle over a (kappa1, n0) grid.
|
||||
|
||||
Run: python -u sweep_full.py full sweep
|
||||
python -u sweep_full.py --smoke a reduced grid, for the pre-flight
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from multiprocessing import Pool
|
||||
|
||||
os.environ.setdefault("OMP_NUM_THREADS", "1")
|
||||
|
||||
from scipy.optimize._tstutils import (_APS_TESTS_DICTS,
|
||||
_CHANDRUPATLA_TESTS_DICTS)
|
||||
|
||||
from simroot.baselines import (anderson_bjorck, best_of, bisection, brent,
|
||||
brent_scipy, chandrupatla, chandrupatla_scipy,
|
||||
dekker, illinois, itp, pegasus, regula_falsi,
|
||||
ridders, toms748)
|
||||
from simroot.pure import PURE_SIMS, pure_root
|
||||
|
||||
XTOL = 1e-12
|
||||
MAXFEV = 200
|
||||
ACC = 1e-6
|
||||
|
||||
SCALES = ("start", "bracket", "derived", "derived3")
|
||||
GUARD_MODES = ("none", "step", "bracket", "both", "project", "safe",
|
||||
"credit")
|
||||
ITP_GRID = [(fac, n0) for fac in (1.0, 0.1, 0.01, 1e-4)
|
||||
for n0 in (0, 1, 2, 5)]
|
||||
|
||||
|
||||
def load_problems():
|
||||
"""(set, id, f, a, b, root). Rebuilt inside each worker so nothing that
|
||||
has to cross a process boundary is a closure."""
|
||||
out = []
|
||||
for d in _APS_TESTS_DICTS:
|
||||
a, b = d["bracket"]
|
||||
args = d.get("args", ())
|
||||
out.append(("APS", d["ID"],
|
||||
(lambda ff, aa: lambda x: ff(x, *aa))(d["f"], args),
|
||||
float(a), float(b), float(d["root"])))
|
||||
for d in _CHANDRUPATLA_TESTS_DICTS:
|
||||
a, b = d["bracket"]
|
||||
out.append(("CHA", d["ID"], (lambda ff: lambda x: ff(x))(d["f"]),
|
||||
float(a), float(b), float(d["root"])))
|
||||
return out
|
||||
|
||||
|
||||
def itp_oracle(f, a, b, xtol, maxfev):
|
||||
best = None
|
||||
for fac, n0 in ITP_GRID:
|
||||
try:
|
||||
x, n, conv = itp(f, a, b, xtol, maxfev,
|
||||
kappa1=fac * 0.2 / (b - a), n0=n0)
|
||||
except ValueError:
|
||||
continue
|
||||
if conv and (best is None or n < best[1]):
|
||||
best = (x, n, True)
|
||||
return best if best is not None else (0.5 * (a + b), maxfev, False)
|
||||
|
||||
|
||||
# Competitors are taken at the better of SciPy's implementation and ours,
|
||||
# problem by problem, wherever SciPy ships the method. Ours turned out to be
|
||||
# the cheaper of the two - 6991 against brentq's 7004, 5209 against
|
||||
# _chandrupatla's 5228 - so quoting the library alone would have flattered us.
|
||||
BASELINES = {
|
||||
"bisection": bisection, "regula_falsi": regula_falsi,
|
||||
"illinois": illinois, "pegasus": pegasus,
|
||||
"anderson_bjorck": anderson_bjorck, "ridders": ridders,
|
||||
"dekker": dekker,
|
||||
"brent": best_of(brent, brent_scipy),
|
||||
"chandrupatla": best_of(chandrupatla, chandrupatla_scipy),
|
||||
"toms748": toms748,
|
||||
"itp": itp, "itp_oracle": itp_oracle,
|
||||
}
|
||||
|
||||
|
||||
def make_method(spec):
|
||||
"""spec is ('base', name) or ('pure', similarity, scale, guard)."""
|
||||
if spec[0] == "base":
|
||||
return BASELINES[spec[1]]
|
||||
_, sim, scale, guard = spec
|
||||
|
||||
def m(f, a, b, xtol, maxfev):
|
||||
return pure_root(f, a, b, xtol, maxfev, similarity=sim,
|
||||
scale=scale, guard=guard)
|
||||
return m
|
||||
|
||||
|
||||
def label(spec):
|
||||
if spec[0] == "base":
|
||||
return spec[1]
|
||||
return f"pure.{spec[1]}.{spec[2]}.{spec[3]}"
|
||||
|
||||
|
||||
def evaluate(spec):
|
||||
"""Run one configuration over all 199 problems. Returns a plain dict so
|
||||
it can cross the process boundary."""
|
||||
method = make_method(spec)
|
||||
probs = load_problems()
|
||||
res = {"label": label(spec), "spec": list(spec)}
|
||||
for setname in ("APS", "CHA"):
|
||||
total = solved = over = 0
|
||||
n_probs = 0
|
||||
per = {}
|
||||
for s, pid, f, a, b, root in probs:
|
||||
if s != setname:
|
||||
continue
|
||||
n_probs += 1
|
||||
try:
|
||||
x, n, conv = method(f, a, b, XTOL, MAXFEV)
|
||||
except Exception:
|
||||
x, n, conv = float("nan"), MAXFEV, False
|
||||
# scipy hands back numpy scalars, whose booleans are numpy.bool_
|
||||
# and do not serialise; coerce at the boundary rather than at the
|
||||
# dump, so nothing numpy-shaped survives into the results
|
||||
ok = bool(conv and math.isfinite(x) and abs(x - root) <= ACC)
|
||||
n = int(n)
|
||||
total += n if ok else MAXFEV
|
||||
per[pid] = [n, ok]
|
||||
if ok:
|
||||
solved += 1
|
||||
if n > 2 * math.ceil(math.log2(abs(b - a) / XTOL)) + 4:
|
||||
over += 1
|
||||
res[setname] = {"total": total, "solved": solved, "n": n_probs,
|
||||
"over": over, "per": per}
|
||||
return res
|
||||
|
||||
|
||||
def build_grid(smoke):
|
||||
specs = [("base", nm) for nm in BASELINES]
|
||||
sims = sorted(PURE_SIMS)
|
||||
scales = SCALES
|
||||
guards = GUARD_MODES
|
||||
if smoke:
|
||||
sims = ["ratio", "exp_abs"]
|
||||
scales = ("derived", "derived3")
|
||||
guards = ("step", "both")
|
||||
specs = [("base", nm) for nm in ("bisection", "brent", "toms748")]
|
||||
for sim in sims:
|
||||
for sc in scales:
|
||||
for gu in guards:
|
||||
specs.append(("pure", sim, sc, gu))
|
||||
return specs
|
||||
|
||||
|
||||
def main():
|
||||
smoke = "--smoke" in sys.argv
|
||||
specs = build_grid(smoke)
|
||||
t0 = time.time()
|
||||
print(f"sweep: {len(specs)} configurations x 199 problems "
|
||||
f"({'SMOKE' if smoke else 'FULL'})", flush=True)
|
||||
|
||||
workers = max(1, min(8, (os.cpu_count() or 2) - 1))
|
||||
with Pool(workers) as pool:
|
||||
results = []
|
||||
for i, r in enumerate(pool.imap_unordered(evaluate, specs), 1):
|
||||
results.append(r)
|
||||
if i % 10 == 0 or i == len(specs):
|
||||
print(f" {i}/{len(specs)} done, "
|
||||
f"{time.time() - t0:.0f}s elapsed", flush=True)
|
||||
|
||||
out = "sweep_smoke.json" if smoke else "sweep_full.json"
|
||||
with open(out, "w", encoding="utf8") as fh:
|
||||
json.dump(results, fh)
|
||||
print(f"\nwrote {out} ({time.time() - t0:.0f}s total)\n", flush=True)
|
||||
report(results)
|
||||
|
||||
|
||||
def report(results, top=20):
|
||||
for setname in ("APS", "CHA"):
|
||||
rows = sorted(results,
|
||||
key=lambda r: (r[setname]["n"] - r[setname]["solved"],
|
||||
r[setname]["total"]))
|
||||
n = rows[0][setname]["n"]
|
||||
print("=" * 84)
|
||||
print(f"{setname}: {n} problems (top {top} of {len(rows)})")
|
||||
print("=" * 84)
|
||||
print(f"{'configuration':40s}{'solved':>9s}{'total fev':>12s}"
|
||||
f"{'over 2x':>9s}")
|
||||
print("-" * 84)
|
||||
for r in rows[:top]:
|
||||
d = r[setname]
|
||||
print(f"{r['label']:40s}{d['solved']:>6d}/{d['n']:<3d}"
|
||||
f"{d['total']:>12d}{d['over']:>9d}")
|
||||
print()
|
||||
|
||||
# the law of stage 0c, tested across the whole grid: similarities whose
|
||||
# (1 - S) has zero slope at 0 must be the slow ones whenever the scale is
|
||||
# chosen rather than derived.
|
||||
print("=" * 84)
|
||||
print("THE STAGE-0C LAW ACROSS THE GRID")
|
||||
print("=" * 84)
|
||||
quad = ("rat_sq", "exp_sq", "cosine") # (1-S) ~ c2*z^2, slope c = 0
|
||||
lin = ("ratio", "rational", "exp_abs") # (1-S) ~ z, slope c = 1
|
||||
for scale in SCALES:
|
||||
for group, name in ((lin, "slope c=1 (linear)"),
|
||||
(quad, "slope c=0 (quadratic)")):
|
||||
tot = solved = cnt = 0
|
||||
for r in results:
|
||||
if r["spec"][0] != "pure":
|
||||
continue
|
||||
_, sim, sc, gu = r["spec"]
|
||||
if sc != scale or gu != "step" or sim not in group:
|
||||
continue
|
||||
cnt += 1
|
||||
tot += r["APS"]["total"] + r["CHA"]["total"]
|
||||
solved += r["APS"]["solved"] + r["CHA"]["solved"]
|
||||
if cnt:
|
||||
print(f" scale={scale:9s} {name:26s} "
|
||||
f"mean total {tot / cnt:9.0f} "
|
||||
f"mean solved {solved / cnt:6.1f}/199")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user