PREREG_modab outcome (server): M1 PASSED - A' ahead of ModAB (authors' package pymodab 1.0.9, same stopping rule, counts equal to the library's own counter). Corpus 4: A' 3656 vs ModAB 5535, per problem 183/23/34 p=7e-26. Paper protocol: APS 2438 vs 2914 (118/11/25), CHA 2005 vs 2436 (32/9/4). Six corpora: 13032 vs 16657, significant on every one. ModAB's own benchmark (Roots.jl f01-f91): all 91 - 1507 vs 1588 (56/13/22 p=1.5e-4); 78 - 1371 vs 1446. ModAB breaks the 2x-bisection bound on 9 problems (1 on corpus 1, 8 on corpus 4), A' on none. ModAB wired outside the published grid (EXTRA_BASELINES), which stays at 264.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,25 @@
|
||||
PAPER PROTOCOL (published roots)
|
||||
APS: A' 2438 ModAB 2914 (solved 153, over 0) Chandrupatla 2787 A' vs ModAB 118/11/25 p=1.2e-15
|
||||
CHA: A' 2005 ModAB 2436 (solved 40, over 0) Chandrupatla 2441 A' vs ModAB 32/9/4 p=1.9e-06
|
||||
|
||||
SIX CORPORA (uniform criterion): A' / ModAB / Chandrupatla totals; ModAB violations; A' vs ModAB
|
||||
APS 2259 / 2726 / 2595 viol 0 118/10/26 p=3.4e-15
|
||||
CHA 1074 / 1681 / 1505 viol 0 34/4/7 p=2.5e-05
|
||||
NEW 3290 / 3809 / 3716 viol 1 125/26/55 p=1.9e-07
|
||||
RJL 1371 / 1446 / 1566 viol 0 47/11/20 p=0.0013
|
||||
APP 1382 / 1460 / 1382 viol 0 76/35/21 p=1.7e-08
|
||||
D 3656 / 5535 / 5717 viol 8 183/23/34 p=7.1e-26
|
||||
all 13032 / 16657
|
||||
|
||||
M1 (corpus 4): PASSED - A' 3656 vs ModAB 5535, 183/23/34 p=7.1e-26
|
||||
D1: A' 433 ModAB 446 24/5/11
|
||||
D2: A' 1163 ModAB 1689 39/0/1
|
||||
D3: A' 924 ModAB 1267 34/0/6
|
||||
D4: A' 304 ModAB 362 36/3/1
|
||||
D5: A' 503 ModAB 1425 26/7/7
|
||||
D6: A' 329 ModAB 346 24/8/8
|
||||
|
||||
MODAB'S OWN BENCHMARK (Roots.jl f01-f91)
|
||||
all 91: Aprime 1507 modab 1588 chandrupatla 1701 A' vs ModAB 56/13/22 p=0.00015 ModAB viol 0
|
||||
78 out of the development sets: Aprime 1371 modab 1446 chandrupatla 1566 A' vs ModAB 47/11/20 p=0.0013 ModAB viol 0
|
||||
modab done
|
||||
@@ -500,6 +500,25 @@ def chandrupatla_lib(f, a, b, xtol, maxfev):
|
||||
return 0.5 * (a + b), g.n, False
|
||||
|
||||
|
||||
def modab_lib(f, a, b, xtol, maxfev):
|
||||
"""ModAB from the AUTHORS' package pymodab (native wrapper of their C code; PREREG_modab
|
||||
2026-09-21). Same stopping rule as every other method, read in ModAB.c: the bracket
|
||||
x2 - x1 <= aTol + rTol*|x3| with aTol = xtol and the library Chandrupatla's relative term
|
||||
8.9e-16, no stopping on |f|. Evaluations counted by the same Counter; an exception
|
||||
raised in f (the budget included) propagates out of the native call."""
|
||||
from pymodab import find_root
|
||||
g = Counter(f, maxfev)
|
||||
try:
|
||||
x = find_root(lambda t: g(float(t)), float(a), float(b), float(xtol), 8.9e-16,
|
||||
int(maxfev))
|
||||
except _Budget:
|
||||
return 0.5 * (a + b), g.n, False
|
||||
except Exception:
|
||||
return 0.5 * (a + b), g.n, False
|
||||
x = float(x)
|
||||
return x, g.n, math.isfinite(x)
|
||||
|
||||
|
||||
# The competitors as benchmarked: the library alone wherever the library ships the
|
||||
# method. Until 2026-09-21 they were best_of(ours, library) problem by problem, which
|
||||
# let a tolerance slip in OUR Chandrupatla leak into the reported numbers.
|
||||
|
||||
+7
-2
@@ -41,7 +41,7 @@ from simroot.baselines import (anderson_bjorck, best_of, bisection, brent,
|
||||
brent_scipy, chandrupatla, chandrupatla_scipy,
|
||||
dekker, illinois, itp, pegasus, regula_falsi,
|
||||
ridders, toms748)
|
||||
from simroot.baselines import BRENT_BASELINE, CHANDRUPATLA_BASELINE
|
||||
from simroot.baselines import BRENT_BASELINE, CHANDRUPATLA_BASELINE, modab_lib
|
||||
from simroot.pure import PURE_SIMS, pure_root
|
||||
|
||||
XTOL = 1e-12
|
||||
@@ -101,6 +101,11 @@ BASELINES = {
|
||||
}
|
||||
|
||||
|
||||
# Rivals measured OUTSIDE the published grid (which iterates over BASELINES and must stay at
|
||||
# 264 configurations): ModAB from the authors' package (PREREG_modab 2026-09-21).
|
||||
EXTRA_BASELINES = {"modab": modab_lib}
|
||||
|
||||
|
||||
def make_method(spec):
|
||||
"""spec is ('base', name) or ('pure', similarity, scale, guard[, space]).
|
||||
|
||||
@@ -108,7 +113,7 @@ def make_method(spec):
|
||||
amendment 1, 2026-09-20): omitted or "x" reproduces the published grid
|
||||
bit-for-bit, "log" is the logarithmic-march mode."""
|
||||
if spec[0] == "base":
|
||||
return BASELINES[spec[1]]
|
||||
return BASELINES[spec[1]] if spec[1] in BASELINES else EXTRA_BASELINES[spec[1]]
|
||||
sim, scale, guard = spec[1], spec[2], spec[3]
|
||||
space = spec[4] if len(spec) > 4 else "x"
|
||||
prestep = spec[5] if len(spec) > 5 else "none"
|
||||
|
||||
File diff suppressed because one or more lines are too long
+139
@@ -0,0 +1,139 @@
|
||||
"""PREREG_modab_2026-09-21: A' against ModAB (the authors' package pymodab) - on the paper's
|
||||
protocol (published roots), on the six corpora of section 9 (uniform criterion), and on
|
||||
ModAB's own benchmark (Roots.jl f01-f91: all 91 and the 78 out of the development sets).
|
||||
A' and ModAB are frozen; nothing is selected by the result.
|
||||
|
||||
python -u sweep_modab.py run -> sweep_modab.json, then the verdicts
|
||||
python -u sweep_modab.py --report
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from multiprocessing import Pool
|
||||
|
||||
import sweep_fourth
|
||||
from corpus_roots_jl import build as build_rjl
|
||||
from corpus_roots_jl import solved
|
||||
from sweep_full import APRIME, MAXFEV, XTOL
|
||||
from sweep_full import evaluate as eval_protocol
|
||||
from sweep_full import make_method
|
||||
from sweep_oos import sign_p
|
||||
|
||||
sweep_fourth.CONFIGS["modab"] = ("base", "modab")
|
||||
SETS = ("APS", "CHA", "NEW", "RJL", "APP", "D")
|
||||
APRIME_SPEC = ("pure", "ratio", "derived3", "project", "x", APRIME, "recent", "distance",
|
||||
"none", "pattern")
|
||||
|
||||
|
||||
def eval_rjl_all(spec):
|
||||
method = make_method(spec)
|
||||
out = {}
|
||||
for pid, f, a, b, ins in build_rjl()[0]:
|
||||
try:
|
||||
x, n, conv = method(f, a, b, XTOL, MAXFEV)
|
||||
x = float(x)
|
||||
except Exception:
|
||||
x, n, conv = float("nan"), MAXFEV, False
|
||||
ok = bool(conv and solved(f, x))
|
||||
over = ok and int(n) > 2 * math.ceil(math.log2(abs(b - a) / XTOL)) + 4
|
||||
out[pid] = [int(n), ok, bool(over), bool(ins)]
|
||||
return out
|
||||
|
||||
|
||||
def job(j):
|
||||
kind, name = j
|
||||
if kind == "protocol":
|
||||
return kind, name, eval_protocol(("base", "modab"))
|
||||
if kind in ("seen", "fourth"):
|
||||
return kind, name, sweep_fourth.evaluate((name, kind))[1]
|
||||
spec = APRIME_SPEC if name == "Aprime" else ("base", name)
|
||||
return kind, name, eval_rjl_all(spec)
|
||||
|
||||
|
||||
def cost(v):
|
||||
return v[0] if v[1] else MAXFEV
|
||||
|
||||
|
||||
def h2h(a, b, keep=None):
|
||||
ps = [p for p in a if keep is None or keep(p)]
|
||||
w = sum(cost(a[p]) < cost(b[p]) for p in ps)
|
||||
l = sum(cost(a[p]) > cost(b[p]) for p in ps)
|
||||
return w, len(ps) - w - l, l
|
||||
|
||||
|
||||
def report(R):
|
||||
ap_proto = {r["label"]: r for r in json.load(open("sweep_aprime.json", encoding="utf8"))}[
|
||||
f"pure+extP+{APRIME}.ratio.derived3.project"]
|
||||
full = {r["label"]: r for r in json.load(open("sweep_full.json", encoding="utf8"))}
|
||||
mp = R["protocol"]["modab"]
|
||||
print("PAPER PROTOCOL (published roots)")
|
||||
for s in ("APS", "CHA"):
|
||||
a = {p: v for p, v in ap_proto[s]["per"].items()}
|
||||
m = {p: v for p, v in mp[s]["per"].items()}
|
||||
w, t, l = h2h(a, m)
|
||||
print(f" {s}: A' {ap_proto[s]['total']} ModAB {mp[s]['total']} (solved {mp[s]['solved']}, "
|
||||
f"over {mp[s]['over']}) Chandrupatla {full['chandrupatla'][s]['total']} "
|
||||
f"A' vs ModAB {w}/{t}/{l} p={sign_p(w, l):.2g}")
|
||||
seen = json.load(open("sweep_fourth_seen.json", encoding="utf8"))
|
||||
four = json.load(open("sweep_fourth_fourth.json", encoding="utf8"))
|
||||
for k in seen:
|
||||
seen[k].update(four[k])
|
||||
md = dict(R["seen"]["modab"])
|
||||
md.update(R["fourth"]["modab"])
|
||||
print("\nSIX CORPORA (uniform criterion): A' / ModAB / Chandrupatla totals; ModAB violations; "
|
||||
"A' vs ModAB")
|
||||
tot_a = tot_m = 0
|
||||
for s in SETS:
|
||||
a, m = seen["Aprime"][s], md[s]
|
||||
ta, tm = sum(cost(v) for v in a.values()), sum(cost(v) for v in m.values())
|
||||
tc = sum(cost(v) for v in seen["chandrupatla"][s].values())
|
||||
tot_a += ta
|
||||
tot_m += tm
|
||||
w, t, l = h2h(a, m)
|
||||
print(f" {s:<4} {ta:6d} / {tm:6d} / {tc:6d} viol {sum(v[2] for v in m.values()):3d} "
|
||||
f"{w}/{t}/{l} p={sign_p(w, l):.2g}")
|
||||
print(f" all {tot_a:6d} / {tot_m:6d}")
|
||||
a, m = seen["Aprime"]["D"], md["D"]
|
||||
w, t, l = h2h(a, m)
|
||||
m1 = (sum(cost(v) for v in a.values()) < sum(cost(v) for v in m.values())
|
||||
and w > l and sign_p(w, l) < 0.05)
|
||||
print(f"\nM1 (corpus 4): {'PASSED' if m1 else 'NOT passed'} - A' "
|
||||
f"{sum(cost(v) for v in a.values())} vs ModAB {sum(cost(v) for v in m.values())}, "
|
||||
f"{w}/{t}/{l} p={sign_p(w, l):.2g}")
|
||||
for cls in ("D1", "D2", "D3", "D4", "D5", "D6"):
|
||||
k = lambda p, c=cls: p.startswith(c)
|
||||
w, t, l = h2h(a, m, k)
|
||||
print(f" {cls}: A' {sum(cost(a[p]) for p in a if k(p))} ModAB "
|
||||
f"{sum(cost(m[p]) for p in m if k(p))} {w}/{t}/{l}")
|
||||
rj = R["rjl"]
|
||||
print("\nMODAB'S OWN BENCHMARK (Roots.jl f01-f91)")
|
||||
for title, keep in (("all 91", None), ("78 out of the development sets", lambda p: not rj["Aprime"][p][3])):
|
||||
sel = [p for p in rj["Aprime"] if keep is None or keep(p)]
|
||||
tots = {k: sum(cost(rj[k][p]) for p in sel) for k in rj}
|
||||
w, t, l = h2h({p: rj["Aprime"][p] for p in sel}, rj["modab"])
|
||||
print(f" {title}: " + " ".join(f"{k} {tots[k]}" for k in ("Aprime", "modab", "chandrupatla"))
|
||||
+ f" A' vs ModAB {w}/{t}/{l} p={sign_p(w, l):.2g} ModAB viol "
|
||||
f"{sum(rj['modab'][p][2] for p in sel)}")
|
||||
|
||||
|
||||
def main():
|
||||
if "--report" not in sys.argv:
|
||||
jobs = [("protocol", "modab"), ("seen", "modab"), ("fourth", "modab"),
|
||||
("rjl", "Aprime"), ("rjl", "modab"), ("rjl", "chandrupatla")]
|
||||
with Pool(len(jobs)) as pool:
|
||||
out = pool.map(job, jobs)
|
||||
R = {}
|
||||
for kind, name, res in out:
|
||||
R.setdefault(kind, {})[name] = res
|
||||
with open("sweep_modab.json", "w", encoding="utf8") as fh:
|
||||
json.dump(R, fh)
|
||||
with open("sweep_modab.json", encoding="utf8") as fh:
|
||||
R = json.load(fh)
|
||||
report(R)
|
||||
print("modab done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user