vvsandClaude Opus 5
60294e84d9
simroot/pure.py: flatlagmempt / flatbandmempt / flatflagmempt - A' with only the flat detector swapped (Lagouanelle without declining outside (0,1), with memory; Chandrupatla's band; none). A' itself is untouched: bit-identical to the compiled simbracket on all 868 problems (x2). sweep_controls.py: K2 through sweep_fourth's harness, K1 with the oracle order m*. Amendment 1: m* <= 0 is undefined (aps.15.22-30, C0.04 crashed the first full attempt). Amendment 2 (after the smokes; only corpus-level K1 sums and corpus-4 K2 sums were seen): each side's slope rounded to 0.01, m* defined only when both sides show a zero (jumps C0.03, aps.15.21), group 'multiple/fractional' = m* >= 2 or m* <= 2/3, the branches of section 8.1 (simple roots measured at 0.99999 fell under the registered 'm* < 1'). tests/test_article_numbers.py: the verifier test runs on Linux only (the paper's numbers are a Linux run; libm's last digit differs elsewhere, e.g. 4461 vs 4462). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
109 lines
4.9 KiB
Python
109 lines
4.9 KiB
Python
"""The paper's numbers are part of the regression.
|
|
|
|
A table transcribed by hand drifts from its data the first time the data is
|
|
regenerated. verify_article.py asserts all 72 stated numbers against
|
|
sweep_full.json; this makes that check part of the suite, so a re-run of the
|
|
sweep that moves a number fails the build instead of quietly invalidating the
|
|
paper.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
# the constant-free variant (the proposed rule is the projection, variant a)
|
|
CREDIT_LABEL = "pure.ratio.derived3.credit"
|
|
|
|
|
|
@pytest.mark.skipif(not os.path.exists(os.path.join(ROOT, "sweep_full.json")),
|
|
reason="sweep_full.json not present; run sweep_full.py")
|
|
@pytest.mark.skipif(not sys.platform.startswith("linux"),
|
|
reason="the paper's numbers come from a Linux run (section 4, Reproducibility); "
|
|
"verify_article.py recomputes some of them, and libm's last digit "
|
|
"differs on other platforms (e.g. 4461 vs 4462 in Table 1b)")
|
|
def test_every_number_in_the_paper_matches_the_sweep():
|
|
r = subprocess.run([sys.executable, "verify_article.py"], cwd=ROOT,
|
|
capture_output=True, text=True, timeout=600)
|
|
tail = "\n".join(r.stdout.strip().splitlines()[-25:])
|
|
assert r.returncode == 0, f"the paper disagrees with the data:\n{tail}"
|
|
assert "every number stated in the paper matches" in r.stdout
|
|
assert r.stdout.count(" OK ") >= 72, r.stdout.count(" OK ")
|
|
|
|
|
|
@pytest.mark.skipif(not os.path.exists(os.path.join(ROOT, "sweep_full.json")),
|
|
reason="sweep_full.json not present")
|
|
def test_the_headline_claim_is_what_the_paper_says_it_is():
|
|
"""Guard against the one mistake that would matter most: the abstract
|
|
claiming a place the data does not support.
|
|
|
|
It caught exactly that. Until Chandrupatla's own method was added - a
|
|
reader asked why the owner of half the benchmark was missing - this test
|
|
asserted that a configuration of ours came first, and it passed only
|
|
because the method that beats us was absent from the field. The claims it
|
|
now guards (variant a, Valeri 2026-09-21): by TOTAL Chandrupatla is first
|
|
on the canonical set and we are within one percent of it; problem by
|
|
problem we beat it with significance; on the wide-bracket set and over
|
|
both sets it beats us. And the constant-free variant may not borrow the
|
|
projection's canonical numbers."""
|
|
import json
|
|
import math
|
|
|
|
from report_sweep import PROPOSED, head_to_head, rank
|
|
with open(os.path.join(ROOT, "sweep_full.json"), encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
by = {r["label"]: r for r in rows}
|
|
assert "chandrupatla" in by, "the sweep predates the missing baselines"
|
|
assert PROPOSED == "pure.ratio.derived3.project"
|
|
|
|
aps = rank(rows, "APS")
|
|
assert aps[0]["label"] == "chandrupatla", aps[0]["label"]
|
|
|
|
def both(lbl):
|
|
return by[lbl]["APS"]["total"] + by[lbl]["CHA"]["total"]
|
|
|
|
def tot(lbl, s):
|
|
return by[lbl][s]["total"]
|
|
|
|
# canonical set: parity by total with the best, a tie with TOMS 748
|
|
assert tot(PROPOSED, "APS") > tot("chandrupatla", "APS")
|
|
assert tot(PROPOSED, "APS") < 1.01 * tot("chandrupatla", "APS")
|
|
assert tot(PROPOSED, "APS") < tot("toms748", "APS") < 1.01 * tot(PROPOSED, "APS")
|
|
|
|
# ... and an edge problem by problem, with significance
|
|
w, _t, l, _ = head_to_head(rows, PROPOSED, "chandrupatla", "APS")
|
|
p = min(1.0, 2.0 * sum(math.comb(w + l, i) for i in range(min(w, l) + 1))
|
|
/ 2.0 ** (w + l))
|
|
assert w > 2 * l - 1 and p < 1e-3, (w, l, p)
|
|
|
|
# wide brackets and both sets: Chandrupatla is stronger, and the paper says so
|
|
assert tot(PROPOSED, "CHA") > tot("chandrupatla", "CHA")
|
|
assert both(PROPOSED) > both("chandrupatla"), "behind Chandrupatla"
|
|
for weaker in ("toms748", "brent", "itp", "ridders", "bisection"):
|
|
assert both(PROPOSED) < both(weaker), weaker
|
|
|
|
# the constant-free variant may not borrow the projection's canonical numbers
|
|
assert tot(CREDIT_LABEL, "APS") > tot("toms748", "APS")
|
|
assert tot(CREDIT_LABEL, "APS") > tot("brent", "APS")
|
|
assert tot(CREDIT_LABEL, "CHA") < tot(PROPOSED, "CHA")
|
|
|
|
# the claim the constant-free variant does own: the worst single problem,
|
|
# over the problems it solves, is close to bisection's and far below the
|
|
# classical methods'
|
|
def worst(lbl):
|
|
wst = 0
|
|
for setname in ("APS", "CHA"):
|
|
for n, ok in by[lbl][setname]["per"].values():
|
|
if ok:
|
|
wst = max(wst, n)
|
|
return wst
|
|
|
|
assert worst(CREDIT_LABEL) < 100, worst(CREDIT_LABEL)
|
|
assert worst(CREDIT_LABEL) < worst("brent")
|
|
assert worst(CREDIT_LABEL) < worst("toms748")
|
|
assert worst(CREDIT_LABEL) < worst(PROPOSED), (worst(CREDIT_LABEL), worst(PROPOSED))
|