vvsandClaude Opus 5
1b6c626f67
(2) The proposed method is again the scheduled projection (2806/3220/6026), its budget multiplier DECLARED as the one free constant: the canonical total is the same for every multiplier 1.25-4, the wide-bracket one is not. The earned projection is the constant-free variant (3568/3020/6588). What the old tables hid: problem by problem the proposed rule beats Chandrupatla on the canonical set 87/24/42, exact sign test p = 9e-5, while trailing him by 0.7% in total; on the wide-bracket set he is genuinely stronger (26 to 9, p = 0.006). Gap decomposition recomputed with the library: 244 (was 262), 160 of it on the flat families (two thirds), fun1 58 (a quarter, not a third as the abstract said). Also corrected on the way: the 'subsumed by Chandrupatla' pin claimed he is better on the worst single problem - true only with the looser tolerance (84 against the credit rule's 83 once fixed); §8.3 still called zero slack degenerate, which Table 1a refutes; §5.2's Brent figure is our pure-Python implementation and now says so. verify_article.py 286/286, 802 tests pass twice. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
105 lines
4.5 KiB
Python
105 lines
4.5 KiB
Python
"""The paper's numbers are part of the regression.
|
|
|
|
A table transcribed by hand drifts from its data the first time the data is
|
|
regenerated. verify_article.py asserts all 72 stated numbers against
|
|
sweep_full.json; this makes that check part of the suite, so a re-run of the
|
|
sweep that moves a number fails the build instead of quietly invalidating the
|
|
paper.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
# the constant-free variant (the proposed rule is the projection, variant a)
|
|
CREDIT_LABEL = "pure.ratio.derived3.credit"
|
|
|
|
|
|
@pytest.mark.skipif(not os.path.exists(os.path.join(ROOT, "sweep_full.json")),
|
|
reason="sweep_full.json not present; run sweep_full.py")
|
|
def test_every_number_in_the_paper_matches_the_sweep():
|
|
r = subprocess.run([sys.executable, "verify_article.py"], cwd=ROOT,
|
|
capture_output=True, text=True, timeout=600)
|
|
tail = "\n".join(r.stdout.strip().splitlines()[-25:])
|
|
assert r.returncode == 0, f"the paper disagrees with the data:\n{tail}"
|
|
assert "every number stated in the paper matches" in r.stdout
|
|
assert r.stdout.count(" OK ") >= 72, r.stdout.count(" OK ")
|
|
|
|
|
|
@pytest.mark.skipif(not os.path.exists(os.path.join(ROOT, "sweep_full.json")),
|
|
reason="sweep_full.json not present")
|
|
def test_the_headline_claim_is_what_the_paper_says_it_is():
|
|
"""Guard against the one mistake that would matter most: the abstract
|
|
claiming a place the data does not support.
|
|
|
|
It caught exactly that. Until Chandrupatla's own method was added - a
|
|
reader asked why the owner of half the benchmark was missing - this test
|
|
asserted that a configuration of ours came first, and it passed only
|
|
because the method that beats us was absent from the field. The claims it
|
|
now guards (variant a, Valeri 2026-09-21): by TOTAL Chandrupatla is first
|
|
on the canonical set and we are within one percent of it; problem by
|
|
problem we beat it with significance; on the wide-bracket set and over
|
|
both sets it beats us. And the constant-free variant may not borrow the
|
|
projection's canonical numbers."""
|
|
import json
|
|
import math
|
|
|
|
from report_sweep import PROPOSED, head_to_head, rank
|
|
with open(os.path.join(ROOT, "sweep_full.json"), encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
by = {r["label"]: r for r in rows}
|
|
assert "chandrupatla" in by, "the sweep predates the missing baselines"
|
|
assert PROPOSED == "pure.ratio.derived3.project"
|
|
|
|
aps = rank(rows, "APS")
|
|
assert aps[0]["label"] == "chandrupatla", aps[0]["label"]
|
|
|
|
def both(lbl):
|
|
return by[lbl]["APS"]["total"] + by[lbl]["CHA"]["total"]
|
|
|
|
def tot(lbl, s):
|
|
return by[lbl][s]["total"]
|
|
|
|
# canonical set: parity by total with the best, a tie with TOMS 748
|
|
assert tot(PROPOSED, "APS") > tot("chandrupatla", "APS")
|
|
assert tot(PROPOSED, "APS") < 1.01 * tot("chandrupatla", "APS")
|
|
assert tot(PROPOSED, "APS") < tot("toms748", "APS") < 1.01 * tot(PROPOSED, "APS")
|
|
|
|
# ... and an edge problem by problem, with significance
|
|
w, _t, l, _ = head_to_head(rows, PROPOSED, "chandrupatla", "APS")
|
|
p = min(1.0, 2.0 * sum(math.comb(w + l, i) for i in range(min(w, l) + 1))
|
|
/ 2.0 ** (w + l))
|
|
assert w > 2 * l - 1 and p < 1e-3, (w, l, p)
|
|
|
|
# wide brackets and both sets: Chandrupatla is stronger, and the paper says so
|
|
assert tot(PROPOSED, "CHA") > tot("chandrupatla", "CHA")
|
|
assert both(PROPOSED) > both("chandrupatla"), "behind Chandrupatla"
|
|
for weaker in ("toms748", "brent", "itp", "ridders", "bisection"):
|
|
assert both(PROPOSED) < both(weaker), weaker
|
|
|
|
# the constant-free variant may not borrow the projection's canonical numbers
|
|
assert tot(CREDIT_LABEL, "APS") > tot("toms748", "APS")
|
|
assert tot(CREDIT_LABEL, "APS") > tot("brent", "APS")
|
|
assert tot(CREDIT_LABEL, "CHA") < tot(PROPOSED, "CHA")
|
|
|
|
# the claim the constant-free variant does own: the worst single problem,
|
|
# over the problems it solves, is close to bisection's and far below the
|
|
# classical methods'
|
|
def worst(lbl):
|
|
wst = 0
|
|
for setname in ("APS", "CHA"):
|
|
for n, ok in by[lbl][setname]["per"].values():
|
|
if ok:
|
|
wst = max(wst, n)
|
|
return wst
|
|
|
|
assert worst(CREDIT_LABEL) < 100, worst(CREDIT_LABEL)
|
|
assert worst(CREDIT_LABEL) < worst("brent")
|
|
assert worst(CREDIT_LABEL) < worst("toms748")
|
|
assert worst(CREDIT_LABEL) < worst(PROPOSED), (worst(CREDIT_LABEL), worst(PROPOSED))
|