baseline: the published sim-numerics artifact as it stands (490 tests, verify_article.py checks all 72 paper numbers against sweep_full.json)

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_article_numbers.py

@@ -0,0 +1,89 @@
+"""The paper's numbers are part of the regression.
+
+A table transcribed by hand drifts from its data the first time the data is
+regenerated.  verify_article.py asserts all 72 stated numbers against
+sweep_full.json; this makes that check part of the suite, so a re-run of the
+sweep that moves a number fails the build instead of quietly invalidating the
+paper.
+"""
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+
+import pytest
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# the other end of the trade the paper states: better average, one constant
+PROJ_LABEL = "pure.ratio.derived3.project"
+
+
+@pytest.mark.skipif(not os.path.exists(os.path.join(ROOT, "sweep_full.json")),
+                    reason="sweep_full.json not present; run sweep_full.py")
+def test_every_number_in_the_paper_matches_the_sweep():
+    r = subprocess.run([sys.executable, "verify_article.py"], cwd=ROOT,
+                       capture_output=True, text=True, timeout=600)
+    tail = "\n".join(r.stdout.strip().splitlines()[-25:])
+    assert r.returncode == 0, f"the paper disagrees with the data:\n{tail}"
+    assert "every number stated in the paper matches" in r.stdout
+    assert r.stdout.count("  OK ") >= 72, r.stdout.count("  OK ")
+
+
+@pytest.mark.skipif(not os.path.exists(os.path.join(ROOT, "sweep_full.json")),
+                    reason="sweep_full.json not present")
+def test_the_headline_claim_is_what_the_paper_says_it_is():
+    """Guard against the one mistake that would matter most: the abstract
+    claiming a place the data does not support.
+
+    It caught exactly that.  Until Chandrupatla's own method was added - a
+    reader asked why the owner of half the benchmark was missing - this test
+    asserted that a configuration of ours came first, and it passed only
+    because the method that beats us was absent from the field.  The claim it
+    now guards is the corrected one: we are not first, Chandrupatla is, and
+    our margin behind it on the canonical set is under one percent."""
+    import json
+
+    from report_sweep import PROPOSED, rank
+    with open(os.path.join(ROOT, "sweep_full.json"), encoding="utf8") as fh:
+        rows = json.load(fh)
+    by = {r["label"]: r for r in rows}
+    assert "chandrupatla" in by, "the sweep predates the missing baselines"
+
+    aps = rank(rows, "APS")
+    assert aps[0]["label"] == "chandrupatla", aps[0]["label"]
+
+    def both(lbl):
+        return by[lbl]["APS"]["total"] + by[lbl]["CHA"]["total"]
+
+    # What the paper may claim for the credit method, and nothing more.
+    # It is NOT within a percent of the best on Alefeld-Potra-Shi - that was
+    # true of the projection variant, which carries a constant.  Buying
+    # constant-freedom and a near-bisection worst case costs average
+    # performance, and the test pins the direction of that trade so the
+    # abstract cannot quietly reclaim the projection variant's numbers.
+    assert both(PROPOSED) > both("chandrupatla"), "behind Chandrupatla"
+    for weaker in ("toms748", "brent", "itp", "ridders", "bisection"):
+        assert both(PROPOSED) < both(weaker), weaker
+
+    # On the canonical set alone it is behind the classical methods; saying
+    # otherwise would be quoting the projection variant.
+    assert by[PROPOSED]["APS"]["total"] > by["toms748"]["APS"]["total"]
+    assert by[PROPOSED]["APS"]["total"] > by["brent"]["APS"]["total"]
+
+    # The claim it does own: the worst single problem, over the problems it
+    # solves, is close to bisection's and far below the classical methods'.
+    def worst(lbl):
+        w = 0
+        for setname in ("APS", "CHA"):
+            for n, ok in by[lbl][setname]["per"].values():
+                if ok:
+                    w = max(w, n)
+        return w
+
+    assert worst(PROPOSED) < 100, worst(PROPOSED)
+    assert worst(PROPOSED) < worst("brent")
+    assert worst(PROPOSED) < worst("toms748")
+    assert worst(PROPOSED) < worst(PROJ_LABEL), (worst(PROPOSED),
+                                                 worst(PROJ_LABEL))