90 lines
3.9 KiB
Python
90 lines
3.9 KiB
Python
"""The paper's numbers are part of the regression.
|
|
|
|
A table transcribed by hand drifts from its data the first time the data is
|
|
regenerated. verify_article.py asserts all 72 stated numbers against
|
|
sweep_full.json; this makes that check part of the suite, so a re-run of the
|
|
sweep that moves a number fails the build instead of quietly invalidating the
|
|
paper.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
# the other end of the trade the paper states: better average, one constant
|
|
PROJ_LABEL = "pure.ratio.derived3.project"
|
|
|
|
|
|
@pytest.mark.skipif(not os.path.exists(os.path.join(ROOT, "sweep_full.json")),
|
|
reason="sweep_full.json not present; run sweep_full.py")
|
|
def test_every_number_in_the_paper_matches_the_sweep():
|
|
r = subprocess.run([sys.executable, "verify_article.py"], cwd=ROOT,
|
|
capture_output=True, text=True, timeout=600)
|
|
tail = "\n".join(r.stdout.strip().splitlines()[-25:])
|
|
assert r.returncode == 0, f"the paper disagrees with the data:\n{tail}"
|
|
assert "every number stated in the paper matches" in r.stdout
|
|
assert r.stdout.count(" OK ") >= 72, r.stdout.count(" OK ")
|
|
|
|
|
|
@pytest.mark.skipif(not os.path.exists(os.path.join(ROOT, "sweep_full.json")),
|
|
reason="sweep_full.json not present")
|
|
def test_the_headline_claim_is_what_the_paper_says_it_is():
|
|
"""Guard against the one mistake that would matter most: the abstract
|
|
claiming a place the data does not support.
|
|
|
|
It caught exactly that. Until Chandrupatla's own method was added - a
|
|
reader asked why the owner of half the benchmark was missing - this test
|
|
asserted that a configuration of ours came first, and it passed only
|
|
because the method that beats us was absent from the field. The claim it
|
|
now guards is the corrected one: we are not first, Chandrupatla is, and
|
|
our margin behind it on the canonical set is under one percent."""
|
|
import json
|
|
|
|
from report_sweep import PROPOSED, rank
|
|
with open(os.path.join(ROOT, "sweep_full.json"), encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
by = {r["label"]: r for r in rows}
|
|
assert "chandrupatla" in by, "the sweep predates the missing baselines"
|
|
|
|
aps = rank(rows, "APS")
|
|
assert aps[0]["label"] == "chandrupatla", aps[0]["label"]
|
|
|
|
def both(lbl):
|
|
return by[lbl]["APS"]["total"] + by[lbl]["CHA"]["total"]
|
|
|
|
# What the paper may claim for the credit method, and nothing more.
|
|
# It is NOT within a percent of the best on Alefeld-Potra-Shi - that was
|
|
# true of the projection variant, which carries a constant. Buying
|
|
# constant-freedom and a near-bisection worst case costs average
|
|
# performance, and the test pins the direction of that trade so the
|
|
# abstract cannot quietly reclaim the projection variant's numbers.
|
|
assert both(PROPOSED) > both("chandrupatla"), "behind Chandrupatla"
|
|
for weaker in ("toms748", "brent", "itp", "ridders", "bisection"):
|
|
assert both(PROPOSED) < both(weaker), weaker
|
|
|
|
# On the canonical set alone it is behind the classical methods; saying
|
|
# otherwise would be quoting the projection variant.
|
|
assert by[PROPOSED]["APS"]["total"] > by["toms748"]["APS"]["total"]
|
|
assert by[PROPOSED]["APS"]["total"] > by["brent"]["APS"]["total"]
|
|
|
|
# The claim it does own: the worst single problem, over the problems it
|
|
# solves, is close to bisection's and far below the classical methods'.
|
|
def worst(lbl):
|
|
w = 0
|
|
for setname in ("APS", "CHA"):
|
|
for n, ok in by[lbl][setname]["per"].values():
|
|
if ok:
|
|
w = max(w, n)
|
|
return w
|
|
|
|
assert worst(PROPOSED) < 100, worst(PROPOSED)
|
|
assert worst(PROPOSED) < worst("brent")
|
|
assert worst(PROPOSED) < worst("toms748")
|
|
assert worst(PROPOSED) < worst(PROJ_LABEL), (worst(PROPOSED),
|
|
worst(PROJ_LABEL))
|