baseline: the published sim-numerics artifact as it stands (490 tests, verify_article.py checks all 72 paper numbers against sweep_full.json)

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_regimes.py

@@ -0,0 +1,107 @@
+"""Pins the regime findings of stage 7.
+
+Three conclusions were drawn from that analysis and used to decide what to do
+next.  Each is asserted here so that a re-run of the sweep which moves them
+fails the build instead of quietly invalidating the conclusion.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+import pytest
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+SWEEP = os.path.join(ROOT, "sweep_full.json")
+pytestmark = pytest.mark.skipif(not os.path.exists(SWEEP),
+                                reason="sweep_full.json not present")
+
+SCALES = ("start", "bracket", "derived", "derived3")
+MAXFEV = 200
+
+
+def load():
+    with open(SWEEP, encoding="utf8") as fh:
+        return {r["label"]: r for r in json.load(fh)}
+
+
+def cost(rows, sim, scale, pid):
+    r = rows[f"pure.{sim}.{scale}.project"]
+    d = r["APS"]["per"].get(pid) or r["CHA"]["per"].get(pid)
+    return d[0] if d[1] else MAXFEV
+
+
+def all_pids(rows):
+    r = rows["pure.ratio.derived3.project"]
+    return list(r["APS"]["per"]) + list(r["CHA"]["per"])
+
+
+def test_the_scale_axis_has_little_headroom():
+    """5.3%.  If this ever grows a lot, the scale is worth revisiting; as it
+    stands it is not where the regimes live."""
+    rows = load()
+    pids = all_pids(rows)
+    ours = sum(cost(rows, "ratio", "derived3", p) for p in pids)
+    oracle = sum(min(cost(rows, "ratio", s, p) for s in SCALES) for p in pids)
+    head = (ours - oracle) / ours
+    assert 0.0 <= head < 0.10, head
+
+
+def test_cbrt_beats_us_consistently_on_the_fun4_family():
+    """The signature of a real regime rather than luck: the same function at
+    five different parameters all prefer the same similarity, by a wide
+    margin.  This is the finding that keeps the line open."""
+    rows = load()
+    fam = [f"fun4.{i}" for i in range(1, 6)]
+    better = 0
+    for pid in fam:
+        ours = cost(rows, "ratio", "derived3", pid)
+        cb = cost(rows, "cbrt", "derived3", pid)
+        if cb < ours:
+            better += 1
+    assert better == 5, f"only {better}/5 of the fun4 family prefer cbrt"
+    tot_ours = sum(cost(rows, "ratio", "derived3", p) for p in fam)
+    tot_cb = sum(cost(rows, "cbrt", "derived3", p) for p in fam)
+    assert tot_cb < 0.6 * tot_ours, (tot_cb, tot_ours)
+
+
+def test_switching_by_benchmark_set_is_worse_than_not_switching():
+    """The finding that closed the obvious exploit: cbrt wins the fun3/4/5
+    families but loses the rest of Chandrupatla, so a set-level switch costs
+    more than a single fixed choice.  Any future 'regime' proposal has to beat
+    this, not merely beat one family."""
+    rows = load()
+    single = (rows["pure.ratio.derived3.project"]["APS"]["total"]
+              + rows["pure.ratio.derived3.project"]["CHA"]["total"])
+    switched = (rows["pure.ratio.derived3.project"]["APS"]["total"]
+                + rows["pure.cbrt.derived3.project"]["CHA"]["total"])
+    assert switched > single, (switched, single)
+
+
+def test_the_proposed_similarity_wins_most_problems_outright():
+    """155 of 199.  The per-problem oracle over similarities is an upper
+    bound on regime detection, not a method, and most of the field is already
+    ours without it."""
+    rows = load()
+    pids = all_pids(rows)
+    sims = ("ratio", "sqrt", "cbrt", "rational", "ratio_soft", "exp_abs",
+            "rat_sq", "exp_sq", "cosine")
+    wins = 0
+    for p in pids:
+        c = {s: cost(rows, s, "derived3", p) for s in sims}
+        if c["ratio"] == min(c.values()):
+            wins += 1
+    assert wins >= 150, wins
+
+
+def test_stage7_runs():
+    """The analysis script itself must still execute against the current
+    sweep file."""
+    import subprocess
+    r = subprocess.run([sys.executable, "stage7_regimes.py"], cwd=ROOT,
+                       capture_output=True, text=True, timeout=600)
+    assert r.returncode == 0, r.stderr[-800:]
+    assert "SETTLING THE TWO SUSPICIONS" in r.stdout