108 lines
3.9 KiB
Python
108 lines
3.9 KiB
Python
"""Pins the regime findings of stage 7.
|
|
|
|
Three conclusions were drawn from that analysis and used to decide what to do
|
|
next. Each is asserted here so that a re-run of the sweep which moves them
|
|
fails the build instead of quietly invalidating the conclusion.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, ROOT)
|
|
|
|
SWEEP = os.path.join(ROOT, "sweep_full.json")
|
|
pytestmark = pytest.mark.skipif(not os.path.exists(SWEEP),
|
|
reason="sweep_full.json not present")
|
|
|
|
SCALES = ("start", "bracket", "derived", "derived3")
|
|
MAXFEV = 200
|
|
|
|
|
|
def load():
|
|
with open(SWEEP, encoding="utf8") as fh:
|
|
return {r["label"]: r for r in json.load(fh)}
|
|
|
|
|
|
def cost(rows, sim, scale, pid):
|
|
r = rows[f"pure.{sim}.{scale}.project"]
|
|
d = r["APS"]["per"].get(pid) or r["CHA"]["per"].get(pid)
|
|
return d[0] if d[1] else MAXFEV
|
|
|
|
|
|
def all_pids(rows):
|
|
r = rows["pure.ratio.derived3.project"]
|
|
return list(r["APS"]["per"]) + list(r["CHA"]["per"])
|
|
|
|
|
|
def test_the_scale_axis_has_little_headroom():
|
|
"""5.3%. If this ever grows a lot, the scale is worth revisiting; as it
|
|
stands it is not where the regimes live."""
|
|
rows = load()
|
|
pids = all_pids(rows)
|
|
ours = sum(cost(rows, "ratio", "derived3", p) for p in pids)
|
|
oracle = sum(min(cost(rows, "ratio", s, p) for s in SCALES) for p in pids)
|
|
head = (ours - oracle) / ours
|
|
assert 0.0 <= head < 0.10, head
|
|
|
|
|
|
def test_cbrt_beats_us_consistently_on_the_fun4_family():
|
|
"""The signature of a real regime rather than luck: the same function at
|
|
five different parameters all prefer the same similarity, by a wide
|
|
margin. This is the finding that keeps the line open."""
|
|
rows = load()
|
|
fam = [f"fun4.{i}" for i in range(1, 6)]
|
|
better = 0
|
|
for pid in fam:
|
|
ours = cost(rows, "ratio", "derived3", pid)
|
|
cb = cost(rows, "cbrt", "derived3", pid)
|
|
if cb < ours:
|
|
better += 1
|
|
assert better == 5, f"only {better}/5 of the fun4 family prefer cbrt"
|
|
tot_ours = sum(cost(rows, "ratio", "derived3", p) for p in fam)
|
|
tot_cb = sum(cost(rows, "cbrt", "derived3", p) for p in fam)
|
|
assert tot_cb < 0.6 * tot_ours, (tot_cb, tot_ours)
|
|
|
|
|
|
def test_switching_by_benchmark_set_is_worse_than_not_switching():
|
|
"""The finding that closed the obvious exploit: cbrt wins the fun3/4/5
|
|
families but loses the rest of Chandrupatla, so a set-level switch costs
|
|
more than a single fixed choice. Any future 'regime' proposal has to beat
|
|
this, not merely beat one family."""
|
|
rows = load()
|
|
single = (rows["pure.ratio.derived3.project"]["APS"]["total"]
|
|
+ rows["pure.ratio.derived3.project"]["CHA"]["total"])
|
|
switched = (rows["pure.ratio.derived3.project"]["APS"]["total"]
|
|
+ rows["pure.cbrt.derived3.project"]["CHA"]["total"])
|
|
assert switched > single, (switched, single)
|
|
|
|
|
|
def test_the_proposed_similarity_wins_most_problems_outright():
|
|
"""155 of 199. The per-problem oracle over similarities is an upper
|
|
bound on regime detection, not a method, and most of the field is already
|
|
ours without it."""
|
|
rows = load()
|
|
pids = all_pids(rows)
|
|
sims = ("ratio", "sqrt", "cbrt", "rational", "ratio_soft", "exp_abs",
|
|
"rat_sq", "exp_sq", "cosine")
|
|
wins = 0
|
|
for p in pids:
|
|
c = {s: cost(rows, s, "derived3", p) for s in sims}
|
|
if c["ratio"] == min(c.values()):
|
|
wins += 1
|
|
assert wins >= 150, wins
|
|
|
|
|
|
def test_stage7_runs():
|
|
"""The analysis script itself must still execute against the current
|
|
sweep file."""
|
|
import subprocess
|
|
r = subprocess.run([sys.executable, "stage7_regimes.py"], cwd=ROOT,
|
|
capture_output=True, text=True, timeout=600)
|
|
assert r.returncode == 0, r.stderr[-800:]
|
|
assert "SETTLING THE TWO SUSPICIONS" in r.stdout
|