564 lines
27 KiB
Python
564 lines
27 KiB
Python
"""Check every number the paper states against sweep_full.json.
|
|
|
|
A paper is a claim about data; transcribing its tables by hand is how a claim
|
|
stops matching its data. A failure here means the paper is wrong, not the
|
|
test.
|
|
|
|
The proposed method is the scheduled projection with its multiplier DECLARED as
|
|
the one free constant (Valeri, 2026-09-21, variant a); the earned projection
|
|
(credit) is the constant-free variant. Both are checked. Baselines are the
|
|
library alone since 2026-09-21: Chandrupatla = scipy elementwise.find_root,
|
|
Brent = brentq.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
from report_sweep import CREDIT, PROPOSED, head_to_head, place, rank
|
|
|
|
FAILS = []
|
|
PROJ = "pure.ratio.derived3.project"
|
|
assert PROPOSED == PROJ, "the paper proposes the projection (variant a)"
|
|
|
|
|
|
def check(label, got, want):
|
|
ok = got == want
|
|
print(f" {'OK ' if ok else 'BAD'} {label:58s} got {got!r:>16} "
|
|
f"want {want!r}")
|
|
if not ok:
|
|
FAILS.append(label)
|
|
|
|
|
|
def row(rows, setname, lbl):
|
|
return next(r for r in rows if r["label"] == lbl)[setname]
|
|
|
|
|
|
def main():
|
|
with open("sweep_full.json", encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
|
|
# Guard against quoting places from a field that is missing a competitor.
|
|
# It has already happened once: before Chandrupatla's own method was
|
|
# added, every ranking assertion here passed only because the method that
|
|
# beats us was absent.
|
|
labels = {r["label"] for r in rows}
|
|
missing = {"chandrupatla", "dekker", PROPOSED} - labels
|
|
if missing:
|
|
print(f" !! THE SWEEP IS STALE: missing {sorted(missing)}. "
|
|
"Re-run sweep_full.py.\n")
|
|
FAILS.append("sweep is stale")
|
|
|
|
check("number of configurations", len(rows), 264)
|
|
|
|
# ---- the proposed method: projection, one declared constant ----------
|
|
print("\nThe proposed method (projection, multiplier declared):")
|
|
for setname, pl, tot, sol in (("APS", 3, 2806, 153),
|
|
("CHA", 47, 3220, 40)):
|
|
d = row(rows, setname, PROPOSED)
|
|
check(f"{setname} place", place(rows, setname, PROPOSED), pl)
|
|
check(f"{setname} total", d["total"], tot)
|
|
check(f"{setname} solved", d["solved"], sol)
|
|
check("both sets", row(rows, "APS", PROPOSED)["total"]
|
|
+ row(rows, "CHA", PROPOSED)["total"], 6026)
|
|
|
|
print("\nHead to head of the proposed method:")
|
|
for setname, opp, want in [
|
|
("APS", "chandrupatla", (87, 24, 42)),
|
|
("APS", "toms748", (86, 16, 51)),
|
|
("APS", "brent", (102, 11, 40)),
|
|
("APS", "itp", (132, 3, 18)),
|
|
("APS", "itp_oracle", (121, 4, 28)),
|
|
("APS", "bisection", (152, 1, 0)),
|
|
("CHA", "chandrupatla", (9, 5, 26)),
|
|
("CHA", "toms748", (31, 1, 8)),
|
|
("CHA", "brent", (30, 4, 4)),
|
|
("CHA", "itp", (17, 0, 23)),
|
|
("CHA", "itp_oracle", (16, 0, 24))]:
|
|
w, t, l, _ = head_to_head(rows, PROPOSED, opp, setname)
|
|
check(f"{setname} vs {opp}", (w, t, l), want)
|
|
|
|
def sign_p(w, l):
|
|
import math as _m
|
|
k, n = min(w, l), w + l
|
|
return min(1.0, 2.0 * sum(_m.comb(n, i) for i in range(k + 1)) / 2.0 ** n)
|
|
w, _t, l, _ = head_to_head(rows, PROPOSED, "chandrupatla", "APS")
|
|
check("APS vs Chandrupatla sign test p < 1e-4", sign_p(w, l) < 1e-4, True)
|
|
w, _t, l, _ = head_to_head(rows, PROPOSED, "chandrupatla", "CHA")
|
|
check("CHA vs Chandrupatla sign test p (rounded)", round(sign_p(w, l), 3), 0.006)
|
|
|
|
# ---- the constant-free variant ---------------------------------------
|
|
print("\nThe constant-free variant (earned projection):")
|
|
for setname, pl, tot in (("APS", 29, 3568), ("CHA", 10, 3020)):
|
|
check(f"credit {setname} place", place(rows, setname, CREDIT), pl)
|
|
check(f"credit {setname} total", row(rows, setname, CREDIT)["total"], tot)
|
|
check("credit both sets", row(rows, "APS", CREDIT)["total"]
|
|
+ row(rows, "CHA", CREDIT)["total"], 6588)
|
|
check("credit costs 762 more on the canonical set",
|
|
row(rows, "APS", CREDIT)["total"] - row(rows, "APS", PROPOSED)["total"], 762)
|
|
for setname, opp, want in [("APS", "chandrupatla", (79, 3, 71)),
|
|
("APS", "toms748", (77, 9, 67)),
|
|
("APS", "brent", (78, 5, 70))]:
|
|
w, t, l, _ = head_to_head(rows, CREDIT, opp, setname)
|
|
check(f"credit {setname} vs {opp}", (w, t, l), want)
|
|
w, _t, l, _ = head_to_head(rows, CREDIT, "chandrupatla", "APS")
|
|
check("credit APS vs Chandrupatla sign test p (rounded)", round(sign_p(w, l), 2), 0.57)
|
|
|
|
# ---- what the earlier, order-destroying guards cost --------------------
|
|
print("\nThe guards that replace the step instead of projecting it:")
|
|
check("APS unguarded", row(rows, "APS", "pure.ratio.derived3.none")["total"],
|
|
2806)
|
|
cha_none = row(rows, "CHA", "pure.ratio.derived3.none")
|
|
check("CHA unguarded total", cha_none["total"], 4104)
|
|
check("CHA unguarded solved", cha_none["solved"], 35)
|
|
check("CHA unguarded bound violations", cha_none["over"], 10)
|
|
check("APS bracket-guard", row(rows, "APS", "pure.ratio.derived3.bracket")["total"],
|
|
7260)
|
|
check("APS both-guard", row(rows, "APS", "pure.ratio.derived3.both")["total"],
|
|
7239)
|
|
|
|
# ---- the field --------------------------------------------------------
|
|
print("\nAlefeld-Potra-Shi:")
|
|
check("APS winner", rank(rows, "APS")[0]["label"], "chandrupatla")
|
|
for lbl, pl, tot in [("chandrupatla", 1, 2787), ("toms748", 4, 2820),
|
|
("brent", 5, 2889), ("dekker", 12, 3158),
|
|
("itp_oracle", 13, 3260), ("ridders", 20, 3313),
|
|
("itp", 34, 3957), ("illinois", 73, 7045),
|
|
("bisection", 79, 7530), ("regula_falsi", 244, 10560)]:
|
|
check(f"APS {lbl} place", place(rows, "APS", lbl), pl)
|
|
check(f"APS {lbl} total", row(rows, "APS", lbl)["total"], tot)
|
|
check("APS regula falsi solved", row(rows, "APS", "regula_falsi")["solved"],
|
|
127)
|
|
|
|
print("\nChandrupatla:")
|
|
top = rank(rows, "CHA")[0]
|
|
check("CHA rank 1 label", top["label"], "pure.sqrt.bracket.project")
|
|
check("CHA rank 1 solves one more than anyone else", top["CHA"]["solved"], 41)
|
|
for lbl, pl, tot in [("chandrupatla", 2, 2441), ("itp_oracle", 3, 2611),
|
|
("itp", 6, 2857), ("bisection", 16, 3090),
|
|
("toms748", 164, 4000), ("brent", 226, 4115)]:
|
|
check(f"CHA {lbl} place", place(rows, "CHA", lbl), pl)
|
|
check(f"CHA {lbl} total", row(rows, "CHA", lbl)["total"], tot)
|
|
check("CHA brent solved", row(rows, "CHA", "brent")["solved"], 38)
|
|
|
|
ours_both = (row(rows, "APS", PROPOSED)["total"]
|
|
+ row(rows, "CHA", PROPOSED)["total"])
|
|
cha_both = (row(rows, "APS", "chandrupatla")["total"]
|
|
+ row(rows, "CHA", "chandrupatla")["total"])
|
|
check("chandrupatla, both sets", cha_both, 5228)
|
|
check("Chandrupatla beats us on both sets", cha_both < ours_both, True)
|
|
|
|
# ---- the regime findings ---------------------------------------------
|
|
print("\nNegative results, the regime claims:")
|
|
check("CHA cbrt over the whole set",
|
|
row(rows, "CHA", "pure.cbrt.derived3.project")["total"], 4099)
|
|
switched = (row(rows, "APS", PROJ)["total"]
|
|
+ row(rows, "CHA", "pure.cbrt.derived3.project")["total"])
|
|
single = row(rows, "APS", PROJ)["total"] + row(rows, "CHA", PROJ)["total"]
|
|
check("single fixed choice, both sets", single, 6026)
|
|
check("switch by benchmark set, both sets", switched, 6905)
|
|
check("switching is worse than not switching", switched > single, True)
|
|
for pid, ours_n, cb_n in (("fun4.1", 89, 32), ("fun4.2", 93, 32),
|
|
("fun4.3", 113, 40), ("fun4.4", 125, 47),
|
|
("fun4.5", 153, 59)):
|
|
o = row(rows, "CHA", PROJ)["per"][pid]
|
|
c = row(rows, "CHA", "pure.cbrt.derived3.project")["per"][pid]
|
|
check(f"{pid} projection variant", o[0], ours_n)
|
|
check(f"{pid} cbrt", c[0], cb_n)
|
|
|
|
# ---- the law across the grid ------------------------------------------
|
|
print("\nThe law across the grid:")
|
|
lin = ("ratio", "rational", "exp_abs")
|
|
quad = ("rat_sq", "exp_sq", "cosine")
|
|
want = {("derived3", "lin"): 6809, ("derived3", "quad"): 22294,
|
|
("derived", "lin"): 6906, ("derived", "quad"): 22167,
|
|
("bracket", "lin"): 10263, ("bracket", "quad"): 19137,
|
|
("start", "lin"): 16102, ("start", "quad"): 15345}
|
|
for scale in ("derived3", "derived", "bracket", "start"):
|
|
for group, gname in ((lin, "lin"), (quad, "quad")):
|
|
tot = cnt = 0
|
|
for r in rows:
|
|
if r["spec"][0] != "pure":
|
|
continue
|
|
_, sim, sc, gu = r["spec"]
|
|
if sc != scale or gu != "step" or sim not in group:
|
|
continue
|
|
cnt += 1
|
|
tot += r["APS"]["total"] + r["CHA"]["total"]
|
|
check(f"mean total, scale={scale}, {gname}",
|
|
round(tot / cnt), want[(scale, gname)])
|
|
|
|
best_by_sim, best_by_scale = {}, {}
|
|
for r in rows:
|
|
if r["spec"][0] != "pure" or r["spec"][3] != "step":
|
|
continue
|
|
t = r["APS"]["total"] + r["CHA"]["total"]
|
|
best_by_sim[r["spec"][1]] = min(best_by_sim.get(r["spec"][1], 10 ** 9), t)
|
|
best_by_scale[r["spec"][2]] = min(best_by_scale.get(r["spec"][2], 10 ** 9), t)
|
|
check("every c=1 similarity beats every c=0 similarity",
|
|
max(best_by_sim[s] for s in lin) < min(best_by_sim[s] for s in quad),
|
|
True)
|
|
check("similarity ranking",
|
|
[s for s, _ in sorted(best_by_sim.items(), key=lambda kv: kv[1])],
|
|
["ratio", "exp_abs", "rational", "ratio_soft", "sqrt", "cbrt",
|
|
"exp_sq", "rat_sq", "cosine"])
|
|
for sim, v in [("ratio", 6496), ("exp_abs", 6822), ("rational", 7038),
|
|
("ratio_soft", 9502), ("sqrt", 10497), ("cbrt", 11941),
|
|
("exp_sq", 14314), ("rat_sq", 15102), ("cosine", 16394)]:
|
|
check(f"similarity {sim} best", best_by_sim[sim], v)
|
|
check("scale ranking",
|
|
[s for s, _ in sorted(best_by_scale.items(), key=lambda kv: kv[1])],
|
|
["derived3", "derived", "bracket", "start"])
|
|
for sc, v in [("derived3", 6496), ("derived", 6628), ("bracket", 9081),
|
|
("start", 14540)]:
|
|
check(f"scale {sc} best", best_by_scale[sc], v)
|
|
|
|
verify_placement_modes()
|
|
verify_per_place()
|
|
verify_decision_place()
|
|
verify_gap_decomposition()
|
|
verify_safeguard_and_tables()
|
|
|
|
print()
|
|
if FAILS:
|
|
print(f"{len(FAILS)} STATED NUMBERS DO NOT MATCH THE DATA:")
|
|
for f in FAILS:
|
|
print(f" {f}")
|
|
sys.exit(1)
|
|
print("every number stated in the paper matches the sweep data.")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Section "where the similarity is read" (PREREGs of 2026-09-20/21). Each new
|
|
# grid lives in its OWN sweep file so the published sweep_full.json stays the
|
|
# artefact of the original claims; every number the new section quotes is
|
|
# re-checked here against that file.
|
|
# --------------------------------------------------------------------------
|
|
PLACEMENT = {
|
|
# file label APS CHA
|
|
"sweep_slope.json": ("slope.ratio.derived3.project", 2848, 3074),
|
|
"sweep_points.json": ("pure+shape+slope.ratio.derived3.safe", 2527, 3108),
|
|
"sweep_direction.json": ("pure+dissim+slope.ratio.derived3.project", 2726, 2785),
|
|
"sweep_combo.json": ("pure+shape+dissim+slope.ratio.derived3.safe", 2884, 2859),
|
|
"sweep_gate.json": ("gate.ratio.derived.project", 4449, 2991),
|
|
"sweep_log.json": ("log.ratio_soft.derived.safe", 6940, 3474),
|
|
}
|
|
# The per-place table of section "the similarity function belongs to the place":
|
|
# seven PAIRED comparisons in sweep_order.json. Checked as pairs, because a pair is
|
|
# what the pre-registration licensed and a single cell would prove nothing.
|
|
ORDER_PAIRS = {
|
|
# guard (APS m=1, APS measured, CHA m=1, CHA measured)
|
|
"both": (4522, 3675, 3387, 3216),
|
|
"bracket": (4606, 3542, 3571, 3303),
|
|
"credit": (5190, 4686, 3013, 2884),
|
|
"project": (6932, 6364, 4794, 4559),
|
|
"safe": (5987, 5229, 4108, 3923),
|
|
"step": (6435, 5551, 4257, 4022),
|
|
"none": (10676, 10043, 8265, 7794),
|
|
}
|
|
# the negative item: the order does not cure false position
|
|
ORDER_FALSEPOS = {("rational", "far", "none", "none"): (127, 10676),
|
|
("rational", "far", "none", "order"): (128, 10043)}
|
|
# and the harmful direction, quoted in the same subsection
|
|
ORDER_SLOPE_HARM = {("ratio", "derived3", "none", "none"): 2806,
|
|
("ratio", "derived3", "none", "order"): 2854}
|
|
|
|
|
|
def verify_per_place():
|
|
print("\nThe per-place reading (sweep_order.json):")
|
|
if not os.path.exists("sweep_order.json"):
|
|
print(" !! sweep_order.json absent — section 'belongs to the place' unverifiable")
|
|
FAILS.append("missing sweep_order.json")
|
|
return
|
|
with open("sweep_order.json", encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
|
|
def cell(sim, sc, gu, ex):
|
|
for r in rows:
|
|
s = r["spec"]
|
|
if (s[1], s[2], s[3], s[8] if len(s) > 8 else "none") == (sim, sc, gu, ex):
|
|
return r
|
|
return None
|
|
|
|
cheaper = 0
|
|
for gu, (a1, a2, c1, c2) in ORDER_PAIRS.items():
|
|
p = cell("rational", "far", gu, "none")
|
|
q = cell("rational", "far", gu, "order")
|
|
if p is None or q is None:
|
|
FAILS.append(f"order pair {gu} missing from sweep_order.json")
|
|
print(f" !! pair {gu} missing — stale sweep")
|
|
continue
|
|
check(f"far+rational {gu} APS m=1", p["APS"]["total"], a1)
|
|
check(f"far+rational {gu} APS order", q["APS"]["total"], a2)
|
|
check(f"far+rational {gu} CHA m=1", p["CHA"]["total"], c1)
|
|
check(f"far+rational {gu} CHA order", q["CHA"]["total"], c2)
|
|
cheaper += (q["APS"]["total"] + q["CHA"]["total"]
|
|
< p["APS"]["total"] + p["CHA"]["total"])
|
|
check("cheaper in 7 of 7 safeguards", cheaper, 7)
|
|
for (sim, sc, gu, ex), (solved, total) in ORDER_FALSEPOS.items():
|
|
r = cell(sim, sc, gu, ex)
|
|
if r is None:
|
|
FAILS.append(f"false-position cell {ex} missing")
|
|
continue
|
|
check(f"false position {ex} solved", r["APS"]["solved"], solved)
|
|
check(f"false position {ex} APS", r["APS"]["total"], total)
|
|
for (sim, sc, gu, ex), want in ORDER_SLOPE_HARM.items():
|
|
r = cell(sim, sc, gu, ex)
|
|
if r is None:
|
|
FAILS.append(f"slope-harm cell {ex} missing")
|
|
continue
|
|
check(f"slope reading {ex} APS", r["APS"]["total"], want)
|
|
fam = [r for r in rows if r["spec"][3] in ("project", "safe", "credit")]
|
|
check("guarantee family violations", sum(r["APS"]["over"] + r["CHA"]["over"]
|
|
for r in fam), 0)
|
|
|
|
|
|
DECISION_CELLS = {
|
|
# (similarity, prestep, guard): {set: total}
|
|
("ratio", "band", "project"): {"APS": 5432, "CHA": 3066},
|
|
("ratio", "band", "credit"): {"CHA": 2899},
|
|
("ratio", "none", "project"): {"APS": 2806},
|
|
("ratio", "defect", "project"): {"APS": 2523, "CHA": 3210},
|
|
("ratio_soft", "simpair", "project"): {"CHA": 2868},
|
|
}
|
|
DECISION_RATES = {"band": 63.6, "defect": 7.0}
|
|
|
|
|
|
def verify_decision_place():
|
|
print("\nThe decision place (sweep_decision.json):")
|
|
if not os.path.exists("sweep_decision.json"):
|
|
print(" !! sweep_decision.json absent — section 'decision place' unverifiable")
|
|
FAILS.append("missing sweep_decision.json")
|
|
return
|
|
with open("sweep_decision.json", encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
by = {(r["spec"][1], r["spec"][5], r["spec"][3]): r for r in rows}
|
|
for k, sets in DECISION_CELLS.items():
|
|
r = by.get(k)
|
|
if r is None:
|
|
FAILS.append(f"decision cell {k} missing")
|
|
print(f" !! {k} missing — stale sweep")
|
|
continue
|
|
for setname, want in sets.items():
|
|
check(f"{'.'.join(k)} {setname}", r[setname]["total"], want)
|
|
d = by[("ratio", "defect", "project")]
|
|
check("defect APS solved", d["APS"]["solved"], 153)
|
|
check("defect both sets", d["APS"]["total"] + d["CHA"]["total"], 5733)
|
|
check("defect is the new canonical best (< 2527)", d["APS"]["total"] < 2527, True)
|
|
fam = [r for r in rows if r["spec"][3] in ("project", "safe", "credit")]
|
|
check("decision grid guarantee violations", sum(r["APS"]["over"] + r["CHA"]["over"]
|
|
for r in fam), 0)
|
|
# the refusal rates are recomputed, not transcribed
|
|
from scipy.optimize._tstutils import _APS_TESTS_DICTS, _CHANDRUPATLA_TESTS_DICTS
|
|
from simroot.pure import pure_root
|
|
probs = [(float(q["bracket"][0]), float(q["bracket"][1]),
|
|
(lambda ff, aa: lambda t: ff(t, *aa))(q["f"], q.get("args", ())))
|
|
for q in _APS_TESTS_DICTS]
|
|
probs += [(float(q["bracket"][0]), float(q["bracket"][1]),
|
|
(lambda ff: lambda t: ff(t))(q["f"])) for q in _CHANDRUPATLA_TESTS_DICTS]
|
|
for mode, want in DECISION_RATES.items():
|
|
ref = nfev = 0
|
|
for a, b, f in probs:
|
|
st = {}
|
|
try:
|
|
_x, n, _c = pure_root(f, a, b, 1e-12, 200, similarity="ratio",
|
|
scale="derived3", guard="project", prestep=mode,
|
|
stats=st)
|
|
except Exception:
|
|
continue
|
|
ref += st.get("pre_refused", 0)
|
|
nfev += int(n)
|
|
check(f"{mode} refusal rate %", round(100.0 * ref / nfev, 1), want)
|
|
|
|
|
|
def verify_gap_decomposition():
|
|
"""Section 7.4 of the .md (sec:widelimit in the .tex): the wide-bracket gap taken
|
|
apart on the same evaluations. Every number is RECOMPUTED by running the diagnosis
|
|
scripts, not transcribed from their saved printouts."""
|
|
import contextlib
|
|
import io
|
|
print("\nThe gap decomposition (fresh runs of the diagnosis scripts):")
|
|
import diag_flat_detectable
|
|
import diag_model_order
|
|
import diag_model_order_checks
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
mo = diag_model_order.main()
|
|
ck = diag_model_order_checks.main()
|
|
fl = diag_flat_detectable.main()
|
|
per = mo["per_problem"]
|
|
check("problems where we lose", len(mo["lose"]), 33)
|
|
check("fun1.4 our bracket halvings/eval", round(per["fun1.4"][2], 2), 0.62)
|
|
check("fun1.4 IQI from our points", round(per["fun1.4"][3], 2), 10.37)
|
|
flat = [p for p in per if p.split(".")[0] in ("fun3", "fun4", "fun5", "fun6")]
|
|
check("flat: ours min", round(min(per[p][2] for p in flat), 2), 0.41)
|
|
check("flat: ours max", round(max(per[p][2] for p in flat), 2), 0.81)
|
|
check("flat: IQI min", round(min(per[p][3] for p in flat), 2), 0.01)
|
|
check("flat: IQI max", round(max(per[p][3] for p in flat), 2), 0.17)
|
|
check("non-degenerate states with one negative weight", mo["neg_ok"], 428)
|
|
check("bracketing states in total", mo["neg_all"], 433)
|
|
check("every exception is an underflowed zero weight",
|
|
all(any(w == 0.0 for w in ws) for _p, _f, ws in ck["exceptions"]), True)
|
|
fam = ck["fam"]
|
|
gap = {k: v[2] - v[3] for k, v in fam.items()}
|
|
check("total gap on the losing problems", sum(gap.values()), 244)
|
|
check("flat families' gap", sum(gap[k] for k in ("fun3", "fun4", "fun5", "fun6")), 160)
|
|
check("fun1 gap", gap["fun1"], 58)
|
|
for k, want in (("fun3", 92), ("fun4", 96), ("fun5", 100), ("fun6", 100)):
|
|
b, q = fam[k][0], fam[k][1]
|
|
check(f"Chandrupatla bisect share {k} %", round(100.0 * b / (b + q)), want)
|
|
for k, want in (("fun4", 297), ("fun5", 233), ("fun6", 233)):
|
|
check(f"Chandrupatla total {k}", fam[k][3], want)
|
|
check("m>=2 on canonical states %", round(fl["APS (canonical)"][2], 1), 0.2)
|
|
check("m>=2 on flat states %", round(fl["CHA flat fun3-6"][2], 1), 5.8)
|
|
# the band per family, from the Linux sweep
|
|
with open("sweep_decision.json", encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
by = {(r["spec"][1], r["spec"][5], r["spec"][3]): r for r in rows}
|
|
base = by[("ratio", "none", "project")]["CHA"]["per"]
|
|
band = by[("ratio", "band", "project")]["CHA"]["per"]
|
|
|
|
def fam_total(per_, k):
|
|
return sum(n if ok else 200 for pid, (n, ok) in per_.items()
|
|
if pid.split(".")[0] == k)
|
|
for k, b0, b1 in (("fun4", 573, 296), ("fun5", 456, 239), ("fun6", 456, 235)):
|
|
check(f"band {k} no-gate", fam_total(base, k), b0)
|
|
check(f"band {k} with band", fam_total(band, k), b1)
|
|
for k, d in (("fun1", 164), ("fun2", 111), ("fun8", 170), ("fun9", 144)):
|
|
check(f"band cost on {k}", fam_total(band, k) - fam_total(base, k), d)
|
|
# negative item 11, corrected pairing: the same configuration in x and in log space
|
|
with open("sweep_full.json", encoding="utf8") as fh:
|
|
full = json.load(fh)
|
|
xcell = next(r for r in full if r["label"] == "pure.ratio_soft.derived.safe")
|
|
check("log item: x-space control APS", xcell["APS"]["total"], 5822)
|
|
check("log item: x-space control CHA", xcell["CHA"]["total"], 3643)
|
|
|
|
|
|
def verify_safeguard_and_tables():
|
|
"""§3.1 of the .md (the multiplier is a free constant; the earned budget), Tables 2-3
|
|
regenerated with Chandrupatla, §5.1, and the constant-free numbers of §7. The
|
|
measured tables are RECOMPUTED by running the measurement scripts."""
|
|
import contextlib
|
|
import io
|
|
import measure_budget_factor
|
|
import measure_records_vs_multiplier
|
|
print("\n§3.1: the multiplier and the endowment (fresh runs):")
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
bf = measure_budget_factor.main()
|
|
rm = measure_records_vs_multiplier.main()
|
|
for c, (aps, cha) in ((1.0, (4817, 2997)), (1.25, (2806, 2752)), (1.5, (2806, 2889)),
|
|
(2.0, (2806, 3220)), (3.0, (2806, 3756)), (4.0, (2806, 4006))):
|
|
check(f"multiplier {c} APS", bf[c][0], aps)
|
|
check(f"multiplier {c} CHA", bf[c][2], cha)
|
|
for c0, (aps, cha) in ((0.1, (5344, 3051)), (0.5, (4656, 3002)), (1.0, (3568, 3020)),
|
|
(2.0, (3692, 2712)), (5.0, (3006, 2615))):
|
|
check(f"endowment {c0} APS", bf["endowment"][c0][0], aps)
|
|
check(f"endowment {c0} CHA", bf["endowment"][c0][1], cha)
|
|
within_one_step = {c0: sum(v) for c0, v in bf["endowment"].items() if c0 <= 1.0}
|
|
check("C0=1 cheapest within the one-step promise",
|
|
min(within_one_step, key=within_one_step.get), 1.0)
|
|
sh = rm["shape reading (2527)"]
|
|
check("2527 at multiplier 1.25", sh[1.25][0], 2541)
|
|
check("2527 flat from 1.5 to 4", {sh[f][0] for f in (1.5, 2.0, 3.0, 4.0)}, {2527})
|
|
de = rm["defect reading (2523)"]
|
|
check("2523 flat from 1.25 to 4", {de[f][0] for f in (1.25, 1.5, 2.0, 3.0, 4.0)}, {2523})
|
|
|
|
print("\nTables 2-3 regenerated:")
|
|
with open("sweep_full.json", encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
by = {r["label"]: r for r in rows}
|
|
none_ = "pure.ratio.derived3.none"
|
|
for setname, lbl, pl in (("APS", PROJ, 3), ("APS", none_, 2), ("CHA", PROJ, 47),
|
|
("CHA", none_, 228), ("CHA", "pure.exp_abs.bracket.both", 4)):
|
|
check(f"{setname} {lbl} place", place(rows, setname, lbl), pl)
|
|
check("CHA exp_abs.bracket.both total", by["pure.exp_abs.bracket.both"]["CHA"]["total"],
|
|
2733)
|
|
check("CHA TOMS 748 over the 2x bound", by["toms748"]["CHA"]["over"], 15)
|
|
check("CHA Brent over the 2x bound", by["brent"]["CHA"]["over"], 18)
|
|
for lbl, setname, opp, want in (
|
|
(CREDIT, "APS", "itp_oracle", (86, 6, 61)),
|
|
(CREDIT, "CHA", "itp_oracle", (2, 3, 35)),
|
|
(CREDIT, "CHA", "bisection", (5, 0, 35)),
|
|
(PROJ, "APS", "chandrupatla", (87, 24, 42)),
|
|
(PROJ, "APS", "itp", (132, 3, 18)),
|
|
(PROJ, "APS", "itp_oracle", (121, 4, 28)),
|
|
(PROJ, "APS", "bisection", (152, 1, 0)),
|
|
(PROJ, "CHA", "toms748", (31, 1, 8)),
|
|
(PROJ, "CHA", "brent", (30, 4, 4)),
|
|
(PROJ, "CHA", "itp", (17, 0, 23)),
|
|
(PROJ, "CHA", "itp_oracle", (16, 0, 24))):
|
|
w, t, l, _ = head_to_head(rows, lbl, opp, setname)
|
|
check(f"{lbl.split('.')[-1]} {setname} vs {opp}", (w, t, l), want)
|
|
for lbl, want in ((CREDIT, 6588), (PROJ, 6026), ("chandrupatla", 5228),
|
|
("itp_oracle", 5871), ("toms748", 6820), ("brent", 7004)):
|
|
check(f"both sets {lbl}", by[lbl]["APS"]["total"] + by[lbl]["CHA"]["total"], want)
|
|
|
|
print("\n§7 under the earned projection:")
|
|
import glob
|
|
cells = {}
|
|
for p in glob.glob("sweep_*.json"):
|
|
if "smoke" in p:
|
|
continue
|
|
with open(p, encoding="utf8") as fh:
|
|
data = json.load(fh)
|
|
# only grid files (a list of labelled cells); the out-of-sample results
|
|
# (sweep_oos.json, sweep_fourth_*.json, ...) are dicts of a different shape
|
|
if not isinstance(data, list):
|
|
continue
|
|
for r in data:
|
|
cells.setdefault(r["label"], r)
|
|
for lbl, aps, cha in (("slope.ratio.derived3.credit", 3507, 2997),
|
|
("pure+shape+slope.ratio.derived3.credit", 3755, 3021),
|
|
("pure+dissim+slope.ratio.derived3.credit", 3857, 3002),
|
|
("pure+shape+dissim+slope.ratio.derived3.credit", 4354, 3037),
|
|
("defect.ratio.derived3.credit", 3545, 2949)):
|
|
check(f"{lbl} APS", cells[lbl]["APS"]["total"], aps)
|
|
check(f"{lbl} CHA", cells[lbl]["CHA"]["total"], cha)
|
|
free = [r for r in cells.values() if r.get("spec") and len(r["spec"]) > 3
|
|
and r["spec"][0] == "pure" and r["spec"][3] == "credit"
|
|
and r["APS"]["solved"] + r["CHA"]["solved"] == 193]
|
|
top = min(free, key=lambda r: r["APS"]["total"] + r["CHA"]["total"])
|
|
check("best constant-free configuration", top["label"], "defect.ratio.derived3.credit")
|
|
check("its total over both sets", top["APS"]["total"] + top["CHA"]["total"], 6494)
|
|
|
|
|
|
PLACEMENT_PER_SET = {
|
|
"sweep_direction.json": ("CHA", "pure+dissim+slope.ratio.derived3.safe", 2683),
|
|
"sweep_gate.json": ("CHA", "gate.ratio_soft.derived.safe", 2851),
|
|
}
|
|
|
|
|
|
def verify_placement_modes():
|
|
print("\nThe placement modes (each in its own sweep file):")
|
|
for path, (lbl, aps, cha) in PLACEMENT.items():
|
|
if not os.path.exists(path):
|
|
print(f" !! {path} absent — the section quoting it cannot be verified")
|
|
FAILS.append(f"missing {path}")
|
|
continue
|
|
with open(path, encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
try:
|
|
d = next(r for r in rows if r["label"] == lbl)
|
|
except StopIteration:
|
|
print(f" !! {lbl} not in {path} — stale sweep")
|
|
FAILS.append(f"{lbl} missing from {path}")
|
|
continue
|
|
check(f"{lbl} APS", d["APS"]["total"], aps)
|
|
check(f"{lbl} CHA", d["CHA"]["total"], cha)
|
|
check(f"{lbl} solved", d["APS"]["solved"] + d["CHA"]["solved"], 193)
|
|
check(f"{lbl} guarantee (over 2x)", d["APS"]["over"] + d["CHA"]["over"], 0)
|
|
for path, (setname, lbl, want) in PLACEMENT_PER_SET.items():
|
|
if not os.path.exists(path):
|
|
continue
|
|
with open(path, encoding="utf8") as fh:
|
|
rows = json.load(fh)
|
|
d = next((r for r in rows if r["label"] == lbl), None)
|
|
if d is None:
|
|
FAILS.append(f"{lbl} missing from {path}")
|
|
continue
|
|
check(f"{lbl} {setname} per-set", d[setname]["total"], want)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|