1219 lines
63 KiB
Python
1219 lines
63 KiB
Python
"""Check every number the paper states against sweep_full.json.
|
|
|
|
A paper is a claim about data; transcribing its tables by hand is how a claim
|
|
stops matching its data. A failure here means the paper is wrong, not the
|
|
test.
|
|
|
|
PROPOSED/PROJ below is the BASE rule of section 3: the scheduled projection with its
|
|
multiplier DECLARED as the one free constant (Valeri, 2026-09-21, variant a); the method
|
|
the paper now proposes, A', is checked in verify_flat_regime_and_oos(); the earned projection
|
|
(credit) is the constant-free variant. Both are checked. Baselines are the
|
|
library alone since 2026-09-21: Chandrupatla = scipy elementwise.find_root,
|
|
Brent = brentq.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
from report_sweep import CREDIT, PROPOSED, head_to_head, place, rank
|
|
|
|
FAILS = []
|
|
PROJ = "pure.ratio.derived3.project"
|
|
assert PROPOSED == PROJ, "the paper proposes the projection (variant a)"
|
|
|
|
|
|
def check(label, got, want):
|
|
ok = got == want
|
|
print(f" {'OK ' if ok else 'BAD'} {label:58s} got {got!r:>16} "
|
|
f"want {want!r}")
|
|
if not ok:
|
|
FAILS.append(label)
|
|
|
|
|
|
def row(rows, setname, lbl):
|
|
return next(r for r in rows if r["label"] == lbl)[setname]
|
|
|
|
|
|
def main():
|
|
rows = _load("sweep_full.json")
|
|
|
|
# Guard against quoting places from a field that is missing a competitor.
|
|
# It has already happened once: before Chandrupatla's own method was
|
|
# added, every ranking assertion here passed only because the method that
|
|
# beats us was absent.
|
|
labels = {r["label"] for r in rows}
|
|
missing = {"chandrupatla", "dekker", PROPOSED} - labels
|
|
if missing:
|
|
print(f" !! THE SWEEP IS STALE: missing {sorted(missing)}. "
|
|
"Re-run sweep_full.py.\n")
|
|
FAILS.append("sweep is stale")
|
|
|
|
check("number of configurations", len(rows), 264)
|
|
|
|
# ---- the proposed method: projection, one declared constant ----------
|
|
print("\nThe proposed method (projection, multiplier declared):")
|
|
for setname, pl, tot, sol in (("APS", 3, 2634, 154),
|
|
("CHA", 49, 2357, 45)):
|
|
d = row(rows, setname, PROPOSED)
|
|
check(f"{setname} place", place(rows, setname, PROPOSED), pl)
|
|
check(f"{setname} total", d["total"], tot)
|
|
check(f"{setname} solved", d["solved"], sol)
|
|
check("both sets", row(rows, "APS", PROPOSED)["total"]
|
|
+ row(rows, "CHA", PROPOSED)["total"], 4991)
|
|
|
|
print("\nHead to head of the proposed method:")
|
|
for setname, opp, want in [
|
|
("APS", "chandrupatla", (87, 24, 43)),
|
|
("APS", "toms748", (86, 16, 52)),
|
|
("APS", "brent", (102, 11, 41)),
|
|
("APS", "itp", (132, 3, 19)),
|
|
("APS", "itp_oracle", (121, 4, 29)),
|
|
("APS", "bisection", (152, 1, 1)),
|
|
("CHA", "chandrupatla", (11, 5, 29)),
|
|
("CHA", "toms748", (33, 1, 11)),
|
|
("CHA", "brent", (34, 4, 7)),
|
|
("CHA", "itp", (18, 1, 26)),
|
|
("CHA", "itp_oracle", (16, 1, 28))]:
|
|
w, t, l, _ = head_to_head(rows, PROPOSED, opp, setname)
|
|
check(f"{setname} vs {opp}", (w, t, l), want)
|
|
|
|
def sign_p(w, l):
|
|
import math as _m
|
|
k, n = min(w, l), w + l
|
|
return min(1.0, 2.0 * sum(_m.comb(n, i) for i in range(k + 1)) / 2.0 ** n)
|
|
w, _t, l, _ = head_to_head(rows, PROPOSED, "chandrupatla", "APS")
|
|
check("APS vs Chandrupatla sign test p (2 digits)", float(f"{sign_p(w, l):.2g}"), 0.00014)
|
|
w, _t, l, _ = head_to_head(rows, PROPOSED, "chandrupatla", "CHA")
|
|
check("CHA vs Chandrupatla sign test p (rounded)", round(sign_p(w, l), 3), 0.006)
|
|
|
|
# ---- the constant-free variant ---------------------------------------
|
|
print("\nThe constant-free variant (earned projection):")
|
|
for setname, pl, tot in (("APS", 25, 3377), ("CHA", 9, 2073)):
|
|
check(f"credit {setname} place", place(rows, setname, CREDIT), pl)
|
|
check(f"credit {setname} total", row(rows, setname, CREDIT)["total"], tot)
|
|
check("credit both sets", row(rows, "APS", CREDIT)["total"]
|
|
+ row(rows, "CHA", CREDIT)["total"], 5450)
|
|
check("credit costs 743 more on the canonical set",
|
|
row(rows, "APS", CREDIT)["total"] - row(rows, "APS", PROPOSED)["total"], 743)
|
|
for setname, opp, want in [("APS", "chandrupatla", (79, 3, 72)),
|
|
("APS", "toms748", (78, 9, 67)),
|
|
("APS", "brent", (79, 5, 70))]:
|
|
w, t, l, _ = head_to_head(rows, CREDIT, opp, setname)
|
|
check(f"credit {setname} vs {opp}", (w, t, l), want)
|
|
w, _t, l, _ = head_to_head(rows, CREDIT, "chandrupatla", "APS")
|
|
check("credit APS vs Chandrupatla sign test p (rounded)", round(sign_p(w, l), 2), 0.63)
|
|
|
|
# ---- what the earlier, order-destroying guards cost --------------------
|
|
print("\nThe guards that replace the step instead of projecting it:")
|
|
check("APS unguarded", row(rows, "APS", "pure.ratio.derived3.none")["total"],
|
|
2634)
|
|
cha_none = row(rows, "CHA", "pure.ratio.derived3.none")
|
|
check("CHA unguarded total", cha_none["total"], 3391)
|
|
check("CHA unguarded solved", cha_none["solved"], 39)
|
|
check("CHA unguarded bound violations", cha_none["over"], 10)
|
|
check("APS bracket-guard", row(rows, "APS", "pure.ratio.derived3.bracket")["total"],
|
|
7068)
|
|
check("APS both-guard", row(rows, "APS", "pure.ratio.derived3.both")["total"],
|
|
7046)
|
|
|
|
# ---- the field --------------------------------------------------------
|
|
print("\nAlefeld-Potra-Shi:")
|
|
check("APS winner", rank(rows, "APS")[0]["label"], "chandrupatla")
|
|
for lbl, pl, tot in [("chandrupatla", 1, 2595), ("toms748", 4, 2636),
|
|
("brent", 5, 2707), ("dekker", 232, 3158),
|
|
("itp_oracle", 11, 3068), ("ridders", 15, 3124),
|
|
("itp", 29, 3767), ("illinois", 237, 7045),
|
|
("bisection", 73, 7338), ("regula_falsi", 244, 10560)]:
|
|
check(f"APS {lbl} place", place(rows, "APS", lbl), pl)
|
|
check(f"APS {lbl} total", row(rows, "APS", lbl)["total"], tot)
|
|
check("APS regula falsi solved", row(rows, "APS", "regula_falsi")["solved"],
|
|
127)
|
|
|
|
print("\nChandrupatla:")
|
|
top = rank(rows, "CHA")[0]
|
|
check("CHA rank 1 label", top["label"], "chandrupatla")
|
|
check("CHA rank 1 solves all 45", top["CHA"]["solved"], 45)
|
|
for lbl, pl, tot in [("chandrupatla", 1, 1505), ("itp_oracle", 2, 1675),
|
|
("itp", 5, 1946), ("bisection", 15, 2136),
|
|
("toms748", 158, 3088), ("brent", 227, 3238)]:
|
|
check(f"CHA {lbl} place", place(rows, "CHA", lbl), pl)
|
|
check(f"CHA {lbl} total", row(rows, "CHA", lbl)["total"], tot)
|
|
check("CHA brent solved", row(rows, "CHA", "brent")["solved"], 43)
|
|
|
|
ours_both = (row(rows, "APS", PROPOSED)["total"]
|
|
+ row(rows, "CHA", PROPOSED)["total"])
|
|
cha_both = (row(rows, "APS", "chandrupatla")["total"]
|
|
+ row(rows, "CHA", "chandrupatla")["total"])
|
|
check("chandrupatla, both sets", cha_both, 4100)
|
|
check("Chandrupatla beats us on both sets", cha_both < ours_both, True)
|
|
|
|
# ---- the regime findings ---------------------------------------------
|
|
print("\nNegative results, the regime claims:")
|
|
check("CHA cbrt over the whole set",
|
|
row(rows, "CHA", "pure.cbrt.derived3.project")["total"], 3204)
|
|
switched = (row(rows, "APS", PROJ)["total"]
|
|
+ row(rows, "CHA", "pure.cbrt.derived3.project")["total"])
|
|
single = row(rows, "APS", PROJ)["total"] + row(rows, "CHA", PROJ)["total"]
|
|
check("single fixed choice, both sets", single, 4991)
|
|
check("switch by benchmark set, both sets", switched, 5838)
|
|
check("switching is worse than not switching", switched > single, True)
|
|
for pid, ours_n, cb_n in (("fun4.1", 89, 32), ("fun4.2", 93, 32),
|
|
("fun4.3", 113, 40), ("fun4.4", 125, 47),
|
|
("fun4.5", 153, 59)):
|
|
o = row(rows, "CHA", PROJ)["per"][pid]
|
|
c = row(rows, "CHA", "pure.cbrt.derived3.project")["per"][pid]
|
|
check(f"{pid} projection variant", o[0], ours_n)
|
|
check(f"{pid} cbrt", c[0], cb_n)
|
|
|
|
# ---- the law across the grid ------------------------------------------
|
|
print("\nThe law across the grid:")
|
|
lin = ("ratio", "rational", "exp_abs")
|
|
quad = ("rat_sq", "exp_sq", "cosine")
|
|
want = {("derived3", "lin"): 5764, ("derived3", "quad"): 21200,
|
|
("derived", "lin"): 5854, ("derived", "quad"): 21089,
|
|
("bracket", "lin"): 9149, ("bracket", "quad"): 18015,
|
|
("start", "lin"): 14983, ("start", "quad"): 14228}
|
|
for scale in ("derived3", "derived", "bracket", "start"):
|
|
for group, gname in ((lin, "lin"), (quad, "quad")):
|
|
tot = cnt = 0
|
|
for r in rows:
|
|
if r["spec"][0] != "pure":
|
|
continue
|
|
_, sim, sc, gu = r["spec"]
|
|
if sc != scale or gu != "step" or sim not in group:
|
|
continue
|
|
cnt += 1
|
|
tot += r["APS"]["total"] + r["CHA"]["total"]
|
|
check(f"mean total, scale={scale}, {gname}",
|
|
round(tot / cnt), want[(scale, gname)])
|
|
|
|
best_by_sim, best_by_scale = {}, {}
|
|
for r in rows:
|
|
if r["spec"][0] != "pure" or r["spec"][3] != "step":
|
|
continue
|
|
t = r["APS"]["total"] + r["CHA"]["total"]
|
|
best_by_sim[r["spec"][1]] = min(best_by_sim.get(r["spec"][1], 10 ** 9), t)
|
|
best_by_scale[r["spec"][2]] = min(best_by_scale.get(r["spec"][2], 10 ** 9), t)
|
|
check("every c=1 similarity beats every c=0 similarity",
|
|
max(best_by_sim[s] for s in lin) < min(best_by_sim[s] for s in quad),
|
|
True)
|
|
check("similarity ranking",
|
|
[s for s, _ in sorted(best_by_sim.items(), key=lambda kv: kv[1])],
|
|
["ratio", "exp_abs", "rational", "ratio_soft", "sqrt", "cbrt",
|
|
"exp_sq", "rat_sq", "cosine"])
|
|
for sim, v in [("ratio", 5422), ("exp_abs", 5772), ("rational", 5993),
|
|
("ratio_soft", 8455), ("sqrt", 9411), ("cbrt", 10874),
|
|
("exp_sq", 13187), ("rat_sq", 13990), ("cosine", 15272)]:
|
|
check(f"similarity {sim} best", best_by_sim[sim], v)
|
|
check("scale ranking",
|
|
[s for s, _ in sorted(best_by_scale.items(), key=lambda kv: kv[1])],
|
|
["derived3", "derived", "bracket", "start"])
|
|
for sc, v in [("derived3", 5422), ("derived", 5564), ("bracket", 7972),
|
|
("start", 13423)]:
|
|
check(f"scale {sc} best", best_by_scale[sc], v)
|
|
|
|
verify_placement_modes()
|
|
verify_per_place()
|
|
verify_decision_place()
|
|
verify_gap_decomposition()
|
|
verify_safeguard_and_tables()
|
|
verify_flat_regime_and_oos()
|
|
verify_revision1()
|
|
verify_controls()
|
|
verify_revision2()
|
|
verify_revision3()
|
|
|
|
print()
|
|
if FAILS:
|
|
print(f"{len(FAILS)} STATED NUMBERS DO NOT MATCH THE DATA:")
|
|
for f in FAILS:
|
|
print(f" {f}")
|
|
sys.exit(1)
|
|
print("every number stated in the paper matches the sweep data.")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Section "where the similarity is read" (PREREGs of 2026-09-20/21). Each new
|
|
# grid lives in its OWN sweep file so the published sweep_full.json stays the
|
|
# artefact of the original claims; every number the new section quotes is
|
|
# re-checked here against that file.
|
|
# --------------------------------------------------------------------------
|
|
PLACEMENT = {
|
|
# file label APS CHA
|
|
"sweep_slope.json": ("slope.ratio.derived3.project", 2663, 2170),
|
|
"sweep_points.json": ("pure+shape+slope.ratio.derived3.safe", 2337, 2177),
|
|
"sweep_direction.json": ("pure+dissim+slope.ratio.derived3.project", 2539, 1864),
|
|
"sweep_combo.json": ("pure+shape+dissim+slope.ratio.derived3.safe", 2695, 1928),
|
|
"sweep_gate.json": ("progress.ratio.derived.project", 4258, 2093),
|
|
"sweep_log.json": ("log.ratio_soft.derived.safe", 6749, 2598),
|
|
}
|
|
# The per-place table of section "the similarity function belongs to the place":
|
|
# seven PAIRED comparisons in sweep_order.json. Checked as pairs, because a pair is
|
|
# what the pre-registration licensed and a single cell would prove nothing.
|
|
ORDER_PAIRS = {
|
|
# guard (APS m=1, APS measured, CHA m=1, CHA measured)
|
|
"both": (4331, 3485, 2469, 2306),
|
|
"bracket": (4418, 3352, 2671, 2428),
|
|
"credit": (4999, 4495, 2064, 1933),
|
|
"project": (6783, 6176, 4057, 3751),
|
|
"safe": (5803, 5040, 3218, 3032),
|
|
"step": (6251, 5362, 3371, 3146),
|
|
"none": (10676, 9855, 8265, 7424),
|
|
}
|
|
# the negative item: the order does not cure false position
|
|
ORDER_FALSEPOS = {("rational", "far", "none", "none"): (127, 10676),
|
|
("rational", "far", "none", "order"): (129, 9855)}
|
|
# and the harmful direction, quoted in the same subsection
|
|
ORDER_SLOPE_HARM = {("ratio", "derived3", "none", "none"): 2634,
|
|
("ratio", "derived3", "none", "order"): 2703}
|
|
|
|
|
|
def verify_per_place():
|
|
print("\nThe per-place reading (sweep_order.json):")
|
|
if not os.path.exists("sweep_order.json"):
|
|
print(" !! sweep_order.json absent — section 'belongs to the place' unverifiable")
|
|
FAILS.append("missing sweep_order.json")
|
|
return
|
|
rows = _load("sweep_order.json")
|
|
|
|
def cell(sim, sc, gu, ex):
|
|
for r in rows:
|
|
s = r["spec"]
|
|
if (s[1], s[2], s[3], s[8] if len(s) > 8 else "none") == (sim, sc, gu, ex):
|
|
return r
|
|
return None
|
|
|
|
cheaper = 0
|
|
for gu, (a1, a2, c1, c2) in ORDER_PAIRS.items():
|
|
p = cell("rational", "far", gu, "none")
|
|
q = cell("rational", "far", gu, "order")
|
|
if p is None or q is None:
|
|
FAILS.append(f"order pair {gu} missing from sweep_order.json")
|
|
print(f" !! pair {gu} missing — stale sweep")
|
|
continue
|
|
check(f"far+rational {gu} APS m=1", p["APS"]["total"], a1)
|
|
check(f"far+rational {gu} APS order", q["APS"]["total"], a2)
|
|
check(f"far+rational {gu} CHA m=1", p["CHA"]["total"], c1)
|
|
check(f"far+rational {gu} CHA order", q["CHA"]["total"], c2)
|
|
cheaper += (q["APS"]["total"] + q["CHA"]["total"]
|
|
< p["APS"]["total"] + p["CHA"]["total"])
|
|
check("cheaper in 7 of 7 safeguards", cheaper, 7)
|
|
for (sim, sc, gu, ex), (solved, total) in ORDER_FALSEPOS.items():
|
|
r = cell(sim, sc, gu, ex)
|
|
if r is None:
|
|
FAILS.append(f"false-position cell {ex} missing")
|
|
continue
|
|
check(f"false position {ex} solved", r["APS"]["solved"], solved)
|
|
check(f"false position {ex} APS", r["APS"]["total"], total)
|
|
for (sim, sc, gu, ex), want in ORDER_SLOPE_HARM.items():
|
|
r = cell(sim, sc, gu, ex)
|
|
if r is None:
|
|
FAILS.append(f"slope-harm cell {ex} missing")
|
|
continue
|
|
check(f"slope reading {ex} APS", r["APS"]["total"], want)
|
|
fam = [r for r in rows if r["spec"][3] in ("project", "safe", "credit")]
|
|
check("guarantee family violations", sum(r["APS"]["over"] + r["CHA"]["over"]
|
|
for r in fam), 0)
|
|
|
|
|
|
DECISION_CELLS = {
|
|
# (similarity, prestep, guard): {set: total}
|
|
("ratio", "band", "project"): {"APS": 5242, "CHA": 2101},
|
|
("ratio", "band", "credit"): {"CHA": 1942},
|
|
("ratio", "none", "project"): {"APS": 2634},
|
|
("ratio", "defect", "project"): {"APS": 2351, "CHA": 2346},
|
|
("ratio_soft", "simpair", "project"): {"CHA": 1941},
|
|
}
|
|
DECISION_RATES = {"band": 63.5, "defect": 6.6}
|
|
|
|
|
|
def verify_decision_place():
|
|
print("\nThe decision place (sweep_decision.json):")
|
|
if not os.path.exists("sweep_decision.json"):
|
|
print(" !! sweep_decision.json absent — section 'decision place' unverifiable")
|
|
FAILS.append("missing sweep_decision.json")
|
|
return
|
|
rows = _load("sweep_decision.json")
|
|
by = {(r["spec"][1], r["spec"][5], r["spec"][3]): r for r in rows}
|
|
for k, sets in DECISION_CELLS.items():
|
|
r = by.get(k)
|
|
if r is None:
|
|
FAILS.append(f"decision cell {k} missing")
|
|
print(f" !! {k} missing — stale sweep")
|
|
continue
|
|
for setname, want in sets.items():
|
|
check(f"{'.'.join(k)} {setname}", r[setname]["total"], want)
|
|
d = by[("ratio", "defect", "project")]
|
|
check("defect APS solved", d["APS"]["solved"], 154)
|
|
check("defect both sets", d["APS"]["total"] + d["CHA"]["total"], 4697)
|
|
check("defect second to the shape reading (2337 < defect)", 2337 < d["APS"]["total"], True)
|
|
fam = [r for r in rows if r["spec"][3] in ("project", "safe", "credit")]
|
|
check("decision grid guarantee violations", sum(r["APS"]["over"] + r["CHA"]["over"]
|
|
for r in fam), 0)
|
|
# the refusal rates are recomputed, not transcribed
|
|
from scipy.optimize._tstutils import _APS_TESTS_DICTS, _CHANDRUPATLA_TESTS_DICTS
|
|
from simroot.pure import pure_root
|
|
probs = [(float(q["bracket"][0]), float(q["bracket"][1]),
|
|
(lambda ff, aa: lambda t: ff(t, *aa))(q["f"], q.get("args", ())))
|
|
for q in _APS_TESTS_DICTS]
|
|
probs += [(float(q["bracket"][0]), float(q["bracket"][1]),
|
|
(lambda ff: lambda t: ff(t))(q["f"])) for q in _CHANDRUPATLA_TESTS_DICTS]
|
|
for mode, want in DECISION_RATES.items():
|
|
ref = nfev = 0
|
|
for a, b, f in probs:
|
|
st = {}
|
|
try:
|
|
_x, n, _c = pure_root(f, a, b, 1e-12, 200, similarity="ratio",
|
|
scale="derived3", guard="project", prestep=mode,
|
|
stats=st)
|
|
except Exception:
|
|
continue
|
|
ref += st.get("pre_refused", 0)
|
|
nfev += int(n)
|
|
check(f"{mode} refusal rate %", round(100.0 * ref / nfev, 1), want)
|
|
|
|
|
|
def verify_gap_decomposition():
|
|
"""Section 7.4 of the .md (sec:widelimit in the .tex): the wide-bracket gap taken
|
|
apart on the same evaluations. Every number is RECOMPUTED by running the diagnosis
|
|
scripts, not transcribed from their saved printouts."""
|
|
import contextlib
|
|
import io
|
|
print("\nThe gap decomposition (fresh runs of the diagnosis scripts):")
|
|
import diag_flat_detectable
|
|
import diag_model_order
|
|
import diag_model_order_checks
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
mo = diag_model_order.main()
|
|
ck = diag_model_order_checks.main()
|
|
fl = diag_flat_detectable.main()
|
|
per = mo["per_problem"]
|
|
check("problems where we lose", len(mo["lose"]), 35)
|
|
check("fun1.4 our bracket halvings/eval", round(per["fun1.4"][2], 2), 0.62)
|
|
check("fun1.4 IQI from our points", round(per["fun1.4"][3], 2), 10.37)
|
|
flat = [p for p in per if p.split(".")[0] in ("fun3", "fun4", "fun5", "fun6")]
|
|
check("flat: ours min", round(min(per[p][2] for p in flat), 2), 0.41)
|
|
check("flat: ours max", round(max(per[p][2] for p in flat), 2), 0.75)
|
|
check("flat: IQI min", round(min(per[p][3] for p in flat), 2), 0.01)
|
|
check("flat: IQI max", round(max(per[p][3] for p in flat), 2), 0.17)
|
|
check("non-degenerate states with one negative weight", int(mo["neg_ok"]), 431)
|
|
check("bracketing states in total", mo["neg_all"], 434)
|
|
check("every exception is an underflowed zero weight",
|
|
all(any(w == 0.0 for w in ws) for _p, _f, ws in ck["exceptions"]), True)
|
|
fam = ck["fam"]
|
|
gap = {k: v[2] - v[3] for k, v in fam.items()}
|
|
check("total gap on the losing problems", sum(gap.values()), 262)
|
|
check("flat families' gap", sum(gap[k] for k in ("fun3", "fun4", "fun5", "fun6")), 172)
|
|
check("fun1 gap", gap["fun1"], 58)
|
|
for k, want in (("fun3", 92), ("fun4", 96), ("fun5", 100), ("fun6", 100)):
|
|
b, q = fam[k][0], fam[k][1]
|
|
check(f"Chandrupatla bisect share {k} %", round(100.0 * b / (b + q)), want)
|
|
for k, want in (("fun4", 297), ("fun5", 233), ("fun6", 233)):
|
|
check(f"Chandrupatla total {k}", fam[k][3], want)
|
|
check("m>=2 on canonical states %", float(round(fl["APS (canonical)"][2], 1)), 0.3)
|
|
check("m>=2 on flat states %", round(fl["CHA flat fun3-6"][2], 1), 0.0)
|
|
# the band per family, from the Linux sweep
|
|
rows = _load("sweep_decision.json")
|
|
by = {(r["spec"][1], r["spec"][5], r["spec"][3]): r for r in rows}
|
|
base = by[("ratio", "none", "project")]["CHA"]["per"]
|
|
band = by[("ratio", "band", "project")]["CHA"]["per"]
|
|
|
|
def fam_total(per_, k):
|
|
return sum(n if ok else 200 for pid, (n, ok) in per_.items()
|
|
if pid.split(".")[0] == k)
|
|
for k, b0, b1 in (("fun4", 573, 296), ("fun5", 456, 239), ("fun6", 456, 228)):
|
|
check(f"band {k} no-gate", fam_total(base, k), b0)
|
|
check(f"band {k} with band", fam_total(band, k), b1)
|
|
for k, d in (("fun1", 164), ("fun2", 111), ("fun8", 170), ("fun9", 144)):
|
|
check(f"band cost on {k}", fam_total(band, k) - fam_total(base, k), d)
|
|
# negative item 11, corrected pairing: the same configuration in x and in log space
|
|
full = _load("sweep_full.json")
|
|
xcell = next(r for r in full if r["label"] == "pure.ratio_soft.derived.safe")
|
|
check("log item: x-space control APS", xcell["APS"]["total"], 5635)
|
|
check("log item: x-space control CHA", xcell["CHA"]["total"], 2759)
|
|
|
|
|
|
def verify_safeguard_and_tables():
|
|
"""§3.1 of the .md (the multiplier is a free constant; the earned budget), Tables 2-3
|
|
regenerated with Chandrupatla, §5.1, and the constant-free numbers of §7. The
|
|
measured tables are RECOMPUTED by running the measurement scripts."""
|
|
import contextlib
|
|
import io
|
|
import measure_budget_factor
|
|
import measure_records_vs_multiplier
|
|
print("\n§3.1: the multiplier and the endowment (fresh runs):")
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
bf = measure_budget_factor.main()
|
|
rm = measure_records_vs_multiplier.main()
|
|
for c, (aps, cha) in ((1.0, (4625, 2039)), (1.25, (2626, 1822)), (1.5, (2636, 1991)),
|
|
(2.0, (2634, 2357)), (3.0, (2634, 2936)), (4.0, (2634, 3229))):
|
|
check(f"multiplier {c} APS", bf[c][0], aps)
|
|
check(f"multiplier {c} CHA", bf[c][2], cha)
|
|
for c0, (aps, cha) in ((0.1, (5150, 2097)), (0.5, (4462, 2037)), (1.0, (3377, 2073)),
|
|
(2.0, (3502, 1757)), (5.0, (2819, 1666))):
|
|
check(f"endowment {c0} APS", bf["endowment"][c0][0], aps)
|
|
check(f"endowment {c0} CHA", bf["endowment"][c0][1], cha)
|
|
within_one_step = {c0: sum(v) for c0, v in bf["endowment"].items() if c0 <= 1.0}
|
|
check("C0=1 cheapest within the one-step promise",
|
|
min(within_one_step, key=within_one_step.get), 1.0)
|
|
sh = rm["shape reading (2527)"]
|
|
check("shape reading at multiplier 1.25", sh[1.25][0], 2351)
|
|
check("shape reading flat from 1.5 to 4", {sh[f][0] for f in (1.5, 2.0, 3.0, 4.0)}, {2337})
|
|
de = rm["defect reading (2523)"]
|
|
dv = [de[f][0] for f in (1.25, 1.5, 2.0, 3.0, 4.0)]
|
|
check("defect reading from 1.25 to 4: range", (min(dv), max(dv)), (2343, 2353))
|
|
|
|
print("\nTables 2-3 regenerated:")
|
|
rows = _load("sweep_full.json")
|
|
by = {r["label"]: r for r in rows}
|
|
none_ = "pure.ratio.derived3.none"
|
|
for setname, lbl, pl in (("APS", PROJ, 3), ("APS", none_, 2), ("CHA", PROJ, 49),
|
|
("CHA", none_, 229), ("CHA", "pure.exp_abs.bracket.both", 3),
|
|
("CHA", "pure.exp_abs.bracket.bracket", 4),
|
|
("CHA", "pure.exp_abs.bracket.credit", 6),
|
|
("APS", CREDIT, 25), ("CHA", CREDIT, 9)):
|
|
check(f"{setname} {lbl} place", place(rows, setname, lbl), pl)
|
|
check("CHA exp_abs.bracket.both total", by["pure.exp_abs.bracket.both"]["CHA"]["total"],
|
|
1778)
|
|
check("CHA TOMS 748 over the 2x bound", by["toms748"]["CHA"]["over"], 15)
|
|
check("CHA Brent over the 2x bound", by["brent"]["CHA"]["over"], 18)
|
|
for lbl, setname, opp, want in (
|
|
(CREDIT, "APS", "itp_oracle", (86, 6, 62)),
|
|
(CREDIT, "CHA", "itp_oracle", (2, 2, 41)),
|
|
(CREDIT, "CHA", "bisection", (5, 0, 40)),
|
|
(PROJ, "APS", "chandrupatla", (87, 24, 43)),
|
|
(PROJ, "APS", "itp", (132, 3, 19)),
|
|
(PROJ, "APS", "itp_oracle", (121, 4, 29)),
|
|
(PROJ, "APS", "bisection", (152, 1, 1)),
|
|
(PROJ, "CHA", "toms748", (33, 1, 11)),
|
|
(PROJ, "CHA", "brent", (34, 4, 7)),
|
|
(PROJ, "CHA", "itp", (18, 1, 26)),
|
|
(PROJ, "CHA", "itp_oracle", (16, 1, 28))):
|
|
w, t, l, _ = head_to_head(rows, lbl, opp, setname)
|
|
check(f"{lbl.split('.')[-1]} {setname} vs {opp}", (w, t, l), want)
|
|
for lbl, want in ((CREDIT, 5450), (PROJ, 4991), ("chandrupatla", 4100),
|
|
("itp_oracle", 4743), ("toms748", 5724), ("brent", 5945)):
|
|
check(f"both sets {lbl}", by[lbl]["APS"]["total"] + by[lbl]["CHA"]["total"], want)
|
|
|
|
print("\n§7 under the earned projection:")
|
|
import glob
|
|
cells = {}
|
|
for p in glob.glob("sweep_*.json"):
|
|
if "smoke" in p:
|
|
continue
|
|
data = _load(p)
|
|
# only grid files (a list of labelled cells); the out-of-sample results
|
|
# (sweep_oos.json, sweep_fourth_*.json, ...) are dicts of a different shape
|
|
if not isinstance(data, list):
|
|
continue
|
|
for r in data:
|
|
cells.setdefault(r["label"], r)
|
|
for lbl, aps, cha in (("slope.ratio.derived3.credit", 3316, 2050),
|
|
("pure+shape+slope.ratio.derived3.credit", 3564, 2074),
|
|
("pure+dissim+slope.ratio.derived3.credit", 3666, 2052),
|
|
("pure+shape+dissim+slope.ratio.derived3.credit", 4163, 2087),
|
|
("defect.ratio.derived3.credit", 3354, 2002)):
|
|
check(f"{lbl} APS", cells[lbl]["APS"]["total"], aps)
|
|
check(f"{lbl} CHA", cells[lbl]["CHA"]["total"], cha)
|
|
free = [r for r in cells.values() if r.get("spec") and len(r["spec"]) > 3
|
|
and r["spec"][0] == "pure" and r["spec"][3] == "credit"
|
|
and r["APS"]["solved"] + r["CHA"]["solved"] == r["APS"]["n"] + r["CHA"]["n"]]
|
|
top = min(free, key=lambda r: r["APS"]["total"] + r["CHA"]["total"])
|
|
check("best constant-free configuration", top["label"],
|
|
"defect+flatanalogmempt+exp+half.ratio.derived3.credit")
|
|
check("its total over both sets", top["APS"]["total"] + top["CHA"]["total"], 4965)
|
|
d = cells["defect.ratio.derived3.credit"]
|
|
check("defect under the earned projection, both sets",
|
|
d["APS"]["total"] + d["CHA"]["total"], 5356)
|
|
|
|
|
|
PLACEMENT_PER_SET = {
|
|
"sweep_direction.json": ("CHA", "pure+dissim+slope.ratio.derived3.safe", 1752),
|
|
"sweep_gate.json": ("CHA", "progress.ratio_soft.derived.safe", 1929),
|
|
}
|
|
|
|
|
|
def verify_placement_modes():
|
|
print("\nThe placement modes (each in its own sweep file):")
|
|
for path, (lbl, aps, cha) in PLACEMENT.items():
|
|
if not os.path.exists(path):
|
|
print(f" !! {path} absent — the section quoting it cannot be verified")
|
|
FAILS.append(f"missing {path}")
|
|
continue
|
|
rows = _load(path)
|
|
try:
|
|
d = next(r for r in rows if r["label"] == lbl)
|
|
except StopIteration:
|
|
print(f" !! {lbl} not in {path} — stale sweep")
|
|
FAILS.append(f"{lbl} missing from {path}")
|
|
continue
|
|
check(f"{lbl} APS", d["APS"]["total"], aps)
|
|
check(f"{lbl} CHA", d["CHA"]["total"], cha)
|
|
check(f"{lbl} solved", d["APS"]["solved"] + d["CHA"]["solved"], 199)
|
|
check(f"{lbl} guarantee (over 2x)", d["APS"]["over"] + d["CHA"]["over"], 0)
|
|
for path, (setname, lbl, want) in PLACEMENT_PER_SET.items():
|
|
if not os.path.exists(path):
|
|
continue
|
|
rows = _load(path)
|
|
d = next((r for r in rows if r["label"] == lbl), None)
|
|
if d is None:
|
|
FAILS.append(f"{lbl} missing from {path}")
|
|
continue
|
|
check(f"{lbl} {setname} per-set", d[setname]["total"], want)
|
|
|
|
|
|
def _load(path):
|
|
"""A results file. A row-format protocol sweep (a list of labelled cells) is served from
|
|
sweep_uniform.json, its re-scoring under the paper's single criterion - a sign change
|
|
within 1e-6 or f(x) == 0 (revision point 3, 2026-09-22); every other file as stored."""
|
|
from rescore_uniform import load_uniform
|
|
with open(path, encoding="utf8") as fh:
|
|
data = json.load(fh)
|
|
if isinstance(data, list) and data and "spec" in data[0]:
|
|
return load_uniform(path)
|
|
return data
|
|
|
|
|
|
def _cell(path, lbl):
|
|
return next(r for r in _load(path) if r["label"] == lbl)
|
|
|
|
|
|
def _flat_sum(r):
|
|
return sum(n if ok else 200 for pid, (n, ok) in r["CHA"]["per"].items()
|
|
if pid.split(".")[0] in ("fun3", "fun4", "fun5", "fun6"))
|
|
|
|
|
|
def verify_flat_regime_and_oos():
|
|
"""Sections 8-9 and negative items 12-16 of the .md, and the abstract that quotes them:
|
|
the flat regime by analogy, the proposed method A', its ablation, and the out-of-sample
|
|
corpora. Every number is read from a server results file or recomputed by the diagnosis
|
|
scripts; none is transcribed."""
|
|
import contextlib
|
|
import io
|
|
import math
|
|
import re
|
|
print("\nSection 8 and negative items 12-16 (the development grids):")
|
|
P = "ratio.derived3.project"
|
|
for path, lbl, aps, cha in (
|
|
("sweep_flat.json", f"flat.{P}", 2839, 2243),
|
|
("sweep_flat2.json", f"flat2.{P}", 2613, 2388),
|
|
("sweep_flatcreep.json", f"flatcreep.{P}", 2688, 2399),
|
|
("sweep_flatstick.json", f"flatstickmid.{P}", 6707, 2063),
|
|
("sweep_flatexit.json", f"flatstickexit.{P}", 2935, 1546),
|
|
("sweep_flatjudge.json", f"flatjudge.{P}", 2820, 1532),
|
|
("sweep_orderpt.json", f"flatanalogmem.{P}", 2633, None),
|
|
("sweep_half.json", f"half.{P}", 2392, None)):
|
|
r = _cell(path, lbl)
|
|
check(f"{lbl} APS", r["APS"]["total"], aps)
|
|
if cha is not None:
|
|
check(f"{lbl} CHA", r["CHA"]["total"], cha)
|
|
check(f"{lbl} guarantee", r["APS"]["over"] + r["CHA"]["over"], 0)
|
|
check("sticky with order from the failed promise, APS",
|
|
_cell("sweep_flatstick.json", f"flatstick.{P}")["APS"]["total"], 6591)
|
|
check("sticky midpoint: flat families",
|
|
_flat_sum(_cell("sweep_flatstick.json", f"flatstickmid.{P}")), 1057)
|
|
full = {r["label"]: r for r in _load("sweep_full.json")}
|
|
check("bisection: flat families", _flat_sum(full["bisection"]), 1044)
|
|
check("Chandrupatla: flat families", _flat_sum(full["chandrupatla"]), 1068)
|
|
op = {r["label"]: r for r in _load("sweep_orderpt.json")}
|
|
check("split from the order: flat families, low end",
|
|
_flat_sum(op[f"flatanalogmempt.{P}"]), 743)
|
|
check("split from the order: flat families, high end",
|
|
_flat_sum(op[f"pure+extP+flatanalogpt.{P}"]), 847)
|
|
check("item 16: remembered + split, with the step pattern, CHA",
|
|
op[f"pure+extP+flatanalogmempt.{P}"]["CHA"]["total"], 1250)
|
|
check("item 16: one-state analogy + split, with the step pattern, CHA",
|
|
op[f"pure+extP+flatanalogpt.{P}"]["CHA"]["total"], 1388)
|
|
|
|
print("\nSection 8 diagnostics (fresh runs):")
|
|
import diag_analogy_decoupled
|
|
import diag_flat_analogy
|
|
import diag_flat_stick
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
diag_flat_analogy.main()
|
|
out = buf.getvalue()
|
|
aps = re.search(r"^APS\s+154\s+(\d+)\s+(\d+)\s+(\d+)", out, re.M)
|
|
flat = re.search(r"^CHA-flat\s+20\s+(\d+)\s+(\d+)\s+(\d+)", out, re.M)
|
|
check("flag rises on canonical problems", int(aps.group(1)), 142)
|
|
check("flag + analogy + m>=2 on canonical problems", int(aps.group(3)), 3)
|
|
check("flag + analogy + m>=2 on flat problems", int(flat.group(3)), 20)
|
|
for fam in ("fun2", "fun8", "fun9"):
|
|
m = re.search(rf"^CHA-{fam}\s+5\s+\d+\s+\d+\s+(\d+)", out, re.M)
|
|
check(f"flag + analogy + m>=2 on {fam}", int(m.group(1)), 0)
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
diag_analogy_decoupled.main()
|
|
m = re.search(r"mean entry state ([\d.]+) \(same state\) -> ([\d.]+)", buf.getvalue())
|
|
check("mean entry state, one state", m.group(1), "12.9")
|
|
check("mean entry state, remembered", m.group(2), "8.2")
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
diag_flat_stick.main()
|
|
inf = tot = 0
|
|
for d in re.findall(r"order read at entry: (\{[^}]*\})", buf.getvalue()):
|
|
dd = eval(d) # noqa: S307 - the diagnosis script's own printed dict
|
|
inf += dd.get("inf", 0)
|
|
tot += sum(dd.values())
|
|
check("order from the failed promise is inf", (inf, tot), (163, 174))
|
|
from simroot.pure import pure_root
|
|
ap = "contra+flatanalogmempt+exp+half+close+sub"
|
|
|
|
def n_of(f, a, b, pre="none", ext="none"):
|
|
return pure_root(f, a, b, 1e-12, 200, similarity="ratio", scale="derived3",
|
|
guard="project", prestep=pre, extrapolate=ext)[1]
|
|
sq = lambda t: math.sqrt(abs(t)) * (0.0 if t == 0 else math.copysign(1.0, t)) # noqa: E731
|
|
check("sqrt|x| sgn x, base rule", n_of(sq, -1 / 3, 1.0), 84)
|
|
check("sin on [3,4], base rule", n_of(math.sin, 3.0, 4.0), 24)
|
|
check("sin on [3,4], A'", n_of(math.sin, 3.0, 4.0, ap, "pattern"), 7)
|
|
|
|
print("\nTable 8 (A' on the protocol of section 4, sweep_aprime.json):")
|
|
rows_ap = _load("sweep_aprime.json")
|
|
A = f"pure+extP+{ap}.ratio.derived3"
|
|
apr = {r["label"]: r for r in rows_ap}
|
|
a = apr[f"{A}.project"]
|
|
check("A' APS", a["APS"]["total"], 2259)
|
|
check("A' CHA", a["CHA"]["total"], 1074)
|
|
check("A' both", a["APS"]["total"] + a["CHA"]["total"], 3333)
|
|
check("A' solved", a["APS"]["solved"] + a["CHA"]["solved"], 199)
|
|
c = apr[f"{A}.credit"]
|
|
check("A' earned budget", (c["APS"]["total"], c["CHA"]["total"]), (3417, 1852))
|
|
check("A' under every safeguard: violations",
|
|
sum(r["APS"]["over"] + r["CHA"]["over"] for lb, r in apr.items() if lb.startswith(A)), 0)
|
|
check("A' safeguards measured", sum(1 for lb in apr if lb.startswith(A + ".")), 7)
|
|
both = [r for r in _load("sweep_full.json")] + rows_ap
|
|
for setname, opp, want in (("APS", "chandrupatla", (103, 20, 31)),
|
|
("CHA", "chandrupatla", (31, 4, 10)),
|
|
("APS", "toms748", (127, 6, 21)), ("CHA", "toms748", (42, 2, 1)),
|
|
("APS", "brent", (118, 17, 19)), ("CHA", "brent", (39, 3, 3)),
|
|
("APS", "itp_oracle", (126, 7, 21)),
|
|
("CHA", "itp_oracle", (35, 1, 9))):
|
|
w, t, l, _ = head_to_head(both, f"{A}.project", opp, setname)
|
|
check(f"A' {setname} vs {opp}", (w, t, l), want)
|
|
|
|
def sign_p(w, l):
|
|
k, n = min(w, l), w + l
|
|
return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n)
|
|
check("A' APS vs Chandrupatla p (1 digit)", float(f"{sign_p(103, 31):.1g}"), 3e-10)
|
|
check("A' CHA vs Chandrupatla p (1 digit)", float(f"{sign_p(31, 10):.1g}"), 0.001)
|
|
check("A' below Chandrupatla, APS %", round(100 * (1 - 2259 / 2595), 1), 12.9)
|
|
check("A' below Chandrupatla, CHA %", round(100 * (1 - 1074 / 1505), 1), 28.6)
|
|
from where_we_lose import per_family
|
|
af, cf = per_family(a, "CHA"), per_family(full["chandrupatla"], "CHA")
|
|
check("A' fun9 vs Chandrupatla", (af["fun9"], cf["fun9"]), (65, 55))
|
|
check("A' ahead on wide-bracket families", sum(af[k] < cf[k] for k in cf), 7)
|
|
check("A' fun7 vs Chandrupatla", (af["fun7"], cf["fun7"]), (69, 64))
|
|
|
|
print("\nTable 9 (ablation of A' on six corpora, sweep_ablation_all.json):")
|
|
ab = _load("sweep_ablation_all.json")
|
|
sets = ("APS", "CHA", "NEW", "RJL", "APP", "D")
|
|
|
|
def tot(r, s):
|
|
return sum(n if ok else 200 for n, ok, _o in r[s].values())
|
|
base_ = {s: tot(ab["ablA'"], s) for s in sets}
|
|
for comp, want in (("-regime", (-7, 1013, 1660, 324, 77, 2269)),
|
|
("-point", (-5, 479, 667, 272, 30, 1672)),
|
|
("-sub", (-6, 0, -7, 16, 63, 893)),
|
|
("-close", (130, 35, 97, 78, 135, -2)),
|
|
("-half", (249, 65, 82, -16, -5, 1)),
|
|
("-exp", (31, 143, 100, -21, -2, 87)),
|
|
("-G2", (-22, -6, 123, -30, -10, 34)),
|
|
("-contra", (11, -12, 29, 0, 1, -1))):
|
|
got = tuple(tot(ab[f"abl{comp}"], s) - base_[s] for s in sets)
|
|
check(f"ablation {comp}", got, want)
|
|
|
|
print("\nSection 9 (out-of-sample corpora):")
|
|
seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json")
|
|
for k in seen:
|
|
seen[k].update(four[k])
|
|
names = {"APS": 154, "CHA": 45, "NEW": 206, "RJL": 78, "APP": 132, "D": 240}
|
|
for s, n in names.items():
|
|
check(f"corpus {s} size", len(seen["Aprime"][s]), n)
|
|
table10 = {"APS": (2259, 2595, 2707, 2636, (103, 20, 31)),
|
|
"CHA": (1074, 1505, 3238, 3088, (31, 4, 10)),
|
|
"NEW": (3290, 3716, 6395, 5746, (108, 33, 65)),
|
|
"RJL": (1371, 1566, 2455, 2339, (44, 14, 20)),
|
|
"APP": (1382, 1382, 1503, 1756, (48, 52, 32)),
|
|
"D": (3656, 5717, 8267, 8084, (177, 35, 28))}
|
|
|
|
def cost(v):
|
|
return v[0] if v[1] else 200
|
|
for s, (ta, tc, tb, tt, wtl) in table10.items():
|
|
got = tuple(tot(seen[k], s) for k in ("Aprime", "chandrupatla", "brent", "toms748"))
|
|
check(f"Table 10 {s} totals", got, (ta, tc, tb, tt))
|
|
ps = list(seen["Aprime"][s])
|
|
w = sum(cost(seen["Aprime"][s][p]) < cost(seen["chandrupatla"][s][p]) for p in ps)
|
|
l = sum(cost(seen["Aprime"][s][p]) > cost(seen["chandrupatla"][s][p]) for p in ps)
|
|
check(f"Table 10 {s} vs Chandrupatla", (w, len(ps) - w - l, l), wtl)
|
|
check(f"Table 10 {s} A' violations", sum(v[2] for v in seen["Aprime"][s].values()), 0)
|
|
viol = {k: sum(v[2] for s in names for v in seen[k][s].values()) for k in ("brent", "toms748")}
|
|
check("Brent / TOMS 748 bound violations over 855", (viol["brent"], viol["toms748"]), (118, 84))
|
|
check("all 855 totals", tuple(sum(tot(seen[k], s) for s in names)
|
|
for k in ("Aprime", "chandrupatla", "brent", "toms748")),
|
|
(13032, 16481, 24565, 23649))
|
|
d4 = seen["Aprime"]["D"]
|
|
for label, cls, want in (("without targeted", ("D1", "D2", "D4", "D5"), (2403, 3569, (107, 31, 22))),
|
|
("simple roots", ("D1", "D4"), (737, 747, (43, 22, 15)))):
|
|
ps = [p for p in d4 if p[:2] in cls]
|
|
a_ = sum(cost(d4[p]) for p in ps)
|
|
c_ = sum(cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
|
w = sum(cost(d4[p]) < cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
|
l = sum(cost(d4[p]) > cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
|
check(f"corpus 4 {label}", (a_, c_, (w, len(ps) - w - l, l)), want)
|
|
for cls, want in (("D1", (433, 419, (20, 8, 12))), ("D2", (1163, 1805, (39, 1, 0))),
|
|
("D3", (924, 1784, (37, 1, 2))), ("D4", (304, 328, (23, 14, 3))),
|
|
("D5", (503, 1017, (25, 8, 7))), ("D6", (329, 364, (33, 3, 4)))):
|
|
ps = [p for p in d4 if p.startswith(cls)]
|
|
a_ = sum(cost(d4[p]) for p in ps)
|
|
c_ = sum(cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
|
w = sum(cost(d4[p]) < cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
|
l = sum(cost(d4[p]) > cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
|
check(f"Table 11 {cls}", (a_, c_, (w, len(ps) - w - l, l)), want)
|
|
c5 = [p for p in seen["Aprime"]["NEW"] if p.startswith("C5")]
|
|
check("near-cancelling class of corpus 1", (sum(cost(seen["Aprime"]["NEW"][p]) for p in c5),
|
|
sum(cost(seen["chandrupatla"]["NEW"][p]) for p in c5)),
|
|
(491, 391))
|
|
# the pre-registered verdicts on corpora 1-3 (variant without close and sub)
|
|
oos = _load("sweep_oos.json")
|
|
|
|
def tot_new(r):
|
|
return sum(n if ok else 200 for n, ok, _o in r["NEW"].values())
|
|
|
|
def h2h_new(r, o, keep=lambda p: True):
|
|
ps = [p for p in r["NEW"] if keep(p)]
|
|
w = sum(cost(r["NEW"][p]) < cost(o["NEW"][p]) for p in ps)
|
|
l = sum(cost(r["NEW"][p]) > cost(o["NEW"][p]) for p in ps)
|
|
return w, len(ps) - w - l, l
|
|
check("corpus 1, variant carried: total vs Chandrupatla",
|
|
(tot_new(oos["A"]), tot_new(oos["chandrupatla"])), (3492, 3899))
|
|
check("corpus 1, variant carried: per problem", h2h_new(oos["A"], oos["chandrupatla"]),
|
|
(105, 34, 67))
|
|
check("corpus 1, with defect: per problem", h2h_new(oos["B"], oos["chandrupatla"]),
|
|
(95, 33, 78))
|
|
check("corpus 1, with defect: exponential class",
|
|
h2h_new(oos["B"], oos["chandrupatla"], lambda p: p.startswith("C4")), (9, 5, 26))
|
|
rj = _load("sweep_rootsjl.json")
|
|
keep = [p for p, v in rj["A"].items() if not v[3]]
|
|
check("Roots.jl, variant carried: totals", (sum(cost(rj["A"][p]) for p in keep),
|
|
sum(cost(rj["chandrupatla"][p]) for p in keep)),
|
|
(1464, 1566))
|
|
w = sum(cost(rj["A"][p]) < cost(rj["chandrupatla"][p]) for p in keep)
|
|
l = sum(cost(rj["A"][p]) > cost(rj["chandrupatla"][p]) for p in keep)
|
|
check("Roots.jl, variant carried: per problem", (w, len(keep) - w - l, l), (38, 10, 30))
|
|
app = _load("sweep_applied.json")
|
|
check("applied, variant carried: totals", (sum(cost(v) for v in app["A"].values()),
|
|
sum(cost(v) for v in app["chandrupatla"].values())),
|
|
(1580, 1382))
|
|
print("\nModAB (sweep_modab.json, the authors' package):")
|
|
mb = _load("sweep_modab.json")
|
|
ms = mb["seen"]["modab"]
|
|
check("Table 8 ModAB APS, CHA, both",
|
|
(sum(cost(v) for v in ms["APS"].values()), sum(cost(v) for v in ms["CHA"].values()),
|
|
sum(cost(v) for s_ in ("APS", "CHA") for v in ms[s_].values())), (2726, 1681, 4407))
|
|
for setname, want in (("APS", (118, 10, 26)), ("CHA", (34, 4, 7))):
|
|
am = {p: v for p, v in a[setname]["per"].items()}
|
|
mm = ms[setname]
|
|
w = sum(cost(am[p]) < cost(mm[p]) for p in am)
|
|
l = sum(cost(am[p]) > cost(mm[p]) for p in am)
|
|
check(f"Table 8 A' vs ModAB {setname}", (w, len(am) - w - l, l), want)
|
|
md = dict(mb["seen"]["modab"])
|
|
md.update(mb["fourth"]["modab"])
|
|
worst_p = 0.0
|
|
for s, (tm, vm, wtl) in {"APS": (2726, 0, (118, 10, 26)), "CHA": (1681, 0, (34, 4, 7)),
|
|
"NEW": (3809, 1, (125, 26, 55)), "RJL": (1446, 0, (47, 11, 20)),
|
|
"APP": (1460, 0, (76, 35, 21)), "D": (5535, 8, (183, 23, 34))}.items():
|
|
m = md[s]
|
|
check(f"Table 10 ModAB {s}", (sum(cost(v) for v in m.values()),
|
|
sum(v[2] for v in m.values())), (tm, vm))
|
|
ps = list(seen["Aprime"][s])
|
|
w = sum(cost(seen["Aprime"][s][p]) < cost(m[p]) for p in ps)
|
|
l = sum(cost(seen["Aprime"][s][p]) > cost(m[p]) for p in ps)
|
|
check(f"Table 10 A' vs ModAB {s}", (w, len(ps) - w - l, l), wtl)
|
|
worst_p = max(worst_p, sign_p(w, l))
|
|
check("A' vs ModAB: every corpus p <= 0.0013", round(worst_p, 4) <= 0.0013, True)
|
|
check("ModAB over all 855", sum(sum(cost(v) for v in md[s].values()) for s in names), 16657)
|
|
check("ModAB violations over 855", sum(v[2] for s in names for v in md[s].values()), 9)
|
|
check("corpus 4 A' vs ModAB p (1 digit)", f"{sign_p(183, 34):.1g}", "7e-26")
|
|
rjm = mb["rjl"]
|
|
ps91 = list(rjm["Aprime"])
|
|
w = sum(cost(rjm["Aprime"][p]) < cost(rjm["modab"][p]) for p in ps91)
|
|
l = sum(cost(rjm["Aprime"][p]) > cost(rjm["modab"][p]) for p in ps91)
|
|
check("ModAB's own benchmark, all 91", (len(ps91), sum(cost(rjm["Aprime"][p]) for p in ps91),
|
|
sum(cost(rjm["modab"][p]) for p in ps91),
|
|
(w, len(ps91) - w - l, l)),
|
|
(91, 1507, 1588, (56, 13, 22)))
|
|
check("ModAB's own benchmark p (2 digits)", f"{sign_p(w, l):.2g}", "0.00015")
|
|
|
|
print("\nTable 9a and section 7: the similarity in the step (fresh audit run):")
|
|
aps_ = {r["spec"][1]: r for r in _load("sweep_aprime_s.json")}
|
|
abs_ = _load("sweep_ablation_s.json")
|
|
t9a = {"ratio": (3333, 13032), "exp_abs": (3965, 14142), "rational": (4424, 14843),
|
|
"ratio_soft": (7249, 28500), "sqrt": (11613, 45552), "cbrt": (16094, 63031),
|
|
"exp_sq": (17429, 67701), "cosine": (17487, 67854), "rat_sq": (17452, 67394)}
|
|
for s_, (two, six) in t9a.items():
|
|
r = aps_[s_]
|
|
check(f"Table 9a {s_} APS+CHA", r["APS"]["total"] + r["CHA"]["total"], two)
|
|
check(f"Table 9a {s_} six corpora", sum(tot(abs_[f"S:{s_}"], s) for s in sets), six)
|
|
check(f"Table 9a {s_} guarantee", r["APS"]["over"] + r["CHA"]["over"]
|
|
+ sum(v[2] for s in sets for v in abs_[f"S:{s_}"][s].values()), 0)
|
|
check("nearest other S costs more, two sets %", round(100 * (3965 / 3333 - 1)), 19)
|
|
check("worst S costs more, two sets %", round(100 * (17487 / 3333 - 1)), 425)
|
|
check("nearest other S costs more, six corpora %", round(100 * (14142 / 13032 - 1)), 9)
|
|
check("quadratic S: five times",
|
|
(int(17429 / 3333), int(67854 / 13032)), (5, 5))
|
|
import audit_similarity_places
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
audit_similarity_places.main()
|
|
rows_a = {ln.split(" ")[0].strip(): re.findall(r"([+-]\d+) \|", ln)
|
|
for ln in buf.getvalue().splitlines() if " | " in ln}
|
|
check("audit: shape gain with the tent (APS)", int(rows_a["tent 1-|z|"][1]), -493)
|
|
check("audit: shape gain with 1/sqrt(1+z^2) (APS)", int(rows_a["1/sqrt(1+z^2)"][1]), 250)
|
|
check("audit: defect gain with the tent (APS)", int(rows_a["tent 1-|z|"][3]), -283)
|
|
check("audit: defect gain with 1-|z|^1/3 (APS)", int(rows_a["1-|z|^1/3"][3]), 215)
|
|
|
|
# every sign-test p the paper prints in Section 9, at the precision printed
|
|
for (w, l), printed in (((103, 31), "3e-10"), ((31, 10), "0.001"), ((108, 65), "0.001"),
|
|
((44, 20), "0.004"), ((48, 32), "0.09"), ((177, 28), "1e-27"),
|
|
((107, 22), "1e-14"), ((43, 15), "0.0003"), ((105, 67), "0.005"),
|
|
((38, 30), "0.4")):
|
|
check(f"sign test p for {w}:{l}", f"{sign_p(w, l):.1g}", printed)
|
|
|
|
|
|
def verify_revision1():
|
|
"""Revision 1 (2026-09-22): the claims added or restated by the revision - the order
|
|
of the base step, section 7.3's solved counts, the families as the unit, ITP in Table 10,
|
|
negative items 5 and 9, and the one canonical problem the multiplier moves. Recomputed."""
|
|
import contextlib
|
|
import io
|
|
import math
|
|
import re
|
|
from collections import defaultdict
|
|
print("\nRevision 1:")
|
|
import order_high_precision
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
order_high_precision.main()
|
|
out = buf.getvalue()
|
|
check("Muller order", re.search(r"Muller order .*: ([\d.]+)", out).group(1), "1.8393")
|
|
for name in ("cos x - x", "exp(x) - 2", "x^3 - x - 2", "sin x - 1/2"):
|
|
line = next(ln for ln in out.splitlines() if ln.startswith(name))
|
|
three = line[14:54].split()
|
|
check(f"three-point order settles, {name}", three[-2], "1.840")
|
|
line = next(ln for ln in out.splitlines() if ln.startswith("x^2 - 2"))
|
|
check("three-point order on x^2 - 2", line[14:54].split()[-1], "2.000")
|
|
check("two-point order, cos x - x", next(ln for ln in out.splitlines()
|
|
if ln.startswith("cos x - x"))[54:].split()[-1], "1.618")
|
|
|
|
rows = _load("sweep_full.json")
|
|
by = {r["label"]: r for r in rows}
|
|
solved = {k: by[k]["APS"]["solved"] + by[k]["CHA"]["solved"]
|
|
for k in ("chandrupatla", "toms748", "itp", "bisection", "brent", "dekker",
|
|
"illinois")}
|
|
check("section 7.3 solved counts", solved, {"chandrupatla": 199, "toms748": 199,
|
|
"itp": 199, "bisection": 199, "brent": 197,
|
|
"dekker": 178, "illinois": 177})
|
|
check("Chandrupatla fewer than the base rule, APS %",
|
|
round(100 * (by[PROJ]["APS"]["total"] / by["chandrupatla"]["APS"]["total"] - 1), 1), 1.5)
|
|
none_ = by["pure.ratio.derived3.none"]["CHA"]
|
|
check("guarantee on CHA: more solved, fewer evaluations",
|
|
(45 - none_["solved"], round(100 * (1 - by[PROJ]["CHA"]["total"] / none_["total"]))),
|
|
(6, 30))
|
|
check("earned costs 28% more on APS",
|
|
round(100 * (by[CREDIT]["APS"]["total"] / by[PROJ]["APS"]["total"] - 1)), 28)
|
|
|
|
def cost(v):
|
|
return v[0] if v[1] else 200
|
|
|
|
def fam_of(pid):
|
|
return ".".join(pid.split(".")[:2]) if pid.startswith("aps") else pid.split(".")[0]
|
|
|
|
def families(a, b, s):
|
|
ta, tb = defaultdict(int), defaultdict(int)
|
|
for p in a[s]["per"]:
|
|
ta[fam_of(p)] += cost(a[s]["per"][p])
|
|
tb[fam_of(p)] += cost(b[s]["per"][p])
|
|
w = sum(ta[k] < tb[k] for k in ta)
|
|
l = sum(ta[k] > tb[k] for k in ta)
|
|
return w, len(ta) - w - l, l
|
|
|
|
def sign_p(w, l):
|
|
k, n = min(w, l), w + l
|
|
return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n)
|
|
ap = {r["label"]: r for r in _load("sweep_aprime.json")}[
|
|
"pure+extP+contra+flatanalogmempt+exp+half+close+sub.ratio.derived3.project"]
|
|
ch = by["chandrupatla"]
|
|
check("base rule vs Chandrupatla by family, APS", families(by[PROJ], ch, "APS"), (6, 1, 8))
|
|
check("A' vs Chandrupatla by family, APS", families(ap, ch, "APS"), (8, 1, 6))
|
|
check("A' vs Chandrupatla by family, CHA", families(ap, ch, "CHA"), (7, 0, 2))
|
|
# revision: every head-to-head row of the paper must sum to the size of its set
|
|
rivals = ("chandrupatla", "toms748", "brent", "itp_oracle", "itp", "bisection")
|
|
sums = {(who, s): sum(head_to_head(rows + [ap], who, opp, s)[:3])
|
|
for who in (PROJ, ap["label"]) for opp in rivals for s in ("APS", "CHA")}
|
|
check("every head-to-head row sums to its set (154 / 45)",
|
|
{v for (w, s), v in sums.items() if s == "APS"} | {v + 1000 for (w, s), v in sums.items()
|
|
if s == "CHA"}, {154, 1045})
|
|
check("family-level p, APS and CHA (2 digits)",
|
|
(float(f"{sign_p(8, 6):.2g}"), float(f"{sign_p(7, 2):.2g}")), (0.79, 0.18))
|
|
|
|
order = _load("sweep_order.json")
|
|
cells = {}
|
|
for r in order:
|
|
s = r["spec"]
|
|
if s[2] == "far":
|
|
cells[(s[1], s[3], s[8] if len(s) > 8 else "none")] = r["APS"]["total"] + r["CHA"]["total"]
|
|
pairs = [(k[0], k[1]) for k in cells if k[2] == "none" and (k[0], k[1], "order") in cells]
|
|
check("far-end reading: measured order cheaper in cells",
|
|
(sum(cells[(s_, g, "order")] < cells[(s_, g, "none")] for s_, g in pairs), len(pairs)),
|
|
(33, 63))
|
|
|
|
cap, dire = _load("sweep_cap.json"), _load("sweep_direction.json")
|
|
for s, want in (("APS", (3101, 2539)), ("CHA", (1766, 1752))):
|
|
bc = min(r[s]["total"] for r in cap if r[s]["solved"] == r[s]["n"])
|
|
bd = min(r[s]["total"] for r in dire if r[s]["solved"] == r[s]["n"] and r["spec"][0] == "pure")
|
|
check(f"item 9: best capped vs best uncapped, {s}", (bc, bd), want)
|
|
|
|
itp = _load("sweep_itp.json")
|
|
t10 = {"APS": (3767, 3068), "CHA": (1946, 1675), "NEW": (5674, 4707), "RJL": (2161, 1629),
|
|
"APP": (3245, 2386), "D": (5721, 4832)}
|
|
for s, want in t10.items():
|
|
check(f"Table 10 ITP / oracle {s}", (sum(cost(v) for v in itp["itp"][s].values()),
|
|
sum(cost(v) for v in itp["itp_oracle"][s].values())), want)
|
|
check("Table 10 ITP / oracle over 855", tuple(sum(cost(v) for s in t10 for v in itp[k][s].values())
|
|
for k in ("itp", "itp_oracle")), (22514, 18297))
|
|
check("ITP bound violations over 855", sum(v[2] for k in ("itp", "itp_oracle")
|
|
for s in t10 for v in itp[k][s].values()), 0)
|
|
seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json")
|
|
apx = dict(seen["Aprime"])
|
|
apx.update(four["Aprime"])
|
|
check("A' ahead of ITP and the oracle on every corpus",
|
|
all(sum(cost(v) for v in apx[s].values()) < sum(cost(v) for v in itp[k][s].values())
|
|
for s in t10 for k in ("itp", "itp_oracle")), True)
|
|
|
|
from simroot.pure import pure_root
|
|
from stage12_validation import load, per_problem
|
|
probs = load()
|
|
aps = [p for p in probs if p[0] == "APS"]
|
|
runs = {fac: per_problem(lambda f, a, b, xt, mx, _f=fac: pure_root(
|
|
f, a, b, xt, mx, similarity="ratio", scale="derived3", guard="project",
|
|
budget_factor=_f), aps, 200, criterion="sign") for fac in (1.25, 1.5, 2.0)}
|
|
moved = {p: tuple(runs[f][p][0] for f in (1.25, 1.5, 2.0)) for p in runs[2.0]
|
|
if len({runs[f][p][0] for f in (1.25, 1.5, 2.0)}) > 1}
|
|
check("Table 1a: the one canonical problem the multiplier moves", moved,
|
|
{"aps.13.00": (20, 30, 28)})
|
|
tot = {}
|
|
for sc in ("derived3", "derived3_m"):
|
|
pp = per_problem(lambda f, a, b, xt, mx, _s=sc: pure_root(
|
|
f, a, b, xt, mx, similarity="ratio", scale=_s, guard="project"), probs, 200,
|
|
criterion="sign")
|
|
tot[sc] = sum(v[0] for v in pp.values())
|
|
check("item 5: phase detector vs base rule, both sets", (tot["derived3_m"], tot["derived3"]),
|
|
(5336, 4991))
|
|
|
|
|
|
def verify_controls():
|
|
"""Section 9.1: the controls (PREREG_controls_2026-09-22, sweep_controls.json)."""
|
|
import math
|
|
print("\nSection 9.1 (controls, sweep_controls.json):")
|
|
R = _load("sweep_controls.json")
|
|
seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json")
|
|
ap = dict(seen["Aprime"])
|
|
ap.update(four["Aprime"])
|
|
chf = dict(seen["chandrupatla"])
|
|
chf.update(four["chandrupatla"])
|
|
sets = ("APS", "CHA", "NEW", "RJL", "APP", "D")
|
|
|
|
def cost(v):
|
|
return v[0] if v[1] else 200
|
|
|
|
def tot(r, keep=None):
|
|
return sum(cost(v) for p, v in r.items() if keep is None or keep(p))
|
|
|
|
def h2h(a, b, keep=None):
|
|
ps = [p for p in a if keep is None or keep(p)]
|
|
w = sum(cost(a[p]) < cost(b[p]) for p in ps)
|
|
l = sum(cost(a[p]) > cost(b[p]) for p in ps)
|
|
return w, len(ps) - w - l, l
|
|
|
|
def sign_p(w, l):
|
|
k, n = min(w, l), w + l
|
|
return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n)
|
|
rows = {"K2L": (2235, 1437, 4026, 1495, 1490, 4488, 15171),
|
|
"K2P": (2682, 2526, 5485, 2117, 1622, 6236, 20668),
|
|
"K2F": (2693, 2526, 5569, 2126, 1670, 6165, 20749)}
|
|
check("Table 9b A' row", tuple(tot(ap[s]) for s in sets) + (sum(tot(ap[s]) for s in sets),),
|
|
(2259, 1074, 3290, 1371, 1382, 3656, 13032))
|
|
for k, want in rows.items():
|
|
k2 = R["K2"][k]
|
|
check(f"Table 9b {k} row", tuple(tot(k2[s]) for s in sets) + (sum(tot(k2[s]) for s in sets),),
|
|
want)
|
|
for k, wtl, pp in (("K2L", (84, 121, 35), ("1g", "8e-06")), ("K2P", (110, 96, 34), ("2g", "1.5e-10")),
|
|
("K2F", (113, 87, 40), ("1g", "3e-09"))):
|
|
got = h2h(ap["D"], R["K2"][k]["D"])
|
|
check(f"Table 9b A' vs {k} on corpus 4", got, wtl)
|
|
check(f"Table 9b p vs {k}", format(sign_p(got[0], got[2]), "." + pp[0]), pp[1])
|
|
check("Table 9b violations", sum(v[2] for k in rows for s in sets for v in R["K2"][k][s].values()), 0)
|
|
for cls, want in (("D2", (1163, 1739)), ("D3", (924, 1207)), ("D5", (503, 477))):
|
|
keep = lambda p, c=cls: p.startswith(c) # noqa: E731
|
|
check(f"9.1 {cls}: A' vs Lagouanelle", (tot(ap["D"], keep), tot(R["K2"]["K2L"]["D"], keep)), want)
|
|
level = [(tot(ap["D"], lambda p, c=c: p.startswith(c)),
|
|
tot(R["K2"]["K2L"]["D"], lambda p, c=c: p.startswith(c))) for c in ("D1", "D4", "D6")]
|
|
check("9.1 level on the rest (within 1.5%)", all(abs(a - b) <= 0.015 * b for a, b in level), True)
|
|
ab = _load("sweep_ablation_all.json")["abl-regime"]
|
|
no_regime_all = sum(tot(ab[s]) for s in sets)
|
|
check("band and no-detector worse than no regime (six corpora)",
|
|
all(sum(tot(R["K2"][k][s]) for s in sets) > no_regime_all for k in ("K2P", "K2F")), True)
|
|
check("band and no-detector worse than no regime (corpus 4)",
|
|
all(tot(R["K2"][k]["D"]) > tot(ab["D"]) for k in ("K2P", "K2F")), True)
|
|
t9c = {"CHA": (20, 654, 96, 66, 1068), "NEW": (40, 1229, 358, 337, 1792),
|
|
"D": (88, 2159, 901, 844, 4151)}
|
|
nonsimple = lambda m: m >= 2.0 or m <= 2.0 / 3.0 # noqa: E731
|
|
wins = 0
|
|
for s, want in t9c.items():
|
|
k1 = R["K1"][s]
|
|
keep = lambda p, _m=k1["mstar"]: nonsimple(_m[p]) # noqa: E731
|
|
got = (sum(1 for p in k1["mstar"] if keep(p)), tot(ap[s], keep), tot(k1["chandrupatla_g"], keep),
|
|
tot(k1["brent_g"], keep), tot(chf[s], keep))
|
|
check(f"Table 9c {s}", got, want)
|
|
wins += h2h({p: v for p, v in ap[s].items() if keep(p)}, k1["chandrupatla_g"])[0]
|
|
check("Table 9c: A' wins none of the 148", wins, 0)
|
|
check("Table 9c: 2.4 to 6.8 times", (round(2159 / 901, 1), round(654 / 96, 1),
|
|
min(round(a / c, 1) for _n, a, c, _b, _f in t9c.values()),
|
|
max(round(a / c, 1) for _n, a, c, _b, _f in t9c.values())),
|
|
(2.4, 6.8, 2.4, 6.8))
|
|
APS_ = "APS"
|
|
for s, want in ((APS_, (2259, 2640)), ("CHA", (420, 437)), ("NEW", (2061, 1924)), ("D", (1497, 1427))):
|
|
k1 = R["K1"][s]
|
|
keep = lambda p, _m=k1["mstar"]: not nonsimple(_m[p]) # noqa: E731
|
|
check(f"9.1 rest {s}: A' vs Chandrupatla on g", (tot(ap[s], keep), tot(k1["chandrupatla_g"], keep)),
|
|
want)
|
|
|
|
|
|
def verify_revision2():
|
|
"""Revision 2 (points 3, 4, 5, 7)."""
|
|
import math
|
|
print("\nRevision 2:")
|
|
|
|
def cost(v):
|
|
return v[0] if v[1] else 200
|
|
|
|
def tot(r, keep=None):
|
|
return sum(cost(v) for p, v in r.items() if keep is None or keep(p))
|
|
|
|
def sign_p(w, l):
|
|
k, n = min(w, l), w + l
|
|
return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n)
|
|
sets = ("APS", "CHA", "NEW", "RJL", "APP", "D")
|
|
K = _load("sweep_estimated.json")
|
|
seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json")
|
|
ap, ch = dict(seen["Aprime"]), dict(seen["chandrupatla"])
|
|
ap.update(four["Aprime"])
|
|
ch.update(four["chandrupatla"])
|
|
K1 = _load("sweep_controls.json")["K1"]
|
|
nonsimple = lambda m: m >= 2.0 or m <= 2.0 / 3.0 # noqa: E731
|
|
|
|
def group(r):
|
|
return sum(tot(r[s], lambda p, _m=K1[s]["mstar"]: nonsimple(_m[p])) for s in ("CHA", "NEW", "D"))
|
|
rows = {"A'": ap, "K3A": K["k3_analogy"], "K3L": K["k3_lag"], "Ch(f)": ch}
|
|
want = {"A'": (3656, 13032, 4042, 2259), "K3A": (3806, 15733, 2809, 3453),
|
|
"K3L": (7526, 30991, 6041, 7215), "Ch(f)": (5717, 16481, 7011, 2595)}
|
|
for k, r in rows.items():
|
|
check(f"Table 12 {k}", (tot(r["D"]), sum(tot(r[s]) for s in sets), group(r), tot(r["APS"])), want[k])
|
|
oracle = sum(tot(K1[s]["chandrupatla_g"], lambda p, _m=K1[s]["mstar"]: nonsimple(_m[p]))
|
|
for s in ("CHA", "NEW", "D"))
|
|
check("Table 12 oracle row (Table 11 sum)", oracle, 1355)
|
|
ps = list(ap["D"])
|
|
w = sum(cost(ap["D"][p]) < cost(K["k3_analogy"]["D"][p]) for p in ps)
|
|
l = sum(cost(ap["D"][p]) > cost(K["k3_analogy"]["D"][p]) for p in ps)
|
|
check("A' vs Chandrupatla-on-g with the analogy order, corpus 4", (w, len(ps) - w - l, l), (126, 38, 76))
|
|
check("its p (1 digit)", f"{sign_p(w, l):.1g}", "0.0005")
|
|
shares = []
|
|
for s in ("CHA", "NEW", "D"):
|
|
keep = lambda p, _m=K1[s]["mstar"]: nonsimple(_m[p]) # noqa: E731
|
|
a_, k_, o_ = tot(ap[s], keep), tot(K["k3_analogy"][s], keep), tot(K1[s]["chandrupatla_g"], keep)
|
|
shares.append((k_, a_, round(100 * (a_ - k_) / (a_ - o_))))
|
|
check("share of the A'-to-oracle gap closed by the analogy order", shares,
|
|
[(322, 654, 59), (721, 1229, 58), (1766, 2159, 31)])
|
|
check("K3 bound violations", sum(v[2] for k in K for s in sets for v in K[k][s].values()), 0)
|
|
# Lagouanelle's fractional readings on the simple roots of the canonical set (fresh)
|
|
from simroot.transformed import chandrupatla_transformed
|
|
from sweep_full import load_problems
|
|
frac = 0
|
|
for s, pid, f, a, b, _r in load_problems():
|
|
if s != "APS":
|
|
continue
|
|
st = {}
|
|
chandrupatla_transformed(f, a, b, 1e-12, 200, estimator="lag", stats=st)
|
|
m = st.get("m_hat", 1.0)
|
|
frac += (0.0 < m <= 2.0 / 3.0)
|
|
check("Lagouanelle holds a fractional order on simple APS roots", frac, 115)
|
|
# point 4: exact-zero stops (fresh)
|
|
import contextlib
|
|
import io
|
|
import diag_revision2
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
z = diag_revision2.point4()
|
|
check("exact-zero stops on CHA: Brent(g), Chandrupatla(g), A'",
|
|
(z["CHA"]["brent_g"][1], z["CHA"]["chandrupatla_g"][1], z["CHA"]["Aprime"][1]), (15, 4, 2))
|
|
check("exact-zero stops of the transformed runs on corpus 1",
|
|
(z["NEW"]["chandrupatla_g"][1], z["NEW"]["brent_g"][1]), (0, 0))
|
|
check("corpus 1: transformation 3.4 times cheaper", round(1229 / 358, 1), 3.4)
|
|
# point 3: the three unchanged rows of Table 2 (fresh)
|
|
from corpus_roots_jl import solved
|
|
from sweep_full import make_method
|
|
f, a, b = next((f, a, b) for s, pid, f, a, b, _r in load_problems() if pid == "aps.13.00")
|
|
got = []
|
|
for name in ("dekker", "illinois", "regula_falsi"):
|
|
x, n, conv = make_method(("base", name))(f, a, b, 1e-12, 200)
|
|
got.append((name, int(n), bool(conv), bool(solved(f, float(x)))))
|
|
check("aps.13.00: Dekker, Illinois, false position spend the budget unsolved",
|
|
got, [("dekker", 200, False, False), ("illinois", 200, False, False),
|
|
("regula_falsi", 200, False, False)])
|
|
small = [abs(float(f(float(make_method(("base", nm))(f, a, b, 1e-12, 200)[0])))) < 1e-62
|
|
for nm in ("dekker", "illinois")]
|
|
check("Dekker and Illinois stall where |f| < 1e-62", small, [True, True])
|
|
# point 7: Table 3's two added rows
|
|
full = {r["label"]: r for r in _load("sweep_full.json")}
|
|
for lb, want_ in (("pure.exp_abs.bracket.bracket", (1848, 45, 0)), ("pure.exp_abs.bracket.credit", (1952, 45, 0))):
|
|
c = full[lb]["CHA"]
|
|
check(f"Table 3 row {lb}", (c["total"], c["solved"], c["over"]), want_)
|
|
|
|
|
|
def verify_revision3():
|
|
"""Revision 3 (point 5): the post hoc branch m >= 2 and the reading of Table 12."""
|
|
print("\nRevision 3:")
|
|
|
|
def cost(v):
|
|
return v[0] if v[1] else 200
|
|
|
|
def tot(r, keep=None):
|
|
return sum(cost(v) for p, v in r.items() if keep is None or keep(p))
|
|
sets = ("APS", "CHA", "NEW", "RJL", "APP", "D")
|
|
P = _load("sweep_estimated_posthoc.json")
|
|
K = _load("sweep_estimated.json")["k3_analogy"]
|
|
seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json")
|
|
ap = dict(seen["Aprime"])
|
|
ap.update(four["Aprime"])
|
|
K1 = _load("sweep_controls.json")["K1"]
|
|
nonsimple = lambda m: m >= 2.0 or m <= 2.0 / 3.0 # noqa: E731
|
|
|
|
def group(r):
|
|
return sum(tot(r[s], lambda p, _m=K1[s]["mstar"]: nonsimple(_m[p])) for s in ("CHA", "NEW", "D"))
|
|
check("Table 12 post hoc row", (tot(P["D"]), sum(tot(P[s]) for s in sets), group(P), tot(P["APS"])),
|
|
(5568, 18372, 6540, 2960))
|
|
ps = list(ap["D"])
|
|
w = sum(cost(ap["D"][p]) < cost(P["D"][p]) for p in ps)
|
|
l = sum(cost(ap["D"][p]) > cost(P["D"][p]) for p in ps)
|
|
check("A' vs the post hoc variant on corpus 4", (w, len(ps) - w - l, l), (178, 34, 28))
|
|
check("post hoc violations", sum(v[2] for s in sets for v in P[s].values()), 0)
|
|
gain = group(ap) - group(K)
|
|
loss = (sum(tot(K[s]) for s in sets) - group(K)) - (sum(tot(ap[s]) for s in sets) - group(ap))
|
|
check("transferred detector: gain on multiple roots < loss elsewhere (vs A')", (gain, loss, gain < loss),
|
|
(1233, 3934, True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|