"""Check every number the paper states against sweep_full.json. A paper is a claim about data; transcribing its tables by hand is how a claim stops matching its data. A failure here means the paper is wrong, not the test. PROPOSED/PROJ below is the BASE rule of section 3: the scheduled projection with its multiplier DECLARED as the one free constant (Valeri, 2026-09-21, variant a); the method the paper now proposes, A', is checked in verify_flat_regime_and_oos(); the earned projection (credit) is the constant-free variant. Both are checked. Baselines are the library alone since 2026-09-21: Chandrupatla = scipy elementwise.find_root, Brent = brentq. """ from __future__ import annotations import json import os import sys from report_sweep import CREDIT, PROPOSED, head_to_head, place, rank FAILS = [] PROJ = "pure.ratio.derived3.project" assert PROPOSED == PROJ, "the paper proposes the projection (variant a)" def check(label, got, want): ok = got == want print(f" {'OK ' if ok else 'BAD'} {label:58s} got {got!r:>16} " f"want {want!r}") if not ok: FAILS.append(label) def row(rows, setname, lbl): return next(r for r in rows if r["label"] == lbl)[setname] def main(): rows = _load("sweep_full.json") # Guard against quoting places from a field that is missing a competitor. # It has already happened once: before Chandrupatla's own method was # added, every ranking assertion here passed only because the method that # beats us was absent. labels = {r["label"] for r in rows} missing = {"chandrupatla", "dekker", PROPOSED} - labels if missing: print(f" !! THE SWEEP IS STALE: missing {sorted(missing)}. " "Re-run sweep_full.py.\n") FAILS.append("sweep is stale") check("number of configurations", len(rows), 264) # ---- the proposed method: projection, one declared constant ---------- print("\nThe proposed method (projection, multiplier declared):") for setname, pl, tot, sol in (("APS", 3, 2634, 154), ("CHA", 49, 2357, 45)): d = row(rows, setname, PROPOSED) check(f"{setname} place", place(rows, setname, PROPOSED), pl) check(f"{setname} total", d["total"], tot) check(f"{setname} solved", d["solved"], sol) check("both sets", row(rows, "APS", PROPOSED)["total"] + row(rows, "CHA", PROPOSED)["total"], 4991) print("\nHead to head of the proposed method:") for setname, opp, want in [ ("APS", "chandrupatla", (87, 24, 43)), ("APS", "toms748", (86, 16, 52)), ("APS", "brent", (102, 11, 41)), ("APS", "itp", (132, 3, 19)), ("APS", "itp_oracle", (121, 4, 29)), ("APS", "bisection", (152, 1, 1)), ("CHA", "chandrupatla", (11, 5, 29)), ("CHA", "toms748", (33, 1, 11)), ("CHA", "brent", (34, 4, 7)), ("CHA", "itp", (18, 1, 26)), ("CHA", "itp_oracle", (16, 1, 28))]: w, t, l, _ = head_to_head(rows, PROPOSED, opp, setname) check(f"{setname} vs {opp}", (w, t, l), want) def sign_p(w, l): import math as _m k, n = min(w, l), w + l return min(1.0, 2.0 * sum(_m.comb(n, i) for i in range(k + 1)) / 2.0 ** n) w, _t, l, _ = head_to_head(rows, PROPOSED, "chandrupatla", "APS") check("APS vs Chandrupatla sign test p (2 digits)", float(f"{sign_p(w, l):.2g}"), 0.00014) w, _t, l, _ = head_to_head(rows, PROPOSED, "chandrupatla", "CHA") check("CHA vs Chandrupatla sign test p (rounded)", round(sign_p(w, l), 3), 0.006) # ---- the constant-free variant --------------------------------------- print("\nThe constant-free variant (earned projection):") for setname, pl, tot in (("APS", 25, 3377), ("CHA", 9, 2073)): check(f"credit {setname} place", place(rows, setname, CREDIT), pl) check(f"credit {setname} total", row(rows, setname, CREDIT)["total"], tot) check("credit both sets", row(rows, "APS", CREDIT)["total"] + row(rows, "CHA", CREDIT)["total"], 5450) check("credit costs 743 more on the canonical set", row(rows, "APS", CREDIT)["total"] - row(rows, "APS", PROPOSED)["total"], 743) for setname, opp, want in [("APS", "chandrupatla", (79, 3, 72)), ("APS", "toms748", (78, 9, 67)), ("APS", "brent", (79, 5, 70))]: w, t, l, _ = head_to_head(rows, CREDIT, opp, setname) check(f"credit {setname} vs {opp}", (w, t, l), want) w, _t, l, _ = head_to_head(rows, CREDIT, "chandrupatla", "APS") check("credit APS vs Chandrupatla sign test p (rounded)", round(sign_p(w, l), 2), 0.63) # ---- what the earlier, order-destroying guards cost -------------------- print("\nThe guards that replace the step instead of projecting it:") check("APS unguarded", row(rows, "APS", "pure.ratio.derived3.none")["total"], 2634) cha_none = row(rows, "CHA", "pure.ratio.derived3.none") check("CHA unguarded total", cha_none["total"], 3391) check("CHA unguarded solved", cha_none["solved"], 39) check("CHA unguarded bound violations", cha_none["over"], 10) check("APS bracket-guard", row(rows, "APS", "pure.ratio.derived3.bracket")["total"], 7068) check("APS both-guard", row(rows, "APS", "pure.ratio.derived3.both")["total"], 7046) # ---- the field -------------------------------------------------------- print("\nAlefeld-Potra-Shi:") check("APS winner", rank(rows, "APS")[0]["label"], "chandrupatla") for lbl, pl, tot in [("chandrupatla", 1, 2595), ("toms748", 4, 2636), ("brent", 5, 2707), ("dekker", 232, 3158), ("itp_oracle", 11, 3068), ("ridders", 15, 3124), ("itp", 29, 3767), ("illinois", 237, 7045), ("bisection", 73, 7338), ("regula_falsi", 244, 10560)]: check(f"APS {lbl} place", place(rows, "APS", lbl), pl) check(f"APS {lbl} total", row(rows, "APS", lbl)["total"], tot) check("APS regula falsi solved", row(rows, "APS", "regula_falsi")["solved"], 127) print("\nChandrupatla:") top = rank(rows, "CHA")[0] check("CHA rank 1 label", top["label"], "chandrupatla") check("CHA rank 1 solves all 45", top["CHA"]["solved"], 45) for lbl, pl, tot in [("chandrupatla", 1, 1505), ("itp_oracle", 2, 1675), ("itp", 5, 1946), ("bisection", 15, 2136), ("toms748", 158, 3088), ("brent", 227, 3238)]: check(f"CHA {lbl} place", place(rows, "CHA", lbl), pl) check(f"CHA {lbl} total", row(rows, "CHA", lbl)["total"], tot) check("CHA brent solved", row(rows, "CHA", "brent")["solved"], 43) ours_both = (row(rows, "APS", PROPOSED)["total"] + row(rows, "CHA", PROPOSED)["total"]) cha_both = (row(rows, "APS", "chandrupatla")["total"] + row(rows, "CHA", "chandrupatla")["total"]) check("chandrupatla, both sets", cha_both, 4100) check("Chandrupatla beats us on both sets", cha_both < ours_both, True) # ---- the regime findings --------------------------------------------- print("\nNegative results, the regime claims:") check("CHA cbrt over the whole set", row(rows, "CHA", "pure.cbrt.derived3.project")["total"], 3204) switched = (row(rows, "APS", PROJ)["total"] + row(rows, "CHA", "pure.cbrt.derived3.project")["total"]) single = row(rows, "APS", PROJ)["total"] + row(rows, "CHA", PROJ)["total"] check("single fixed choice, both sets", single, 4991) check("switch by benchmark set, both sets", switched, 5838) check("switching is worse than not switching", switched > single, True) for pid, ours_n, cb_n in (("fun4.1", 89, 32), ("fun4.2", 93, 32), ("fun4.3", 113, 40), ("fun4.4", 125, 47), ("fun4.5", 153, 59)): o = row(rows, "CHA", PROJ)["per"][pid] c = row(rows, "CHA", "pure.cbrt.derived3.project")["per"][pid] check(f"{pid} projection variant", o[0], ours_n) check(f"{pid} cbrt", c[0], cb_n) # ---- the law across the grid ------------------------------------------ print("\nThe law across the grid:") lin = ("ratio", "rational", "exp_abs") quad = ("rat_sq", "exp_sq", "cosine") want = {("derived3", "lin"): 5764, ("derived3", "quad"): 21200, ("derived", "lin"): 5854, ("derived", "quad"): 21089, ("bracket", "lin"): 9149, ("bracket", "quad"): 18015, ("start", "lin"): 14983, ("start", "quad"): 14228} for scale in ("derived3", "derived", "bracket", "start"): for group, gname in ((lin, "lin"), (quad, "quad")): tot = cnt = 0 for r in rows: if r["spec"][0] != "pure": continue _, sim, sc, gu = r["spec"] if sc != scale or gu != "step" or sim not in group: continue cnt += 1 tot += r["APS"]["total"] + r["CHA"]["total"] check(f"mean total, scale={scale}, {gname}", round(tot / cnt), want[(scale, gname)]) best_by_sim, best_by_scale = {}, {} for r in rows: if r["spec"][0] != "pure" or r["spec"][3] != "step": continue t = r["APS"]["total"] + r["CHA"]["total"] best_by_sim[r["spec"][1]] = min(best_by_sim.get(r["spec"][1], 10 ** 9), t) best_by_scale[r["spec"][2]] = min(best_by_scale.get(r["spec"][2], 10 ** 9), t) check("every c=1 similarity beats every c=0 similarity", max(best_by_sim[s] for s in lin) < min(best_by_sim[s] for s in quad), True) check("similarity ranking", [s for s, _ in sorted(best_by_sim.items(), key=lambda kv: kv[1])], ["ratio", "exp_abs", "rational", "ratio_soft", "sqrt", "cbrt", "exp_sq", "rat_sq", "cosine"]) for sim, v in [("ratio", 5422), ("exp_abs", 5772), ("rational", 5993), ("ratio_soft", 8455), ("sqrt", 9411), ("cbrt", 10874), ("exp_sq", 13187), ("rat_sq", 13990), ("cosine", 15272)]: check(f"similarity {sim} best", best_by_sim[sim], v) check("scale ranking", [s for s, _ in sorted(best_by_scale.items(), key=lambda kv: kv[1])], ["derived3", "derived", "bracket", "start"]) for sc, v in [("derived3", 5422), ("derived", 5564), ("bracket", 7972), ("start", 13423)]: check(f"scale {sc} best", best_by_scale[sc], v) verify_placement_modes() verify_per_place() verify_decision_place() verify_gap_decomposition() verify_safeguard_and_tables() verify_flat_regime_and_oos() verify_revision1() verify_controls() verify_revision2() verify_revision3() print() if FAILS: print(f"{len(FAILS)} STATED NUMBERS DO NOT MATCH THE DATA:") for f in FAILS: print(f" {f}") sys.exit(1) print("every number stated in the paper matches the sweep data.") # -------------------------------------------------------------------------- # Section "where the similarity is read" (PREREGs of 2026-09-20/21). Each new # grid lives in its OWN sweep file so the published sweep_full.json stays the # artefact of the original claims; every number the new section quotes is # re-checked here against that file. # -------------------------------------------------------------------------- PLACEMENT = { # file label APS CHA "sweep_slope.json": ("slope.ratio.derived3.project", 2663, 2170), "sweep_points.json": ("pure+shape+slope.ratio.derived3.safe", 2337, 2177), "sweep_direction.json": ("pure+dissim+slope.ratio.derived3.project", 2539, 1864), "sweep_combo.json": ("pure+shape+dissim+slope.ratio.derived3.safe", 2695, 1928), "sweep_gate.json": ("progress.ratio.derived.project", 4258, 2093), "sweep_log.json": ("log.ratio_soft.derived.safe", 6749, 2598), } # The per-place table of section "the similarity function belongs to the place": # seven PAIRED comparisons in sweep_order.json. Checked as pairs, because a pair is # what the pre-registration licensed and a single cell would prove nothing. ORDER_PAIRS = { # guard (APS m=1, APS measured, CHA m=1, CHA measured) "both": (4331, 3485, 2469, 2306), "bracket": (4418, 3352, 2671, 2428), "credit": (4999, 4495, 2064, 1933), "project": (6783, 6176, 4057, 3751), "safe": (5803, 5040, 3218, 3032), "step": (6251, 5362, 3371, 3146), "none": (10676, 9855, 8265, 7424), } # the negative item: the order does not cure false position ORDER_FALSEPOS = {("rational", "far", "none", "none"): (127, 10676), ("rational", "far", "none", "order"): (129, 9855)} # and the harmful direction, quoted in the same subsection ORDER_SLOPE_HARM = {("ratio", "derived3", "none", "none"): 2634, ("ratio", "derived3", "none", "order"): 2703} def verify_per_place(): print("\nThe per-place reading (sweep_order.json):") if not os.path.exists("sweep_order.json"): print(" !! sweep_order.json absent — section 'belongs to the place' unverifiable") FAILS.append("missing sweep_order.json") return rows = _load("sweep_order.json") def cell(sim, sc, gu, ex): for r in rows: s = r["spec"] if (s[1], s[2], s[3], s[8] if len(s) > 8 else "none") == (sim, sc, gu, ex): return r return None cheaper = 0 for gu, (a1, a2, c1, c2) in ORDER_PAIRS.items(): p = cell("rational", "far", gu, "none") q = cell("rational", "far", gu, "order") if p is None or q is None: FAILS.append(f"order pair {gu} missing from sweep_order.json") print(f" !! pair {gu} missing — stale sweep") continue check(f"far+rational {gu} APS m=1", p["APS"]["total"], a1) check(f"far+rational {gu} APS order", q["APS"]["total"], a2) check(f"far+rational {gu} CHA m=1", p["CHA"]["total"], c1) check(f"far+rational {gu} CHA order", q["CHA"]["total"], c2) cheaper += (q["APS"]["total"] + q["CHA"]["total"] < p["APS"]["total"] + p["CHA"]["total"]) check("cheaper in 7 of 7 safeguards", cheaper, 7) for (sim, sc, gu, ex), (solved, total) in ORDER_FALSEPOS.items(): r = cell(sim, sc, gu, ex) if r is None: FAILS.append(f"false-position cell {ex} missing") continue check(f"false position {ex} solved", r["APS"]["solved"], solved) check(f"false position {ex} APS", r["APS"]["total"], total) for (sim, sc, gu, ex), want in ORDER_SLOPE_HARM.items(): r = cell(sim, sc, gu, ex) if r is None: FAILS.append(f"slope-harm cell {ex} missing") continue check(f"slope reading {ex} APS", r["APS"]["total"], want) fam = [r for r in rows if r["spec"][3] in ("project", "safe", "credit")] check("guarantee family violations", sum(r["APS"]["over"] + r["CHA"]["over"] for r in fam), 0) DECISION_CELLS = { # (similarity, prestep, guard): {set: total} ("ratio", "band", "project"): {"APS": 5242, "CHA": 2101}, ("ratio", "band", "credit"): {"CHA": 1942}, ("ratio", "none", "project"): {"APS": 2634}, ("ratio", "defect", "project"): {"APS": 2351, "CHA": 2346}, ("ratio_soft", "simpair", "project"): {"CHA": 1941}, } DECISION_RATES = {"band": 63.5, "defect": 6.6} def verify_decision_place(): print("\nThe decision place (sweep_decision.json):") if not os.path.exists("sweep_decision.json"): print(" !! sweep_decision.json absent — section 'decision place' unverifiable") FAILS.append("missing sweep_decision.json") return rows = _load("sweep_decision.json") by = {(r["spec"][1], r["spec"][5], r["spec"][3]): r for r in rows} for k, sets in DECISION_CELLS.items(): r = by.get(k) if r is None: FAILS.append(f"decision cell {k} missing") print(f" !! {k} missing — stale sweep") continue for setname, want in sets.items(): check(f"{'.'.join(k)} {setname}", r[setname]["total"], want) d = by[("ratio", "defect", "project")] check("defect APS solved", d["APS"]["solved"], 154) check("defect both sets", d["APS"]["total"] + d["CHA"]["total"], 4697) check("defect second to the shape reading (2337 < defect)", 2337 < d["APS"]["total"], True) fam = [r for r in rows if r["spec"][3] in ("project", "safe", "credit")] check("decision grid guarantee violations", sum(r["APS"]["over"] + r["CHA"]["over"] for r in fam), 0) # the refusal rates are recomputed, not transcribed from scipy.optimize._tstutils import _APS_TESTS_DICTS, _CHANDRUPATLA_TESTS_DICTS from simroot.pure import pure_root probs = [(float(q["bracket"][0]), float(q["bracket"][1]), (lambda ff, aa: lambda t: ff(t, *aa))(q["f"], q.get("args", ()))) for q in _APS_TESTS_DICTS] probs += [(float(q["bracket"][0]), float(q["bracket"][1]), (lambda ff: lambda t: ff(t))(q["f"])) for q in _CHANDRUPATLA_TESTS_DICTS] for mode, want in DECISION_RATES.items(): ref = nfev = 0 for a, b, f in probs: st = {} try: _x, n, _c = pure_root(f, a, b, 1e-12, 200, similarity="ratio", scale="derived3", guard="project", prestep=mode, stats=st) except Exception: continue ref += st.get("pre_refused", 0) nfev += int(n) check(f"{mode} refusal rate %", round(100.0 * ref / nfev, 1), want) def verify_gap_decomposition(): """Section 7.4 of the .md (sec:widelimit in the .tex): the wide-bracket gap taken apart on the same evaluations. Every number is RECOMPUTED by running the diagnosis scripts, not transcribed from their saved printouts.""" import contextlib import io print("\nThe gap decomposition (fresh runs of the diagnosis scripts):") import diag_flat_detectable import diag_model_order import diag_model_order_checks with contextlib.redirect_stdout(io.StringIO()): mo = diag_model_order.main() ck = diag_model_order_checks.main() fl = diag_flat_detectable.main() per = mo["per_problem"] check("problems where we lose", len(mo["lose"]), 35) check("fun1.4 our bracket halvings/eval", round(per["fun1.4"][2], 2), 0.62) check("fun1.4 IQI from our points", round(per["fun1.4"][3], 2), 10.37) flat = [p for p in per if p.split(".")[0] in ("fun3", "fun4", "fun5", "fun6")] check("flat: ours min", round(min(per[p][2] for p in flat), 2), 0.41) check("flat: ours max", round(max(per[p][2] for p in flat), 2), 0.75) check("flat: IQI min", round(min(per[p][3] for p in flat), 2), 0.01) check("flat: IQI max", round(max(per[p][3] for p in flat), 2), 0.17) check("non-degenerate states with one negative weight", int(mo["neg_ok"]), 431) check("bracketing states in total", mo["neg_all"], 434) check("every exception is an underflowed zero weight", all(any(w == 0.0 for w in ws) for _p, _f, ws in ck["exceptions"]), True) fam = ck["fam"] gap = {k: v[2] - v[3] for k, v in fam.items()} check("total gap on the losing problems", sum(gap.values()), 262) check("flat families' gap", sum(gap[k] for k in ("fun3", "fun4", "fun5", "fun6")), 172) check("fun1 gap", gap["fun1"], 58) for k, want in (("fun3", 92), ("fun4", 96), ("fun5", 100), ("fun6", 100)): b, q = fam[k][0], fam[k][1] check(f"Chandrupatla bisect share {k} %", round(100.0 * b / (b + q)), want) for k, want in (("fun4", 297), ("fun5", 233), ("fun6", 233)): check(f"Chandrupatla total {k}", fam[k][3], want) check("m>=2 on canonical states %", float(round(fl["APS (canonical)"][2], 1)), 0.3) check("m>=2 on flat states %", round(fl["CHA flat fun3-6"][2], 1), 0.0) # the band per family, from the Linux sweep rows = _load("sweep_decision.json") by = {(r["spec"][1], r["spec"][5], r["spec"][3]): r for r in rows} base = by[("ratio", "none", "project")]["CHA"]["per"] band = by[("ratio", "band", "project")]["CHA"]["per"] def fam_total(per_, k): return sum(n if ok else 200 for pid, (n, ok) in per_.items() if pid.split(".")[0] == k) for k, b0, b1 in (("fun4", 573, 296), ("fun5", 456, 239), ("fun6", 456, 228)): check(f"band {k} no-gate", fam_total(base, k), b0) check(f"band {k} with band", fam_total(band, k), b1) for k, d in (("fun1", 164), ("fun2", 111), ("fun8", 170), ("fun9", 144)): check(f"band cost on {k}", fam_total(band, k) - fam_total(base, k), d) # negative item 11, corrected pairing: the same configuration in x and in log space full = _load("sweep_full.json") xcell = next(r for r in full if r["label"] == "pure.ratio_soft.derived.safe") check("log item: x-space control APS", xcell["APS"]["total"], 5635) check("log item: x-space control CHA", xcell["CHA"]["total"], 2759) def verify_safeguard_and_tables(): """§3.1 of the .md (the multiplier is a free constant; the earned budget), Tables 2-3 regenerated with Chandrupatla, §5.1, and the constant-free numbers of §7. The measured tables are RECOMPUTED by running the measurement scripts.""" import contextlib import io import measure_budget_factor import measure_records_vs_multiplier print("\n§3.1: the multiplier and the endowment (fresh runs):") with contextlib.redirect_stdout(io.StringIO()): bf = measure_budget_factor.main() rm = measure_records_vs_multiplier.main() for c, (aps, cha) in ((1.0, (4625, 2039)), (1.25, (2626, 1822)), (1.5, (2636, 1991)), (2.0, (2634, 2357)), (3.0, (2634, 2936)), (4.0, (2634, 3229))): check(f"multiplier {c} APS", bf[c][0], aps) check(f"multiplier {c} CHA", bf[c][2], cha) for c0, (aps, cha) in ((0.1, (5150, 2097)), (0.5, (4462, 2037)), (1.0, (3377, 2073)), (2.0, (3502, 1757)), (5.0, (2819, 1666))): check(f"endowment {c0} APS", bf["endowment"][c0][0], aps) check(f"endowment {c0} CHA", bf["endowment"][c0][1], cha) within_one_step = {c0: sum(v) for c0, v in bf["endowment"].items() if c0 <= 1.0} check("C0=1 cheapest within the one-step promise", min(within_one_step, key=within_one_step.get), 1.0) sh = rm["shape reading (2527)"] check("shape reading at multiplier 1.25", sh[1.25][0], 2351) check("shape reading flat from 1.5 to 4", {sh[f][0] for f in (1.5, 2.0, 3.0, 4.0)}, {2337}) de = rm["defect reading (2523)"] dv = [de[f][0] for f in (1.25, 1.5, 2.0, 3.0, 4.0)] check("defect reading from 1.25 to 4: range", (min(dv), max(dv)), (2343, 2353)) print("\nTables 2-3 regenerated:") rows = _load("sweep_full.json") by = {r["label"]: r for r in rows} none_ = "pure.ratio.derived3.none" for setname, lbl, pl in (("APS", PROJ, 3), ("APS", none_, 2), ("CHA", PROJ, 49), ("CHA", none_, 229), ("CHA", "pure.exp_abs.bracket.both", 3), ("CHA", "pure.exp_abs.bracket.bracket", 4), ("CHA", "pure.exp_abs.bracket.credit", 6), ("APS", CREDIT, 25), ("CHA", CREDIT, 9)): check(f"{setname} {lbl} place", place(rows, setname, lbl), pl) check("CHA exp_abs.bracket.both total", by["pure.exp_abs.bracket.both"]["CHA"]["total"], 1778) check("CHA TOMS 748 over the 2x bound", by["toms748"]["CHA"]["over"], 15) check("CHA Brent over the 2x bound", by["brent"]["CHA"]["over"], 18) for lbl, setname, opp, want in ( (CREDIT, "APS", "itp_oracle", (86, 6, 62)), (CREDIT, "CHA", "itp_oracle", (2, 2, 41)), (CREDIT, "CHA", "bisection", (5, 0, 40)), (PROJ, "APS", "chandrupatla", (87, 24, 43)), (PROJ, "APS", "itp", (132, 3, 19)), (PROJ, "APS", "itp_oracle", (121, 4, 29)), (PROJ, "APS", "bisection", (152, 1, 1)), (PROJ, "CHA", "toms748", (33, 1, 11)), (PROJ, "CHA", "brent", (34, 4, 7)), (PROJ, "CHA", "itp", (18, 1, 26)), (PROJ, "CHA", "itp_oracle", (16, 1, 28))): w, t, l, _ = head_to_head(rows, lbl, opp, setname) check(f"{lbl.split('.')[-1]} {setname} vs {opp}", (w, t, l), want) for lbl, want in ((CREDIT, 5450), (PROJ, 4991), ("chandrupatla", 4100), ("itp_oracle", 4743), ("toms748", 5724), ("brent", 5945)): check(f"both sets {lbl}", by[lbl]["APS"]["total"] + by[lbl]["CHA"]["total"], want) print("\n§7 under the earned projection:") import glob cells = {} for p in glob.glob("sweep_*.json"): if "smoke" in p: continue data = _load(p) # only grid files (a list of labelled cells); the out-of-sample results # (sweep_oos.json, sweep_fourth_*.json, ...) are dicts of a different shape if not isinstance(data, list): continue for r in data: cells.setdefault(r["label"], r) for lbl, aps, cha in (("slope.ratio.derived3.credit", 3316, 2050), ("pure+shape+slope.ratio.derived3.credit", 3564, 2074), ("pure+dissim+slope.ratio.derived3.credit", 3666, 2052), ("pure+shape+dissim+slope.ratio.derived3.credit", 4163, 2087), ("defect.ratio.derived3.credit", 3354, 2002)): check(f"{lbl} APS", cells[lbl]["APS"]["total"], aps) check(f"{lbl} CHA", cells[lbl]["CHA"]["total"], cha) free = [r for r in cells.values() if r.get("spec") and len(r["spec"]) > 3 and r["spec"][0] == "pure" and r["spec"][3] == "credit" and r["APS"]["solved"] + r["CHA"]["solved"] == r["APS"]["n"] + r["CHA"]["n"]] top = min(free, key=lambda r: r["APS"]["total"] + r["CHA"]["total"]) check("best constant-free configuration", top["label"], "defect+flatanalogmempt+exp+half.ratio.derived3.credit") check("its total over both sets", top["APS"]["total"] + top["CHA"]["total"], 4965) d = cells["defect.ratio.derived3.credit"] check("defect under the earned projection, both sets", d["APS"]["total"] + d["CHA"]["total"], 5356) PLACEMENT_PER_SET = { "sweep_direction.json": ("CHA", "pure+dissim+slope.ratio.derived3.safe", 1752), "sweep_gate.json": ("CHA", "progress.ratio_soft.derived.safe", 1929), } def verify_placement_modes(): print("\nThe placement modes (each in its own sweep file):") for path, (lbl, aps, cha) in PLACEMENT.items(): if not os.path.exists(path): print(f" !! {path} absent — the section quoting it cannot be verified") FAILS.append(f"missing {path}") continue rows = _load(path) try: d = next(r for r in rows if r["label"] == lbl) except StopIteration: print(f" !! {lbl} not in {path} — stale sweep") FAILS.append(f"{lbl} missing from {path}") continue check(f"{lbl} APS", d["APS"]["total"], aps) check(f"{lbl} CHA", d["CHA"]["total"], cha) check(f"{lbl} solved", d["APS"]["solved"] + d["CHA"]["solved"], 199) check(f"{lbl} guarantee (over 2x)", d["APS"]["over"] + d["CHA"]["over"], 0) for path, (setname, lbl, want) in PLACEMENT_PER_SET.items(): if not os.path.exists(path): continue rows = _load(path) d = next((r for r in rows if r["label"] == lbl), None) if d is None: FAILS.append(f"{lbl} missing from {path}") continue check(f"{lbl} {setname} per-set", d[setname]["total"], want) def _load(path): """A results file. A row-format protocol sweep (a list of labelled cells) is served from sweep_uniform.json, its re-scoring under the paper's single criterion - a sign change within 1e-6 or f(x) == 0 (revision point 3, 2026-09-22); every other file as stored.""" from rescore_uniform import load_uniform with open(path, encoding="utf8") as fh: data = json.load(fh) if isinstance(data, list) and data and "spec" in data[0]: return load_uniform(path) return data def _cell(path, lbl): return next(r for r in _load(path) if r["label"] == lbl) def _flat_sum(r): return sum(n if ok else 200 for pid, (n, ok) in r["CHA"]["per"].items() if pid.split(".")[0] in ("fun3", "fun4", "fun5", "fun6")) def verify_flat_regime_and_oos(): """Sections 8-9 and negative items 12-16 of the .md, and the abstract that quotes them: the flat regime by analogy, the proposed method A', its ablation, and the out-of-sample corpora. Every number is read from a server results file or recomputed by the diagnosis scripts; none is transcribed.""" import contextlib import io import math import re print("\nSection 8 and negative items 12-16 (the development grids):") P = "ratio.derived3.project" for path, lbl, aps, cha in ( ("sweep_flat.json", f"flat.{P}", 2839, 2243), ("sweep_flat2.json", f"flat2.{P}", 2613, 2388), ("sweep_flatcreep.json", f"flatcreep.{P}", 2688, 2399), ("sweep_flatstick.json", f"flatstickmid.{P}", 6707, 2063), ("sweep_flatexit.json", f"flatstickexit.{P}", 2935, 1546), ("sweep_flatjudge.json", f"flatjudge.{P}", 2820, 1532), ("sweep_orderpt.json", f"flatanalogmem.{P}", 2633, None), ("sweep_half.json", f"half.{P}", 2392, None)): r = _cell(path, lbl) check(f"{lbl} APS", r["APS"]["total"], aps) if cha is not None: check(f"{lbl} CHA", r["CHA"]["total"], cha) check(f"{lbl} guarantee", r["APS"]["over"] + r["CHA"]["over"], 0) check("sticky with order from the failed promise, APS", _cell("sweep_flatstick.json", f"flatstick.{P}")["APS"]["total"], 6591) check("sticky midpoint: flat families", _flat_sum(_cell("sweep_flatstick.json", f"flatstickmid.{P}")), 1057) full = {r["label"]: r for r in _load("sweep_full.json")} check("bisection: flat families", _flat_sum(full["bisection"]), 1044) check("Chandrupatla: flat families", _flat_sum(full["chandrupatla"]), 1068) op = {r["label"]: r for r in _load("sweep_orderpt.json")} check("split from the order: flat families, low end", _flat_sum(op[f"flatanalogmempt.{P}"]), 743) check("split from the order: flat families, high end", _flat_sum(op[f"pure+extP+flatanalogpt.{P}"]), 847) check("item 16: remembered + split, with the step pattern, CHA", op[f"pure+extP+flatanalogmempt.{P}"]["CHA"]["total"], 1250) check("item 16: one-state analogy + split, with the step pattern, CHA", op[f"pure+extP+flatanalogpt.{P}"]["CHA"]["total"], 1388) print("\nSection 8 diagnostics (fresh runs):") import diag_analogy_decoupled import diag_flat_analogy import diag_flat_stick buf = io.StringIO() with contextlib.redirect_stdout(buf): diag_flat_analogy.main() out = buf.getvalue() aps = re.search(r"^APS\s+154\s+(\d+)\s+(\d+)\s+(\d+)", out, re.M) flat = re.search(r"^CHA-flat\s+20\s+(\d+)\s+(\d+)\s+(\d+)", out, re.M) check("flag rises on canonical problems", int(aps.group(1)), 142) check("flag + analogy + m>=2 on canonical problems", int(aps.group(3)), 3) check("flag + analogy + m>=2 on flat problems", int(flat.group(3)), 20) for fam in ("fun2", "fun8", "fun9"): m = re.search(rf"^CHA-{fam}\s+5\s+\d+\s+\d+\s+(\d+)", out, re.M) check(f"flag + analogy + m>=2 on {fam}", int(m.group(1)), 0) buf = io.StringIO() with contextlib.redirect_stdout(buf): diag_analogy_decoupled.main() m = re.search(r"mean entry state ([\d.]+) \(same state\) -> ([\d.]+)", buf.getvalue()) check("mean entry state, one state", m.group(1), "12.9") check("mean entry state, remembered", m.group(2), "8.2") buf = io.StringIO() with contextlib.redirect_stdout(buf): diag_flat_stick.main() inf = tot = 0 for d in re.findall(r"order read at entry: (\{[^}]*\})", buf.getvalue()): dd = eval(d) # noqa: S307 - the diagnosis script's own printed dict inf += dd.get("inf", 0) tot += sum(dd.values()) check("order from the failed promise is inf", (inf, tot), (163, 174)) from simroot.pure import pure_root ap = "contra+flatanalogmempt+exp+half+close+sub" def n_of(f, a, b, pre="none", ext="none"): return pure_root(f, a, b, 1e-12, 200, similarity="ratio", scale="derived3", guard="project", prestep=pre, extrapolate=ext)[1] sq = lambda t: math.sqrt(abs(t)) * (0.0 if t == 0 else math.copysign(1.0, t)) # noqa: E731 check("sqrt|x| sgn x, base rule", n_of(sq, -1 / 3, 1.0), 84) check("sin on [3,4], base rule", n_of(math.sin, 3.0, 4.0), 24) check("sin on [3,4], A'", n_of(math.sin, 3.0, 4.0, ap, "pattern"), 7) print("\nTable 8 (A' on the protocol of section 4, sweep_aprime.json):") rows_ap = _load("sweep_aprime.json") A = f"pure+extP+{ap}.ratio.derived3" apr = {r["label"]: r for r in rows_ap} a = apr[f"{A}.project"] check("A' APS", a["APS"]["total"], 2259) check("A' CHA", a["CHA"]["total"], 1074) check("A' both", a["APS"]["total"] + a["CHA"]["total"], 3333) check("A' solved", a["APS"]["solved"] + a["CHA"]["solved"], 199) c = apr[f"{A}.credit"] check("A' earned budget", (c["APS"]["total"], c["CHA"]["total"]), (3417, 1852)) check("A' under every safeguard: violations", sum(r["APS"]["over"] + r["CHA"]["over"] for lb, r in apr.items() if lb.startswith(A)), 0) check("A' safeguards measured", sum(1 for lb in apr if lb.startswith(A + ".")), 7) both = [r for r in _load("sweep_full.json")] + rows_ap for setname, opp, want in (("APS", "chandrupatla", (103, 20, 31)), ("CHA", "chandrupatla", (31, 4, 10)), ("APS", "toms748", (127, 6, 21)), ("CHA", "toms748", (42, 2, 1)), ("APS", "brent", (118, 17, 19)), ("CHA", "brent", (39, 3, 3)), ("APS", "itp_oracle", (126, 7, 21)), ("CHA", "itp_oracle", (35, 1, 9))): w, t, l, _ = head_to_head(both, f"{A}.project", opp, setname) check(f"A' {setname} vs {opp}", (w, t, l), want) def sign_p(w, l): k, n = min(w, l), w + l return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n) check("A' APS vs Chandrupatla p (1 digit)", float(f"{sign_p(103, 31):.1g}"), 3e-10) check("A' CHA vs Chandrupatla p (1 digit)", float(f"{sign_p(31, 10):.1g}"), 0.001) check("A' below Chandrupatla, APS %", round(100 * (1 - 2259 / 2595), 1), 12.9) check("A' below Chandrupatla, CHA %", round(100 * (1 - 1074 / 1505), 1), 28.6) from where_we_lose import per_family af, cf = per_family(a, "CHA"), per_family(full["chandrupatla"], "CHA") check("A' fun9 vs Chandrupatla", (af["fun9"], cf["fun9"]), (65, 55)) check("A' ahead on wide-bracket families", sum(af[k] < cf[k] for k in cf), 7) check("A' fun7 vs Chandrupatla", (af["fun7"], cf["fun7"]), (69, 64)) print("\nTable 9 (ablation of A' on six corpora, sweep_ablation_all.json):") ab = _load("sweep_ablation_all.json") sets = ("APS", "CHA", "NEW", "RJL", "APP", "D") def tot(r, s): return sum(n if ok else 200 for n, ok, _o in r[s].values()) base_ = {s: tot(ab["ablA'"], s) for s in sets} for comp, want in (("-regime", (-7, 1013, 1660, 324, 77, 2269)), ("-point", (-5, 479, 667, 272, 30, 1672)), ("-sub", (-6, 0, -7, 16, 63, 893)), ("-close", (130, 35, 97, 78, 135, -2)), ("-half", (249, 65, 82, -16, -5, 1)), ("-exp", (31, 143, 100, -21, -2, 87)), ("-G2", (-22, -6, 123, -30, -10, 34)), ("-contra", (11, -12, 29, 0, 1, -1))): got = tuple(tot(ab[f"abl{comp}"], s) - base_[s] for s in sets) check(f"ablation {comp}", got, want) print("\nSection 9 (out-of-sample corpora):") seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json") for k in seen: seen[k].update(four[k]) names = {"APS": 154, "CHA": 45, "NEW": 206, "RJL": 78, "APP": 132, "D": 240} for s, n in names.items(): check(f"corpus {s} size", len(seen["Aprime"][s]), n) table10 = {"APS": (2259, 2595, 2707, 2636, (103, 20, 31)), "CHA": (1074, 1505, 3238, 3088, (31, 4, 10)), "NEW": (3290, 3716, 6395, 5746, (108, 33, 65)), "RJL": (1371, 1566, 2455, 2339, (44, 14, 20)), "APP": (1382, 1382, 1503, 1756, (48, 52, 32)), "D": (3656, 5717, 8267, 8084, (177, 35, 28))} def cost(v): return v[0] if v[1] else 200 for s, (ta, tc, tb, tt, wtl) in table10.items(): got = tuple(tot(seen[k], s) for k in ("Aprime", "chandrupatla", "brent", "toms748")) check(f"Table 10 {s} totals", got, (ta, tc, tb, tt)) ps = list(seen["Aprime"][s]) w = sum(cost(seen["Aprime"][s][p]) < cost(seen["chandrupatla"][s][p]) for p in ps) l = sum(cost(seen["Aprime"][s][p]) > cost(seen["chandrupatla"][s][p]) for p in ps) check(f"Table 10 {s} vs Chandrupatla", (w, len(ps) - w - l, l), wtl) check(f"Table 10 {s} A' violations", sum(v[2] for v in seen["Aprime"][s].values()), 0) viol = {k: sum(v[2] for s in names for v in seen[k][s].values()) for k in ("brent", "toms748")} check("Brent / TOMS 748 bound violations over 855", (viol["brent"], viol["toms748"]), (118, 84)) check("all 855 totals", tuple(sum(tot(seen[k], s) for s in names) for k in ("Aprime", "chandrupatla", "brent", "toms748")), (13032, 16481, 24565, 23649)) d4 = seen["Aprime"]["D"] for label, cls, want in (("without targeted", ("D1", "D2", "D4", "D5"), (2403, 3569, (107, 31, 22))), ("simple roots", ("D1", "D4"), (737, 747, (43, 22, 15)))): ps = [p for p in d4 if p[:2] in cls] a_ = sum(cost(d4[p]) for p in ps) c_ = sum(cost(seen["chandrupatla"]["D"][p]) for p in ps) w = sum(cost(d4[p]) < cost(seen["chandrupatla"]["D"][p]) for p in ps) l = sum(cost(d4[p]) > cost(seen["chandrupatla"]["D"][p]) for p in ps) check(f"corpus 4 {label}", (a_, c_, (w, len(ps) - w - l, l)), want) for cls, want in (("D1", (433, 419, (20, 8, 12))), ("D2", (1163, 1805, (39, 1, 0))), ("D3", (924, 1784, (37, 1, 2))), ("D4", (304, 328, (23, 14, 3))), ("D5", (503, 1017, (25, 8, 7))), ("D6", (329, 364, (33, 3, 4)))): ps = [p for p in d4 if p.startswith(cls)] a_ = sum(cost(d4[p]) for p in ps) c_ = sum(cost(seen["chandrupatla"]["D"][p]) for p in ps) w = sum(cost(d4[p]) < cost(seen["chandrupatla"]["D"][p]) for p in ps) l = sum(cost(d4[p]) > cost(seen["chandrupatla"]["D"][p]) for p in ps) check(f"Table 11 {cls}", (a_, c_, (w, len(ps) - w - l, l)), want) c5 = [p for p in seen["Aprime"]["NEW"] if p.startswith("C5")] check("near-cancelling class of corpus 1", (sum(cost(seen["Aprime"]["NEW"][p]) for p in c5), sum(cost(seen["chandrupatla"]["NEW"][p]) for p in c5)), (491, 391)) # the pre-registered verdicts on corpora 1-3 (variant without close and sub) oos = _load("sweep_oos.json") def tot_new(r): return sum(n if ok else 200 for n, ok, _o in r["NEW"].values()) def h2h_new(r, o, keep=lambda p: True): ps = [p for p in r["NEW"] if keep(p)] w = sum(cost(r["NEW"][p]) < cost(o["NEW"][p]) for p in ps) l = sum(cost(r["NEW"][p]) > cost(o["NEW"][p]) for p in ps) return w, len(ps) - w - l, l check("corpus 1, variant carried: total vs Chandrupatla", (tot_new(oos["A"]), tot_new(oos["chandrupatla"])), (3492, 3899)) check("corpus 1, variant carried: per problem", h2h_new(oos["A"], oos["chandrupatla"]), (105, 34, 67)) check("corpus 1, with defect: per problem", h2h_new(oos["B"], oos["chandrupatla"]), (95, 33, 78)) check("corpus 1, with defect: exponential class", h2h_new(oos["B"], oos["chandrupatla"], lambda p: p.startswith("C4")), (9, 5, 26)) rj = _load("sweep_rootsjl.json") keep = [p for p, v in rj["A"].items() if not v[3]] check("Roots.jl, variant carried: totals", (sum(cost(rj["A"][p]) for p in keep), sum(cost(rj["chandrupatla"][p]) for p in keep)), (1464, 1566)) w = sum(cost(rj["A"][p]) < cost(rj["chandrupatla"][p]) for p in keep) l = sum(cost(rj["A"][p]) > cost(rj["chandrupatla"][p]) for p in keep) check("Roots.jl, variant carried: per problem", (w, len(keep) - w - l, l), (38, 10, 30)) app = _load("sweep_applied.json") check("applied, variant carried: totals", (sum(cost(v) for v in app["A"].values()), sum(cost(v) for v in app["chandrupatla"].values())), (1580, 1382)) print("\nModAB (sweep_modab.json, the authors' package):") mb = _load("sweep_modab.json") ms = mb["seen"]["modab"] check("Table 8 ModAB APS, CHA, both", (sum(cost(v) for v in ms["APS"].values()), sum(cost(v) for v in ms["CHA"].values()), sum(cost(v) for s_ in ("APS", "CHA") for v in ms[s_].values())), (2726, 1681, 4407)) for setname, want in (("APS", (118, 10, 26)), ("CHA", (34, 4, 7))): am = {p: v for p, v in a[setname]["per"].items()} mm = ms[setname] w = sum(cost(am[p]) < cost(mm[p]) for p in am) l = sum(cost(am[p]) > cost(mm[p]) for p in am) check(f"Table 8 A' vs ModAB {setname}", (w, len(am) - w - l, l), want) md = dict(mb["seen"]["modab"]) md.update(mb["fourth"]["modab"]) worst_p = 0.0 for s, (tm, vm, wtl) in {"APS": (2726, 0, (118, 10, 26)), "CHA": (1681, 0, (34, 4, 7)), "NEW": (3809, 1, (125, 26, 55)), "RJL": (1446, 0, (47, 11, 20)), "APP": (1460, 0, (76, 35, 21)), "D": (5535, 8, (183, 23, 34))}.items(): m = md[s] check(f"Table 10 ModAB {s}", (sum(cost(v) for v in m.values()), sum(v[2] for v in m.values())), (tm, vm)) ps = list(seen["Aprime"][s]) w = sum(cost(seen["Aprime"][s][p]) < cost(m[p]) for p in ps) l = sum(cost(seen["Aprime"][s][p]) > cost(m[p]) for p in ps) check(f"Table 10 A' vs ModAB {s}", (w, len(ps) - w - l, l), wtl) worst_p = max(worst_p, sign_p(w, l)) check("A' vs ModAB: every corpus p <= 0.0013", round(worst_p, 4) <= 0.0013, True) check("ModAB over all 855", sum(sum(cost(v) for v in md[s].values()) for s in names), 16657) check("ModAB violations over 855", sum(v[2] for s in names for v in md[s].values()), 9) check("corpus 4 A' vs ModAB p (1 digit)", f"{sign_p(183, 34):.1g}", "7e-26") rjm = mb["rjl"] ps91 = list(rjm["Aprime"]) w = sum(cost(rjm["Aprime"][p]) < cost(rjm["modab"][p]) for p in ps91) l = sum(cost(rjm["Aprime"][p]) > cost(rjm["modab"][p]) for p in ps91) check("ModAB's own benchmark, all 91", (len(ps91), sum(cost(rjm["Aprime"][p]) for p in ps91), sum(cost(rjm["modab"][p]) for p in ps91), (w, len(ps91) - w - l, l)), (91, 1507, 1588, (56, 13, 22))) check("ModAB's own benchmark p (2 digits)", f"{sign_p(w, l):.2g}", "0.00015") print("\nTable 9a and section 7: the similarity in the step (fresh audit run):") aps_ = {r["spec"][1]: r for r in _load("sweep_aprime_s.json")} abs_ = _load("sweep_ablation_s.json") t9a = {"ratio": (3333, 13032), "exp_abs": (3965, 14142), "rational": (4424, 14843), "ratio_soft": (7249, 28500), "sqrt": (11613, 45552), "cbrt": (16094, 63031), "exp_sq": (17429, 67701), "cosine": (17487, 67854), "rat_sq": (17452, 67394)} for s_, (two, six) in t9a.items(): r = aps_[s_] check(f"Table 9a {s_} APS+CHA", r["APS"]["total"] + r["CHA"]["total"], two) check(f"Table 9a {s_} six corpora", sum(tot(abs_[f"S:{s_}"], s) for s in sets), six) check(f"Table 9a {s_} guarantee", r["APS"]["over"] + r["CHA"]["over"] + sum(v[2] for s in sets for v in abs_[f"S:{s_}"][s].values()), 0) check("nearest other S costs more, two sets %", round(100 * (3965 / 3333 - 1)), 19) check("worst S costs more, two sets %", round(100 * (17487 / 3333 - 1)), 425) check("nearest other S costs more, six corpora %", round(100 * (14142 / 13032 - 1)), 9) check("quadratic S: five times", (int(17429 / 3333), int(67854 / 13032)), (5, 5)) import audit_similarity_places buf = io.StringIO() with contextlib.redirect_stdout(buf): audit_similarity_places.main() rows_a = {ln.split(" ")[0].strip(): re.findall(r"([+-]\d+) \|", ln) for ln in buf.getvalue().splitlines() if " | " in ln} check("audit: shape gain with the tent (APS)", int(rows_a["tent 1-|z|"][1]), -493) check("audit: shape gain with 1/sqrt(1+z^2) (APS)", int(rows_a["1/sqrt(1+z^2)"][1]), 250) check("audit: defect gain with the tent (APS)", int(rows_a["tent 1-|z|"][3]), -283) check("audit: defect gain with 1-|z|^1/3 (APS)", int(rows_a["1-|z|^1/3"][3]), 215) # every sign-test p the paper prints in Section 9, at the precision printed for (w, l), printed in (((103, 31), "3e-10"), ((31, 10), "0.001"), ((108, 65), "0.001"), ((44, 20), "0.004"), ((48, 32), "0.09"), ((177, 28), "1e-27"), ((107, 22), "1e-14"), ((43, 15), "0.0003"), ((105, 67), "0.005"), ((38, 30), "0.4")): check(f"sign test p for {w}:{l}", f"{sign_p(w, l):.1g}", printed) def verify_revision1(): """Revision 1 (2026-09-22): the claims added or restated by the revision - the order of the base step, section 7.3's solved counts, the families as the unit, ITP in Table 10, negative items 5 and 9, and the one canonical problem the multiplier moves. Recomputed.""" import contextlib import io import math import re from collections import defaultdict print("\nRevision 1:") import order_high_precision buf = io.StringIO() with contextlib.redirect_stdout(buf): order_high_precision.main() out = buf.getvalue() check("Muller order", re.search(r"Muller order .*: ([\d.]+)", out).group(1), "1.8393") for name in ("cos x - x", "exp(x) - 2", "x^3 - x - 2", "sin x - 1/2"): line = next(ln for ln in out.splitlines() if ln.startswith(name)) three = line[14:54].split() check(f"three-point order settles, {name}", three[-2], "1.840") line = next(ln for ln in out.splitlines() if ln.startswith("x^2 - 2")) check("three-point order on x^2 - 2", line[14:54].split()[-1], "2.000") check("two-point order, cos x - x", next(ln for ln in out.splitlines() if ln.startswith("cos x - x"))[54:].split()[-1], "1.618") rows = _load("sweep_full.json") by = {r["label"]: r for r in rows} solved = {k: by[k]["APS"]["solved"] + by[k]["CHA"]["solved"] for k in ("chandrupatla", "toms748", "itp", "bisection", "brent", "dekker", "illinois")} check("section 7.3 solved counts", solved, {"chandrupatla": 199, "toms748": 199, "itp": 199, "bisection": 199, "brent": 197, "dekker": 178, "illinois": 177}) check("Chandrupatla fewer than the base rule, APS %", round(100 * (by[PROJ]["APS"]["total"] / by["chandrupatla"]["APS"]["total"] - 1), 1), 1.5) none_ = by["pure.ratio.derived3.none"]["CHA"] check("guarantee on CHA: more solved, fewer evaluations", (45 - none_["solved"], round(100 * (1 - by[PROJ]["CHA"]["total"] / none_["total"]))), (6, 30)) check("earned costs 28% more on APS", round(100 * (by[CREDIT]["APS"]["total"] / by[PROJ]["APS"]["total"] - 1)), 28) def cost(v): return v[0] if v[1] else 200 def fam_of(pid): return ".".join(pid.split(".")[:2]) if pid.startswith("aps") else pid.split(".")[0] def families(a, b, s): ta, tb = defaultdict(int), defaultdict(int) for p in a[s]["per"]: ta[fam_of(p)] += cost(a[s]["per"][p]) tb[fam_of(p)] += cost(b[s]["per"][p]) w = sum(ta[k] < tb[k] for k in ta) l = sum(ta[k] > tb[k] for k in ta) return w, len(ta) - w - l, l def sign_p(w, l): k, n = min(w, l), w + l return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n) ap = {r["label"]: r for r in _load("sweep_aprime.json")}[ "pure+extP+contra+flatanalogmempt+exp+half+close+sub.ratio.derived3.project"] ch = by["chandrupatla"] check("base rule vs Chandrupatla by family, APS", families(by[PROJ], ch, "APS"), (6, 1, 8)) check("A' vs Chandrupatla by family, APS", families(ap, ch, "APS"), (8, 1, 6)) check("A' vs Chandrupatla by family, CHA", families(ap, ch, "CHA"), (7, 0, 2)) # revision: every head-to-head row of the paper must sum to the size of its set rivals = ("chandrupatla", "toms748", "brent", "itp_oracle", "itp", "bisection") sums = {(who, s): sum(head_to_head(rows + [ap], who, opp, s)[:3]) for who in (PROJ, ap["label"]) for opp in rivals for s in ("APS", "CHA")} check("every head-to-head row sums to its set (154 / 45)", {v for (w, s), v in sums.items() if s == "APS"} | {v + 1000 for (w, s), v in sums.items() if s == "CHA"}, {154, 1045}) check("family-level p, APS and CHA (2 digits)", (float(f"{sign_p(8, 6):.2g}"), float(f"{sign_p(7, 2):.2g}")), (0.79, 0.18)) order = _load("sweep_order.json") cells = {} for r in order: s = r["spec"] if s[2] == "far": cells[(s[1], s[3], s[8] if len(s) > 8 else "none")] = r["APS"]["total"] + r["CHA"]["total"] pairs = [(k[0], k[1]) for k in cells if k[2] == "none" and (k[0], k[1], "order") in cells] check("far-end reading: measured order cheaper in cells", (sum(cells[(s_, g, "order")] < cells[(s_, g, "none")] for s_, g in pairs), len(pairs)), (33, 63)) cap, dire = _load("sweep_cap.json"), _load("sweep_direction.json") for s, want in (("APS", (3101, 2539)), ("CHA", (1766, 1752))): bc = min(r[s]["total"] for r in cap if r[s]["solved"] == r[s]["n"]) bd = min(r[s]["total"] for r in dire if r[s]["solved"] == r[s]["n"] and r["spec"][0] == "pure") check(f"item 9: best capped vs best uncapped, {s}", (bc, bd), want) itp = _load("sweep_itp.json") t10 = {"APS": (3767, 3068), "CHA": (1946, 1675), "NEW": (5674, 4707), "RJL": (2161, 1629), "APP": (3245, 2386), "D": (5721, 4832)} for s, want in t10.items(): check(f"Table 10 ITP / oracle {s}", (sum(cost(v) for v in itp["itp"][s].values()), sum(cost(v) for v in itp["itp_oracle"][s].values())), want) check("Table 10 ITP / oracle over 855", tuple(sum(cost(v) for s in t10 for v in itp[k][s].values()) for k in ("itp", "itp_oracle")), (22514, 18297)) check("ITP bound violations over 855", sum(v[2] for k in ("itp", "itp_oracle") for s in t10 for v in itp[k][s].values()), 0) seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json") apx = dict(seen["Aprime"]) apx.update(four["Aprime"]) check("A' ahead of ITP and the oracle on every corpus", all(sum(cost(v) for v in apx[s].values()) < sum(cost(v) for v in itp[k][s].values()) for s in t10 for k in ("itp", "itp_oracle")), True) from simroot.pure import pure_root from stage12_validation import load, per_problem probs = load() aps = [p for p in probs if p[0] == "APS"] runs = {fac: per_problem(lambda f, a, b, xt, mx, _f=fac: pure_root( f, a, b, xt, mx, similarity="ratio", scale="derived3", guard="project", budget_factor=_f), aps, 200, criterion="sign") for fac in (1.25, 1.5, 2.0)} moved = {p: tuple(runs[f][p][0] for f in (1.25, 1.5, 2.0)) for p in runs[2.0] if len({runs[f][p][0] for f in (1.25, 1.5, 2.0)}) > 1} check("Table 1a: the one canonical problem the multiplier moves", moved, {"aps.13.00": (20, 30, 28)}) tot = {} for sc in ("derived3", "derived3_m"): pp = per_problem(lambda f, a, b, xt, mx, _s=sc: pure_root( f, a, b, xt, mx, similarity="ratio", scale=_s, guard="project"), probs, 200, criterion="sign") tot[sc] = sum(v[0] for v in pp.values()) check("item 5: phase detector vs base rule, both sets", (tot["derived3_m"], tot["derived3"]), (5336, 4991)) def verify_controls(): """Section 9.1: the controls (PREREG_controls_2026-09-22, sweep_controls.json).""" import math print("\nSection 9.1 (controls, sweep_controls.json):") R = _load("sweep_controls.json") seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json") ap = dict(seen["Aprime"]) ap.update(four["Aprime"]) chf = dict(seen["chandrupatla"]) chf.update(four["chandrupatla"]) sets = ("APS", "CHA", "NEW", "RJL", "APP", "D") def cost(v): return v[0] if v[1] else 200 def tot(r, keep=None): return sum(cost(v) for p, v in r.items() if keep is None or keep(p)) def h2h(a, b, keep=None): ps = [p for p in a if keep is None or keep(p)] w = sum(cost(a[p]) < cost(b[p]) for p in ps) l = sum(cost(a[p]) > cost(b[p]) for p in ps) return w, len(ps) - w - l, l def sign_p(w, l): k, n = min(w, l), w + l return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n) rows = {"K2L": (2235, 1437, 4026, 1495, 1490, 4488, 15171), "K2P": (2682, 2526, 5485, 2117, 1622, 6236, 20668), "K2F": (2693, 2526, 5569, 2126, 1670, 6165, 20749)} check("Table 9b A' row", tuple(tot(ap[s]) for s in sets) + (sum(tot(ap[s]) for s in sets),), (2259, 1074, 3290, 1371, 1382, 3656, 13032)) for k, want in rows.items(): k2 = R["K2"][k] check(f"Table 9b {k} row", tuple(tot(k2[s]) for s in sets) + (sum(tot(k2[s]) for s in sets),), want) for k, wtl, pp in (("K2L", (84, 121, 35), ("1g", "8e-06")), ("K2P", (110, 96, 34), ("2g", "1.5e-10")), ("K2F", (113, 87, 40), ("1g", "3e-09"))): got = h2h(ap["D"], R["K2"][k]["D"]) check(f"Table 9b A' vs {k} on corpus 4", got, wtl) check(f"Table 9b p vs {k}", format(sign_p(got[0], got[2]), "." + pp[0]), pp[1]) check("Table 9b violations", sum(v[2] for k in rows for s in sets for v in R["K2"][k][s].values()), 0) for cls, want in (("D2", (1163, 1739)), ("D3", (924, 1207)), ("D5", (503, 477))): keep = lambda p, c=cls: p.startswith(c) # noqa: E731 check(f"9.1 {cls}: A' vs Lagouanelle", (tot(ap["D"], keep), tot(R["K2"]["K2L"]["D"], keep)), want) level = [(tot(ap["D"], lambda p, c=c: p.startswith(c)), tot(R["K2"]["K2L"]["D"], lambda p, c=c: p.startswith(c))) for c in ("D1", "D4", "D6")] check("9.1 level on the rest (within 1.5%)", all(abs(a - b) <= 0.015 * b for a, b in level), True) ab = _load("sweep_ablation_all.json")["abl-regime"] no_regime_all = sum(tot(ab[s]) for s in sets) check("band and no-detector worse than no regime (six corpora)", all(sum(tot(R["K2"][k][s]) for s in sets) > no_regime_all for k in ("K2P", "K2F")), True) check("band and no-detector worse than no regime (corpus 4)", all(tot(R["K2"][k]["D"]) > tot(ab["D"]) for k in ("K2P", "K2F")), True) t9c = {"CHA": (20, 654, 96, 66, 1068), "NEW": (40, 1229, 358, 337, 1792), "D": (88, 2159, 901, 844, 4151)} nonsimple = lambda m: m >= 2.0 or m <= 2.0 / 3.0 # noqa: E731 wins = 0 for s, want in t9c.items(): k1 = R["K1"][s] keep = lambda p, _m=k1["mstar"]: nonsimple(_m[p]) # noqa: E731 got = (sum(1 for p in k1["mstar"] if keep(p)), tot(ap[s], keep), tot(k1["chandrupatla_g"], keep), tot(k1["brent_g"], keep), tot(chf[s], keep)) check(f"Table 9c {s}", got, want) wins += h2h({p: v for p, v in ap[s].items() if keep(p)}, k1["chandrupatla_g"])[0] check("Table 9c: A' wins none of the 148", wins, 0) check("Table 9c: 2.4 to 6.8 times", (round(2159 / 901, 1), round(654 / 96, 1), min(round(a / c, 1) for _n, a, c, _b, _f in t9c.values()), max(round(a / c, 1) for _n, a, c, _b, _f in t9c.values())), (2.4, 6.8, 2.4, 6.8)) APS_ = "APS" for s, want in ((APS_, (2259, 2640)), ("CHA", (420, 437)), ("NEW", (2061, 1924)), ("D", (1497, 1427))): k1 = R["K1"][s] keep = lambda p, _m=k1["mstar"]: not nonsimple(_m[p]) # noqa: E731 check(f"9.1 rest {s}: A' vs Chandrupatla on g", (tot(ap[s], keep), tot(k1["chandrupatla_g"], keep)), want) def verify_revision2(): """Revision 2 (points 3, 4, 5, 7).""" import math print("\nRevision 2:") def cost(v): return v[0] if v[1] else 200 def tot(r, keep=None): return sum(cost(v) for p, v in r.items() if keep is None or keep(p)) def sign_p(w, l): k, n = min(w, l), w + l return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n) sets = ("APS", "CHA", "NEW", "RJL", "APP", "D") K = _load("sweep_estimated.json") seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json") ap, ch = dict(seen["Aprime"]), dict(seen["chandrupatla"]) ap.update(four["Aprime"]) ch.update(four["chandrupatla"]) K1 = _load("sweep_controls.json")["K1"] nonsimple = lambda m: m >= 2.0 or m <= 2.0 / 3.0 # noqa: E731 def group(r): return sum(tot(r[s], lambda p, _m=K1[s]["mstar"]: nonsimple(_m[p])) for s in ("CHA", "NEW", "D")) rows = {"A'": ap, "K3A": K["k3_analogy"], "K3L": K["k3_lag"], "Ch(f)": ch} want = {"A'": (3656, 13032, 4042, 2259), "K3A": (3806, 15733, 2809, 3453), "K3L": (7526, 30991, 6041, 7215), "Ch(f)": (5717, 16481, 7011, 2595)} for k, r in rows.items(): check(f"Table 12 {k}", (tot(r["D"]), sum(tot(r[s]) for s in sets), group(r), tot(r["APS"])), want[k]) oracle = sum(tot(K1[s]["chandrupatla_g"], lambda p, _m=K1[s]["mstar"]: nonsimple(_m[p])) for s in ("CHA", "NEW", "D")) check("Table 12 oracle row (Table 11 sum)", oracle, 1355) ps = list(ap["D"]) w = sum(cost(ap["D"][p]) < cost(K["k3_analogy"]["D"][p]) for p in ps) l = sum(cost(ap["D"][p]) > cost(K["k3_analogy"]["D"][p]) for p in ps) check("A' vs Chandrupatla-on-g with the analogy order, corpus 4", (w, len(ps) - w - l, l), (126, 38, 76)) check("its p (1 digit)", f"{sign_p(w, l):.1g}", "0.0005") shares = [] for s in ("CHA", "NEW", "D"): keep = lambda p, _m=K1[s]["mstar"]: nonsimple(_m[p]) # noqa: E731 a_, k_, o_ = tot(ap[s], keep), tot(K["k3_analogy"][s], keep), tot(K1[s]["chandrupatla_g"], keep) shares.append((k_, a_, round(100 * (a_ - k_) / (a_ - o_)))) check("share of the A'-to-oracle gap closed by the analogy order", shares, [(322, 654, 59), (721, 1229, 58), (1766, 2159, 31)]) check("K3 bound violations", sum(v[2] for k in K for s in sets for v in K[k][s].values()), 0) # Lagouanelle's fractional readings on the simple roots of the canonical set (fresh) from simroot.transformed import chandrupatla_transformed from sweep_full import load_problems frac = 0 for s, pid, f, a, b, _r in load_problems(): if s != "APS": continue st = {} chandrupatla_transformed(f, a, b, 1e-12, 200, estimator="lag", stats=st) m = st.get("m_hat", 1.0) frac += (0.0 < m <= 2.0 / 3.0) check("Lagouanelle holds a fractional order on simple APS roots", frac, 115) # point 4: exact-zero stops (fresh) import contextlib import io import diag_revision2 with contextlib.redirect_stdout(io.StringIO()): z = diag_revision2.point4() check("exact-zero stops on CHA: Brent(g), Chandrupatla(g), A'", (z["CHA"]["brent_g"][1], z["CHA"]["chandrupatla_g"][1], z["CHA"]["Aprime"][1]), (15, 4, 2)) check("exact-zero stops of the transformed runs on corpus 1", (z["NEW"]["chandrupatla_g"][1], z["NEW"]["brent_g"][1]), (0, 0)) check("corpus 1: transformation 3.4 times cheaper", round(1229 / 358, 1), 3.4) # point 3: the three unchanged rows of Table 2 (fresh) from corpus_roots_jl import solved from sweep_full import make_method f, a, b = next((f, a, b) for s, pid, f, a, b, _r in load_problems() if pid == "aps.13.00") got = [] for name in ("dekker", "illinois", "regula_falsi"): x, n, conv = make_method(("base", name))(f, a, b, 1e-12, 200) got.append((name, int(n), bool(conv), bool(solved(f, float(x))))) check("aps.13.00: Dekker, Illinois, false position spend the budget unsolved", got, [("dekker", 200, False, False), ("illinois", 200, False, False), ("regula_falsi", 200, False, False)]) small = [abs(float(f(float(make_method(("base", nm))(f, a, b, 1e-12, 200)[0])))) < 1e-62 for nm in ("dekker", "illinois")] check("Dekker and Illinois stall where |f| < 1e-62", small, [True, True]) # point 7: Table 3's two added rows full = {r["label"]: r for r in _load("sweep_full.json")} for lb, want_ in (("pure.exp_abs.bracket.bracket", (1848, 45, 0)), ("pure.exp_abs.bracket.credit", (1952, 45, 0))): c = full[lb]["CHA"] check(f"Table 3 row {lb}", (c["total"], c["solved"], c["over"]), want_) def verify_revision3(): """Revision 3 (point 5): the post hoc branch m >= 2 and the reading of Table 12.""" print("\nRevision 3:") def cost(v): return v[0] if v[1] else 200 def tot(r, keep=None): return sum(cost(v) for p, v in r.items() if keep is None or keep(p)) sets = ("APS", "CHA", "NEW", "RJL", "APP", "D") P = _load("sweep_estimated_posthoc.json") K = _load("sweep_estimated.json")["k3_analogy"] seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json") ap = dict(seen["Aprime"]) ap.update(four["Aprime"]) K1 = _load("sweep_controls.json")["K1"] nonsimple = lambda m: m >= 2.0 or m <= 2.0 / 3.0 # noqa: E731 def group(r): return sum(tot(r[s], lambda p, _m=K1[s]["mstar"]: nonsimple(_m[p])) for s in ("CHA", "NEW", "D")) check("Table 12 post hoc row", (tot(P["D"]), sum(tot(P[s]) for s in sets), group(P), tot(P["APS"])), (5568, 18372, 6540, 2960)) ps = list(ap["D"]) w = sum(cost(ap["D"][p]) < cost(P["D"][p]) for p in ps) l = sum(cost(ap["D"][p]) > cost(P["D"][p]) for p in ps) check("A' vs the post hoc variant on corpus 4", (w, len(ps) - w - l, l), (178, 34, 28)) check("post hoc violations", sum(v[2] for s in sets for v in P[s].values()), 0) gain = group(ap) - group(K) loss = (sum(tot(K[s]) for s in sets) - group(K)) - (sum(tot(ap[s]) for s in sets) - group(ap)) check("transferred detector: gain on multiple roots < loss elsewhere (vs A')", (gain, loss, gain < loss), (1233, 3934, True)) if __name__ == "__main__": main()