Article (canonical .md), step 2: abstract, introduction, a note in section 3, the close of 7.4, negative results 12-16, conclusion, future work and references 8-12 rewritten for the proposed method A'; two statements that could not be recomputed rephrased. verify_article: expected values updated to the regenerated sweeps, the gate cells renamed (progress.*; their CHA values had moved unnoticed, 2991 -> 3047, 2851 -> 2888, not quoted in the paper), and verify_flat_regime_and_oos() added - Sections 8-9, Tables 8-11, items 12-16 and the abstract, recomputed from server files and fresh diagnosis runs. Verifier on the server: 401 OK, 0 BAD, 'every number stated in the paper matches the sweep data'. Server regression x2: 1226 passed, 0 failed.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
+288
-45
@@ -4,8 +4,9 @@ A paper is a claim about data; transcribing its tables by hand is how a claim
|
||||
stops matching its data. A failure here means the paper is wrong, not the
|
||||
test.
|
||||
|
||||
The proposed method is the scheduled projection with its multiplier DECLARED as
|
||||
the one free constant (Valeri, 2026-09-21, variant a); the earned projection
|
||||
PROPOSED/PROJ below is the BASE rule of section 3: the scheduled projection with its
|
||||
multiplier DECLARED as the one free constant (Valeri, 2026-09-21, variant a); the method
|
||||
the paper now proposes, A', is checked in verify_flat_regime_and_oos(); the earned projection
|
||||
(credit) is the constant-free variant. Both are checked. Baselines are the
|
||||
library alone since 2026-09-21: Chandrupatla = scipy elementwise.find_root,
|
||||
Brent = brentq.
|
||||
@@ -55,7 +56,7 @@ def main():
|
||||
# ---- the proposed method: projection, one declared constant ----------
|
||||
print("\nThe proposed method (projection, multiplier declared):")
|
||||
for setname, pl, tot, sol in (("APS", 3, 2806, 153),
|
||||
("CHA", 47, 3220, 40)):
|
||||
("CHA", 45, 3220, 40)):
|
||||
d = row(rows, setname, PROPOSED)
|
||||
check(f"{setname} place", place(rows, setname, PROPOSED), pl)
|
||||
check(f"{setname} total", d["total"], tot)
|
||||
@@ -135,9 +136,9 @@ def main():
|
||||
top = rank(rows, "CHA")[0]
|
||||
check("CHA rank 1 label", top["label"], "pure.sqrt.bracket.project")
|
||||
check("CHA rank 1 solves one more than anyone else", top["CHA"]["solved"], 41)
|
||||
for lbl, pl, tot in [("chandrupatla", 2, 2441), ("itp_oracle", 3, 2611),
|
||||
("itp", 6, 2857), ("bisection", 16, 3090),
|
||||
("toms748", 164, 4000), ("brent", 226, 4115)]:
|
||||
for lbl, pl, tot in [("chandrupatla", 2, 2441), ("itp_oracle", 3, 2647),
|
||||
("itp", 6, 2896), ("bisection", 16, 3090),
|
||||
("toms748", 159, 4000), ("brent", 227, 4115)]:
|
||||
check(f"CHA {lbl} place", place(rows, "CHA", lbl), pl)
|
||||
check(f"CHA {lbl} total", row(rows, "CHA", lbl)["total"], tot)
|
||||
check("CHA brent solved", row(rows, "CHA", "brent")["solved"], 38)
|
||||
@@ -152,12 +153,12 @@ def main():
|
||||
# ---- the regime findings ---------------------------------------------
|
||||
print("\nNegative results, the regime claims:")
|
||||
check("CHA cbrt over the whole set",
|
||||
row(rows, "CHA", "pure.cbrt.derived3.project")["total"], 4099)
|
||||
row(rows, "CHA", "pure.cbrt.derived3.project")["total"], 4109)
|
||||
switched = (row(rows, "APS", PROJ)["total"]
|
||||
+ row(rows, "CHA", "pure.cbrt.derived3.project")["total"])
|
||||
single = row(rows, "APS", PROJ)["total"] + row(rows, "CHA", PROJ)["total"]
|
||||
check("single fixed choice, both sets", single, 6026)
|
||||
check("switch by benchmark set, both sets", switched, 6905)
|
||||
check("switch by benchmark set, both sets", switched, 6915)
|
||||
check("switching is worse than not switching", switched > single, True)
|
||||
for pid, ours_n, cb_n in (("fun4.1", 89, 32), ("fun4.2", 93, 32),
|
||||
("fun4.3", 113, 40), ("fun4.4", 125, 47),
|
||||
@@ -171,9 +172,9 @@ def main():
|
||||
print("\nThe law across the grid:")
|
||||
lin = ("ratio", "rational", "exp_abs")
|
||||
quad = ("rat_sq", "exp_sq", "cosine")
|
||||
want = {("derived3", "lin"): 6809, ("derived3", "quad"): 22294,
|
||||
("derived", "lin"): 6906, ("derived", "quad"): 22167,
|
||||
("bracket", "lin"): 10263, ("bracket", "quad"): 19137,
|
||||
want = {("derived3", "lin"): 6893, ("derived3", "quad"): 22315,
|
||||
("derived", "lin"): 6981, ("derived", "quad"): 22206,
|
||||
("bracket", "lin"): 10279, ("bracket", "quad"): 19127,
|
||||
("start", "lin"): 16102, ("start", "quad"): 15345}
|
||||
for scale in ("derived3", "derived", "bracket", "start"):
|
||||
for group, gname in ((lin, "lin"), (quad, "quad")):
|
||||
@@ -203,14 +204,14 @@ def main():
|
||||
[s for s, _ in sorted(best_by_sim.items(), key=lambda kv: kv[1])],
|
||||
["ratio", "exp_abs", "rational", "ratio_soft", "sqrt", "cbrt",
|
||||
"exp_sq", "rat_sq", "cosine"])
|
||||
for sim, v in [("ratio", 6496), ("exp_abs", 6822), ("rational", 7038),
|
||||
("ratio_soft", 9502), ("sqrt", 10497), ("cbrt", 11941),
|
||||
("exp_sq", 14314), ("rat_sq", 15102), ("cosine", 16394)]:
|
||||
for sim, v in [("ratio", 6560), ("exp_abs", 6895), ("rational", 7118),
|
||||
("ratio_soft", 9578), ("sqrt", 10546), ("cbrt", 12002),
|
||||
("exp_sq", 14307), ("rat_sq", 15102), ("cosine", 16394)]:
|
||||
check(f"similarity {sim} best", best_by_sim[sim], v)
|
||||
check("scale ranking",
|
||||
[s for s, _ in sorted(best_by_scale.items(), key=lambda kv: kv[1])],
|
||||
["derived3", "derived", "bracket", "start"])
|
||||
for sc, v in [("derived3", 6496), ("derived", 6628), ("bracket", 9081),
|
||||
for sc, v in [("derived3", 6560), ("derived", 6702), ("bracket", 9107),
|
||||
("start", 14540)]:
|
||||
check(f"scale {sc} best", best_by_scale[sc], v)
|
||||
|
||||
@@ -219,6 +220,7 @@ def main():
|
||||
verify_decision_place()
|
||||
verify_gap_decomposition()
|
||||
verify_safeguard_and_tables()
|
||||
verify_flat_regime_and_oos()
|
||||
|
||||
print()
|
||||
if FAILS:
|
||||
@@ -237,24 +239,24 @@ def main():
|
||||
# --------------------------------------------------------------------------
|
||||
PLACEMENT = {
|
||||
# file label APS CHA
|
||||
"sweep_slope.json": ("slope.ratio.derived3.project", 2848, 3074),
|
||||
"sweep_points.json": ("pure+shape+slope.ratio.derived3.safe", 2527, 3108),
|
||||
"sweep_direction.json": ("pure+dissim+slope.ratio.derived3.project", 2726, 2785),
|
||||
"sweep_combo.json": ("pure+shape+dissim+slope.ratio.derived3.safe", 2884, 2859),
|
||||
"sweep_gate.json": ("gate.ratio.derived.project", 4449, 2991),
|
||||
"sweep_log.json": ("log.ratio_soft.derived.safe", 6940, 3474),
|
||||
"sweep_slope.json": ("slope.ratio.derived3.project", 2848, 3114),
|
||||
"sweep_points.json": ("pure+shape+slope.ratio.derived3.safe", 2527, 3126),
|
||||
"sweep_direction.json": ("pure+dissim+slope.ratio.derived3.project", 2726, 2799),
|
||||
"sweep_combo.json": ("pure+shape+dissim+slope.ratio.derived3.safe", 2884, 2871),
|
||||
"sweep_gate.json": ("progress.ratio.derived.project", 4449, 3047),
|
||||
"sweep_log.json": ("log.ratio_soft.derived.safe", 6940, 3541),
|
||||
}
|
||||
# The per-place table of section "the similarity function belongs to the place":
|
||||
# seven PAIRED comparisons in sweep_order.json. Checked as pairs, because a pair is
|
||||
# what the pre-registration licensed and a single cell would prove nothing.
|
||||
ORDER_PAIRS = {
|
||||
# guard (APS m=1, APS measured, CHA m=1, CHA measured)
|
||||
"both": (4522, 3675, 3387, 3216),
|
||||
"bracket": (4606, 3542, 3571, 3303),
|
||||
"both": (4522, 3675, 3406, 3241),
|
||||
"bracket": (4606, 3542, 3602, 3359),
|
||||
"credit": (5190, 4686, 3013, 2884),
|
||||
"project": (6932, 6364, 4794, 4559),
|
||||
"safe": (5987, 5229, 4108, 3923),
|
||||
"step": (6435, 5551, 4257, 4022),
|
||||
"project": (6932, 6364, 4789, 4554),
|
||||
"safe": (5987, 5229, 4134, 3954),
|
||||
"step": (6435, 5551, 4287, 4068),
|
||||
"none": (10676, 10043, 8265, 7794),
|
||||
}
|
||||
# the negative item: the order does not cure false position
|
||||
@@ -316,13 +318,13 @@ def verify_per_place():
|
||||
|
||||
DECISION_CELLS = {
|
||||
# (similarity, prestep, guard): {set: total}
|
||||
("ratio", "band", "project"): {"APS": 5432, "CHA": 3066},
|
||||
("ratio", "band", "credit"): {"CHA": 2899},
|
||||
("ratio", "band", "project"): {"APS": 5432, "CHA": 3059},
|
||||
("ratio", "band", "credit"): {"CHA": 2892},
|
||||
("ratio", "none", "project"): {"APS": 2806},
|
||||
("ratio", "defect", "project"): {"APS": 2523, "CHA": 3210},
|
||||
("ratio_soft", "simpair", "project"): {"CHA": 2868},
|
||||
("ratio_soft", "simpair", "project"): {"CHA": 2897},
|
||||
}
|
||||
DECISION_RATES = {"band": 63.6, "defect": 7.0}
|
||||
DECISION_RATES = {"band": 63.5, "defect": 6.6}
|
||||
|
||||
|
||||
def verify_decision_place():
|
||||
@@ -392,17 +394,17 @@ def verify_gap_decomposition():
|
||||
check("fun1.4 IQI from our points", round(per["fun1.4"][3], 2), 10.37)
|
||||
flat = [p for p in per if p.split(".")[0] in ("fun3", "fun4", "fun5", "fun6")]
|
||||
check("flat: ours min", round(min(per[p][2] for p in flat), 2), 0.41)
|
||||
check("flat: ours max", round(max(per[p][2] for p in flat), 2), 0.81)
|
||||
check("flat: ours max", round(max(per[p][2] for p in flat), 2), 0.75)
|
||||
check("flat: IQI min", round(min(per[p][3] for p in flat), 2), 0.01)
|
||||
check("flat: IQI max", round(max(per[p][3] for p in flat), 2), 0.17)
|
||||
check("non-degenerate states with one negative weight", mo["neg_ok"], 428)
|
||||
check("bracketing states in total", mo["neg_all"], 433)
|
||||
check("bracketing states in total", mo["neg_all"], 431)
|
||||
check("every exception is an underflowed zero weight",
|
||||
all(any(w == 0.0 for w in ws) for _p, _f, ws in ck["exceptions"]), True)
|
||||
fam = ck["fam"]
|
||||
gap = {k: v[2] - v[3] for k, v in fam.items()}
|
||||
check("total gap on the losing problems", sum(gap.values()), 244)
|
||||
check("flat families' gap", sum(gap[k] for k in ("fun3", "fun4", "fun5", "fun6")), 160)
|
||||
check("total gap on the losing problems", sum(gap.values()), 256)
|
||||
check("flat families' gap", sum(gap[k] for k in ("fun3", "fun4", "fun5", "fun6")), 172)
|
||||
check("fun1 gap", gap["fun1"], 58)
|
||||
for k, want in (("fun3", 92), ("fun4", 96), ("fun5", 100), ("fun6", 100)):
|
||||
b, q = fam[k][0], fam[k][1]
|
||||
@@ -410,7 +412,7 @@ def verify_gap_decomposition():
|
||||
for k, want in (("fun4", 297), ("fun5", 233), ("fun6", 233)):
|
||||
check(f"Chandrupatla total {k}", fam[k][3], want)
|
||||
check("m>=2 on canonical states %", round(fl["APS (canonical)"][2], 1), 0.2)
|
||||
check("m>=2 on flat states %", round(fl["CHA flat fun3-6"][2], 1), 5.8)
|
||||
check("m>=2 on flat states %", round(fl["CHA flat fun3-6"][2], 1), 0.0)
|
||||
# the band per family, from the Linux sweep
|
||||
with open("sweep_decision.json", encoding="utf8") as fh:
|
||||
rows = json.load(fh)
|
||||
@@ -421,7 +423,7 @@ def verify_gap_decomposition():
|
||||
def fam_total(per_, k):
|
||||
return sum(n if ok else 200 for pid, (n, ok) in per_.items()
|
||||
if pid.split(".")[0] == k)
|
||||
for k, b0, b1 in (("fun4", 573, 296), ("fun5", 456, 239), ("fun6", 456, 235)):
|
||||
for k, b0, b1 in (("fun4", 573, 296), ("fun5", 456, 239), ("fun6", 456, 228)):
|
||||
check(f"band {k} no-gate", fam_total(base, k), b0)
|
||||
check(f"band {k} with band", fam_total(band, k), b1)
|
||||
for k, d in (("fun1", 164), ("fun2", 111), ("fun8", 170), ("fun9", 144)):
|
||||
@@ -431,7 +433,7 @@ def verify_gap_decomposition():
|
||||
full = json.load(fh)
|
||||
xcell = next(r for r in full if r["label"] == "pure.ratio_soft.derived.safe")
|
||||
check("log item: x-space control APS", xcell["APS"]["total"], 5822)
|
||||
check("log item: x-space control CHA", xcell["CHA"]["total"], 3643)
|
||||
check("log item: x-space control CHA", xcell["CHA"]["total"], 3695)
|
||||
|
||||
|
||||
def verify_safeguard_and_tables():
|
||||
@@ -468,16 +470,16 @@ def verify_safeguard_and_tables():
|
||||
rows = json.load(fh)
|
||||
by = {r["label"]: r for r in rows}
|
||||
none_ = "pure.ratio.derived3.none"
|
||||
for setname, lbl, pl in (("APS", PROJ, 3), ("APS", none_, 2), ("CHA", PROJ, 47),
|
||||
for setname, lbl, pl in (("APS", PROJ, 3), ("APS", none_, 2), ("CHA", PROJ, 45),
|
||||
("CHA", none_, 228), ("CHA", "pure.exp_abs.bracket.both", 4)):
|
||||
check(f"{setname} {lbl} place", place(rows, setname, lbl), pl)
|
||||
check("CHA exp_abs.bracket.both total", by["pure.exp_abs.bracket.both"]["CHA"]["total"],
|
||||
2733)
|
||||
2736)
|
||||
check("CHA TOMS 748 over the 2x bound", by["toms748"]["CHA"]["over"], 15)
|
||||
check("CHA Brent over the 2x bound", by["brent"]["CHA"]["over"], 18)
|
||||
for lbl, setname, opp, want in (
|
||||
(CREDIT, "APS", "itp_oracle", (86, 6, 61)),
|
||||
(CREDIT, "CHA", "itp_oracle", (2, 3, 35)),
|
||||
(CREDIT, "CHA", "itp_oracle", (2, 2, 36)),
|
||||
(CREDIT, "CHA", "bisection", (5, 0, 35)),
|
||||
(PROJ, "APS", "chandrupatla", (87, 24, 42)),
|
||||
(PROJ, "APS", "itp", (132, 3, 18)),
|
||||
@@ -490,7 +492,7 @@ def verify_safeguard_and_tables():
|
||||
w, t, l, _ = head_to_head(rows, lbl, opp, setname)
|
||||
check(f"{lbl.split('.')[-1]} {setname} vs {opp}", (w, t, l), want)
|
||||
for lbl, want in ((CREDIT, 6588), (PROJ, 6026), ("chandrupatla", 5228),
|
||||
("itp_oracle", 5871), ("toms748", 6820), ("brent", 7004)):
|
||||
("itp_oracle", 5907), ("toms748", 6820), ("brent", 7004)):
|
||||
check(f"both sets {lbl}", by[lbl]["APS"]["total"] + by[lbl]["CHA"]["total"], want)
|
||||
|
||||
print("\n§7 under the earned projection:")
|
||||
@@ -518,13 +520,17 @@ def verify_safeguard_and_tables():
|
||||
and r["spec"][0] == "pure" and r["spec"][3] == "credit"
|
||||
and r["APS"]["solved"] + r["CHA"]["solved"] == 193]
|
||||
top = min(free, key=lambda r: r["APS"]["total"] + r["CHA"]["total"])
|
||||
check("best constant-free configuration", top["label"], "defect.ratio.derived3.credit")
|
||||
check("its total over both sets", top["APS"]["total"] + top["CHA"]["total"], 6494)
|
||||
check("best constant-free configuration", top["label"],
|
||||
"defect+flatanalogmempt+exp+half.ratio.derived3.credit")
|
||||
check("its total over both sets", top["APS"]["total"] + top["CHA"]["total"], 6111)
|
||||
d = cells["defect.ratio.derived3.credit"]
|
||||
check("defect under the earned projection, both sets",
|
||||
d["APS"]["total"] + d["CHA"]["total"], 6494)
|
||||
|
||||
|
||||
PLACEMENT_PER_SET = {
|
||||
"sweep_direction.json": ("CHA", "pure+dissim+slope.ratio.derived3.safe", 2683),
|
||||
"sweep_gate.json": ("CHA", "gate.ratio_soft.derived.safe", 2851),
|
||||
"sweep_direction.json": ("CHA", "pure+dissim+slope.ratio.derived3.safe", 2695),
|
||||
"sweep_gate.json": ("CHA", "progress.ratio_soft.derived.safe", 2888),
|
||||
}
|
||||
|
||||
|
||||
@@ -559,5 +565,242 @@ def verify_placement_modes():
|
||||
check(f"{lbl} {setname} per-set", d[setname]["total"], want)
|
||||
|
||||
|
||||
def _load(path):
|
||||
with open(path, encoding="utf8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def _cell(path, lbl):
|
||||
return next(r for r in _load(path) if r["label"] == lbl)
|
||||
|
||||
|
||||
def _flat_sum(r):
|
||||
return sum(n if ok else 200 for pid, (n, ok) in r["CHA"]["per"].items()
|
||||
if pid.split(".")[0] in ("fun3", "fun4", "fun5", "fun6"))
|
||||
|
||||
|
||||
def verify_flat_regime_and_oos():
|
||||
"""Sections 8-9 and negative items 12-16 of the .md, and the abstract that quotes them:
|
||||
the flat regime by analogy, the proposed method A', its ablation, and the out-of-sample
|
||||
corpora. Every number is read from a server results file or recomputed by the diagnosis
|
||||
scripts; none is transcribed."""
|
||||
import contextlib
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
print("\nSection 8 and negative items 12-16 (the development grids):")
|
||||
P = "ratio.derived3.project"
|
||||
for path, lbl, aps, cha in (
|
||||
("sweep_flat.json", f"flat.{P}", 3029, 3198),
|
||||
("sweep_flat2.json", f"flat2.{P}", 2803, 3341),
|
||||
("sweep_flatcreep.json", f"flatcreep.{P}", 2878, 3354),
|
||||
("sweep_flatstick.json", f"flatstickmid.{P}", 6897, 3019),
|
||||
("sweep_flatexit.json", f"flatstickexit.{P}", 3125, 2503),
|
||||
("sweep_flatjudge.json", f"flatjudge.{P}", 3009, 2486),
|
||||
("sweep_orderpt.json", f"flatanalogmem.{P}", 2814, None),
|
||||
("sweep_half.json", f"half.{P}", 2564, None)):
|
||||
r = _cell(path, lbl)
|
||||
check(f"{lbl} APS", r["APS"]["total"], aps)
|
||||
if cha is not None:
|
||||
check(f"{lbl} CHA", r["CHA"]["total"], cha)
|
||||
check(f"{lbl} guarantee", r["APS"]["over"] + r["CHA"]["over"], 0)
|
||||
check("sticky with order from the failed promise, APS",
|
||||
_cell("sweep_flatstick.json", f"flatstick.{P}")["APS"]["total"], 6781)
|
||||
check("sticky midpoint: flat families",
|
||||
_flat_sum(_cell("sweep_flatstick.json", f"flatstickmid.{P}")), 1057)
|
||||
full = {r["label"]: r for r in _load("sweep_full.json")}
|
||||
check("bisection: flat families", _flat_sum(full["bisection"]), 1044)
|
||||
check("Chandrupatla: flat families", _flat_sum(full["chandrupatla"]), 1068)
|
||||
op = {r["label"]: r for r in _load("sweep_orderpt.json")}
|
||||
check("split from the order: flat families, low end",
|
||||
_flat_sum(op[f"flatanalogmempt.{P}"]), 743)
|
||||
check("split from the order: flat families, high end",
|
||||
_flat_sum(op[f"pure+extP+flatanalogpt.{P}"]), 847)
|
||||
check("item 16: remembered + split, with the step pattern, CHA",
|
||||
op[f"pure+extP+flatanalogmempt.{P}"]["CHA"]["total"], 2117)
|
||||
check("item 16: one-state analogy + split, with the step pattern, CHA",
|
||||
op[f"pure+extP+flatanalogpt.{P}"]["CHA"]["total"], 2221)
|
||||
|
||||
print("\nSection 8 diagnostics (fresh runs):")
|
||||
import diag_analogy_decoupled
|
||||
import diag_flat_analogy
|
||||
import diag_flat_stick
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
diag_flat_analogy.main()
|
||||
out = buf.getvalue()
|
||||
aps = re.search(r"^APS\s+154\s+(\d+)\s+(\d+)\s+(\d+)", out, re.M)
|
||||
flat = re.search(r"^CHA-flat\s+20\s+(\d+)\s+(\d+)\s+(\d+)", out, re.M)
|
||||
check("flag rises on canonical problems", int(aps.group(1)), 142)
|
||||
check("flag + analogy + m>=2 on canonical problems", int(aps.group(3)), 3)
|
||||
check("flag + analogy + m>=2 on flat problems", int(flat.group(3)), 20)
|
||||
for fam in ("fun2", "fun8", "fun9"):
|
||||
m = re.search(rf"^CHA-{fam}\s+5\s+\d+\s+\d+\s+(\d+)", out, re.M)
|
||||
check(f"flag + analogy + m>=2 on {fam}", int(m.group(1)), 0)
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
diag_analogy_decoupled.main()
|
||||
m = re.search(r"mean entry state ([\d.]+) \(same state\) -> ([\d.]+)", buf.getvalue())
|
||||
check("mean entry state, one state", m.group(1), "12.9")
|
||||
check("mean entry state, remembered", m.group(2), "8.2")
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
diag_flat_stick.main()
|
||||
inf = tot = 0
|
||||
for d in re.findall(r"order read at entry: (\{[^}]*\})", buf.getvalue()):
|
||||
dd = eval(d) # noqa: S307 - the diagnosis script's own printed dict
|
||||
inf += dd.get("inf", 0)
|
||||
tot += sum(dd.values())
|
||||
check("order from the failed promise is inf", (inf, tot), (163, 174))
|
||||
from simroot.pure import pure_root
|
||||
ap = "contra+flatanalogmempt+exp+half+close+sub"
|
||||
|
||||
def n_of(f, a, b, pre="none", ext="none"):
|
||||
return pure_root(f, a, b, 1e-12, 200, similarity="ratio", scale="derived3",
|
||||
guard="project", prestep=pre, extrapolate=ext)[1]
|
||||
sq = lambda t: math.sqrt(abs(t)) * (0.0 if t == 0 else math.copysign(1.0, t)) # noqa: E731
|
||||
check("sqrt|x| sgn x, base rule", n_of(sq, -1 / 3, 1.0), 84)
|
||||
check("sin on [3,4], base rule", n_of(math.sin, 3.0, 4.0), 24)
|
||||
check("sin on [3,4], A'", n_of(math.sin, 3.0, 4.0, ap, "pattern"), 7)
|
||||
|
||||
print("\nTable 8 (A' on the protocol of section 4, sweep_aprime.json):")
|
||||
rows_ap = _load("sweep_aprime.json")
|
||||
A = f"pure+extP+{ap}.ratio.derived3"
|
||||
apr = {r["label"]: r for r in rows_ap}
|
||||
a = apr[f"{A}.project"]
|
||||
check("A' APS", a["APS"]["total"], 2438)
|
||||
check("A' CHA", a["CHA"]["total"], 2005)
|
||||
check("A' both", a["APS"]["total"] + a["CHA"]["total"], 4443)
|
||||
check("A' solved", a["APS"]["solved"] + a["CHA"]["solved"], 193)
|
||||
c = apr[f"{A}.credit"]
|
||||
check("A' earned budget", (c["APS"]["total"], c["CHA"]["total"]), (3608, 2807))
|
||||
check("A' under every safeguard: violations",
|
||||
sum(r["APS"]["over"] + r["CHA"]["over"] for lb, r in apr.items() if lb.startswith(A)), 0)
|
||||
check("A' safeguards measured", sum(1 for lb in apr if lb.startswith(A + ".")), 7)
|
||||
both = [r for r in _load("sweep_full.json")] + rows_ap
|
||||
for setname, opp, want in (("APS", "chandrupatla", (103, 20, 30)),
|
||||
("CHA", "chandrupatla", (29, 4, 7)),
|
||||
("APS", "toms748", (127, 6, 20)), ("CHA", "toms748", (38, 2, 0)),
|
||||
("APS", "brent", (118, 17, 18)), ("CHA", "brent", (33, 3, 2)),
|
||||
("APS", "itp_oracle", (126, 7, 20)),
|
||||
("CHA", "itp_oracle", (35, 1, 4))):
|
||||
w, t, l, _ = head_to_head(both, f"{A}.project", opp, setname)
|
||||
check(f"A' {setname} vs {opp}", (w, t, l), want)
|
||||
|
||||
def sign_p(w, l):
|
||||
k, n = min(w, l), w + l
|
||||
return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n)
|
||||
check("A' APS vs Chandrupatla p (2 digits)", float(f"{sign_p(103, 30):.2g}"), 1.5e-10)
|
||||
check("A' CHA vs Chandrupatla p (1 digit)", float(f"{sign_p(29, 7):.1g}"), 3e-4)
|
||||
check("A' below Chandrupatla, APS %", round(100 * (1 - 2438 / 2787), 1), 12.5)
|
||||
check("A' below Chandrupatla, CHA %", round(100 * (1 - 2005 / 2441), 1), 17.9)
|
||||
from where_we_lose import per_family
|
||||
af, cf = per_family(a, "CHA"), per_family(full["chandrupatla"], "CHA")
|
||||
check("A' fun9 vs Chandrupatla", (af["fun9"], cf["fun9"]), (65, 55))
|
||||
check("A' ahead on wide-bracket families", sum(af[k] < cf[k] for k in cf), 7)
|
||||
check("A' level on fun7", af["fun7"] == cf["fun7"], True)
|
||||
|
||||
print("\nTable 9 (ablation of A' on six corpora, sweep_ablation_all.json):")
|
||||
ab = _load("sweep_ablation_all.json")
|
||||
sets = ("APS", "CHA", "NEW", "RJL", "APP", "D")
|
||||
|
||||
def tot(r, s):
|
||||
return sum(n if ok else 200 for n, ok, _o in r[s].values())
|
||||
base_ = {s: tot(ab["ablA'"], s) for s in sets}
|
||||
for comp, want in (("-regime", (-7, 1013, 1660, 324, 77, 2269)),
|
||||
("-point", (-5, 479, 667, 272, 30, 1672)),
|
||||
("-sub", (-6, 0, -7, 16, 63, 893)),
|
||||
("-close", (130, 35, 97, 78, 135, -2)),
|
||||
("-half", (249, 65, 82, -16, -5, 1)),
|
||||
("-exp", (31, 143, 100, -21, -2, 87)),
|
||||
("-G2", (-22, -6, 123, -30, -10, 34)),
|
||||
("-contra", (11, -12, 29, 0, 1, -1))):
|
||||
got = tuple(tot(ab[f"abl{comp}"], s) - base_[s] for s in sets)
|
||||
check(f"ablation {comp}", got, want)
|
||||
|
||||
print("\nSection 9 (out-of-sample corpora):")
|
||||
seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json")
|
||||
for k in seen:
|
||||
seen[k].update(four[k])
|
||||
names = {"APS": 154, "CHA": 45, "NEW": 206, "RJL": 78, "APP": 132, "D": 240}
|
||||
for s, n in names.items():
|
||||
check(f"corpus {s} size", len(seen["Aprime"][s]), n)
|
||||
table10 = {"APS": (2259, 2595, 2707, 2636, (103, 20, 31)),
|
||||
"CHA": (1074, 1505, 3238, 3088, (31, 4, 10)),
|
||||
"NEW": (3290, 3716, 6395, 5746, (108, 33, 65)),
|
||||
"RJL": (1371, 1566, 2455, 2339, (44, 14, 20)),
|
||||
"APP": (1382, 1382, 1503, 1756, (48, 52, 32)),
|
||||
"D": (3656, 5717, 8267, 8084, (177, 35, 28))}
|
||||
|
||||
def cost(v):
|
||||
return v[0] if v[1] else 200
|
||||
for s, (ta, tc, tb, tt, wtl) in table10.items():
|
||||
got = tuple(tot(seen[k], s) for k in ("Aprime", "chandrupatla", "brent", "toms748"))
|
||||
check(f"Table 10 {s} totals", got, (ta, tc, tb, tt))
|
||||
ps = list(seen["Aprime"][s])
|
||||
w = sum(cost(seen["Aprime"][s][p]) < cost(seen["chandrupatla"][s][p]) for p in ps)
|
||||
l = sum(cost(seen["Aprime"][s][p]) > cost(seen["chandrupatla"][s][p]) for p in ps)
|
||||
check(f"Table 10 {s} vs Chandrupatla", (w, len(ps) - w - l, l), wtl)
|
||||
check(f"Table 10 {s} A' violations", sum(v[2] for v in seen["Aprime"][s].values()), 0)
|
||||
viol = {k: sum(v[2] for s in names for v in seen[k][s].values()) for k in ("brent", "toms748")}
|
||||
check("Brent / TOMS 748 bound violations over 855", (viol["brent"], viol["toms748"]), (118, 84))
|
||||
check("all 855 totals", tuple(sum(tot(seen[k], s) for s in names)
|
||||
for k in ("Aprime", "chandrupatla", "brent", "toms748")),
|
||||
(13032, 16481, 24565, 23649))
|
||||
d4 = seen["Aprime"]["D"]
|
||||
for label, cls, want in (("without targeted", ("D1", "D2", "D4", "D5"), (2403, 3569, (107, 31, 22))),
|
||||
("simple roots", ("D1", "D4"), (737, 747, (43, 22, 15)))):
|
||||
ps = [p for p in d4 if p[:2] in cls]
|
||||
a_ = sum(cost(d4[p]) for p in ps)
|
||||
c_ = sum(cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
||||
w = sum(cost(d4[p]) < cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
||||
l = sum(cost(d4[p]) > cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
||||
check(f"corpus 4 {label}", (a_, c_, (w, len(ps) - w - l, l)), want)
|
||||
for cls, want in (("D1", (433, 419, (20, 8, 12))), ("D2", (1163, 1805, (39, 1, 0))),
|
||||
("D3", (924, 1784, (37, 1, 2))), ("D4", (304, 328, (23, 14, 3))),
|
||||
("D5", (503, 1017, (25, 8, 7))), ("D6", (329, 364, (33, 3, 4)))):
|
||||
ps = [p for p in d4 if p.startswith(cls)]
|
||||
a_ = sum(cost(d4[p]) for p in ps)
|
||||
c_ = sum(cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
||||
w = sum(cost(d4[p]) < cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
||||
l = sum(cost(d4[p]) > cost(seen["chandrupatla"]["D"][p]) for p in ps)
|
||||
check(f"Table 11 {cls}", (a_, c_, (w, len(ps) - w - l, l)), want)
|
||||
c5 = [p for p in seen["Aprime"]["NEW"] if p.startswith("C5")]
|
||||
check("near-cancelling class of corpus 1", (sum(cost(seen["Aprime"]["NEW"][p]) for p in c5),
|
||||
sum(cost(seen["chandrupatla"]["NEW"][p]) for p in c5)),
|
||||
(491, 391))
|
||||
# the pre-registered verdicts on corpora 1-3 (variant without close and sub)
|
||||
oos = _load("sweep_oos.json")
|
||||
|
||||
def tot_new(r):
|
||||
return sum(n if ok else 200 for n, ok, _o in r["NEW"].values())
|
||||
|
||||
def h2h_new(r, o, keep=lambda p: True):
|
||||
ps = [p for p in r["NEW"] if keep(p)]
|
||||
w = sum(cost(r["NEW"][p]) < cost(o["NEW"][p]) for p in ps)
|
||||
l = sum(cost(r["NEW"][p]) > cost(o["NEW"][p]) for p in ps)
|
||||
return w, len(ps) - w - l, l
|
||||
check("corpus 1, variant carried: total vs Chandrupatla",
|
||||
(tot_new(oos["A"]), tot_new(oos["chandrupatla"])), (3492, 3899))
|
||||
check("corpus 1, variant carried: per problem", h2h_new(oos["A"], oos["chandrupatla"]),
|
||||
(105, 34, 67))
|
||||
check("corpus 1, with defect: per problem", h2h_new(oos["B"], oos["chandrupatla"]),
|
||||
(95, 33, 78))
|
||||
check("corpus 1, with defect: exponential class",
|
||||
h2h_new(oos["B"], oos["chandrupatla"], lambda p: p.startswith("C4")), (9, 5, 26))
|
||||
rj = _load("sweep_rootsjl.json")
|
||||
keep = [p for p, v in rj["A"].items() if not v[3]]
|
||||
check("Roots.jl, variant carried: totals", (sum(cost(rj["A"][p]) for p in keep),
|
||||
sum(cost(rj["chandrupatla"][p]) for p in keep)),
|
||||
(1464, 1566))
|
||||
w = sum(cost(rj["A"][p]) < cost(rj["chandrupatla"][p]) for p in keep)
|
||||
l = sum(cost(rj["A"][p]) > cost(rj["chandrupatla"][p]) for p in keep)
|
||||
check("Roots.jl, variant carried: per problem", (w, len(keep) - w - l, l), (38, 10, 30))
|
||||
app = _load("sweep_applied.json")
|
||||
check("applied, variant carried: totals", (sum(cost(v) for v in app["A"].values()),
|
||||
sum(cost(v) for v in app["chandrupatla"].values())),
|
||||
(1580, 1382))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user