A' on the paper's own protocol (published roots, server): APS 2438, CHA 2005 (4443) vs Chandrupatla 2787/2441 (5228); per problem 103/20/30 and 29/4/7; earned-budget variant 3608/2807; guarantee held under every safeguard. Leave-one-out ablation of A' on all six corpora: regime +5336, point +3115, sub +959, close +473, half +376, exp +338, G2 +89 (2/6 corpora), contra +28 (3/6) - every component positive in sum, G2 and contra marginal.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,11 @@
|
||||
A' totals: APS 2259 CHA 1074 NEW 3290 RJL 1371 APP 1382 D 3656
|
||||
delta = (without) - (with); + = the component earns its place
|
||||
-G2 APS -22 CHA -6 NEW +123 RJL -30 APP -10 D +34 sum +89 earned on 2/6
|
||||
-contra APS +11 CHA -12 NEW +29 RJL +0 APP +1 D -1 sum +28 earned on 3/6
|
||||
-half APS +249 CHA +65 NEW +82 RJL -16 APP -5 D +1 sum +376 earned on 4/6
|
||||
-exp APS +31 CHA +143 NEW +100 RJL -21 APP -2 D +87 sum +338 earned on 4/6
|
||||
-point APS -5 CHA +479 NEW +667 RJL +272 APP +30 D +1672 sum +3115 earned on 5/6
|
||||
-regime APS -7 CHA +1013 NEW +1660 RJL +324 APP +77 D +2269 sum +5336 earned on 5/6
|
||||
-close APS +130 CHA +35 NEW +97 RJL +78 APP +135 D -2 sum +473 earned on 5/6
|
||||
-sub APS -6 CHA +0 NEW -7 RJL +16 APP +63 D +893 sum +959 earned on 3/6
|
||||
ablation done
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,64 @@
|
||||
"""Leave-one-out ablation of A' on all six corpora (APS, CHA, new corpus 1, Roots.jl, applied
|
||||
+ GSL, fourth corpus), uniform success criterion (a sign change within 1e-6), for the paper's
|
||||
label-honesty table. Descriptive: every corpus has been seen by now.
|
||||
|
||||
python -u sweep_ablation_all.py run -> sweep_ablation_all.json, then the table
|
||||
python -u sweep_ablation_all.py --report
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from multiprocessing import Pool
|
||||
|
||||
from sweep_fourth import corpus, evaluate as _eval_one
|
||||
import sweep_fourth
|
||||
from sweep_full import APRIME_ABLATIONS
|
||||
|
||||
SETS = ("APS", "CHA", "NEW", "RJL", "APP", "D")
|
||||
|
||||
|
||||
def spec(pre, ext):
|
||||
return ("pure", "ratio", "derived3", "project", "x", pre, "recent", "distance", "none", ext)
|
||||
|
||||
|
||||
for _name, (_pre, _ext) in APRIME_ABLATIONS.items():
|
||||
sweep_fourth.CONFIGS[f"abl{_name}"] = spec(_pre, _ext)
|
||||
|
||||
|
||||
def evaluate(job):
|
||||
name, which = job
|
||||
return name, which, _eval_one((name, which))[1]
|
||||
|
||||
|
||||
def cost(v):
|
||||
return v[0] if v[1] else 200
|
||||
|
||||
|
||||
def main():
|
||||
names = [f"abl{n}" for n in APRIME_ABLATIONS]
|
||||
if "--report" not in sys.argv:
|
||||
jobs = [(n, w) for n in names for w in ("seen", "fourth")]
|
||||
with Pool(min(12, len(jobs))) as pool:
|
||||
out = pool.map(evaluate, jobs)
|
||||
R = {}
|
||||
for n, w, res in out:
|
||||
R.setdefault(n, {}).update(res)
|
||||
with open("sweep_ablation_all.json", "w", encoding="utf8") as fh:
|
||||
json.dump(R, fh)
|
||||
with open("sweep_ablation_all.json", encoding="utf8") as fh:
|
||||
R = json.load(fh)
|
||||
tot = {n: {s: sum(cost(v) for v in R[n][s].values()) for s in SETS} for n in R}
|
||||
base = tot["ablA'"]
|
||||
print("A' totals: " + " ".join(f"{s} {base[s]}" for s in SETS))
|
||||
print("delta = (without) - (with); + = the component earns its place")
|
||||
for n in names[1:]:
|
||||
d = {s: tot[n][s] - base[s] for s in SETS}
|
||||
earned = [s for s in SETS if d[s] > 0]
|
||||
print(f" {n[3:]:<8} " + " ".join(f"{s} {d[s]:+5d}" for s in SETS)
|
||||
+ f" sum {sum(d.values()):+6d} earned on {len(earned)}/6")
|
||||
print("ablation done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because one or more lines are too long
+35
-3
@@ -323,6 +323,35 @@ def build_grid_half(smoke):
|
||||
for gu in guards for ext in ("none", "pattern") for pre in presteps]
|
||||
|
||||
|
||||
APRIME = "contra+flatanalogmempt+exp+half+close+sub"
|
||||
APRIME_ABLATIONS = { # leave one component out; ext = the extrapolation mode
|
||||
"A'": (APRIME, "pattern"),
|
||||
"-G2": (APRIME, "none"),
|
||||
"-contra": ("flatanalogmempt+exp+half+close+sub", "pattern"),
|
||||
"-half": ("contra+flatanalogmempt+exp+close+sub", "pattern"),
|
||||
"-exp": ("contra+flatanalogmempt+half+close+sub", "pattern"),
|
||||
"-point": ("contra+flatanalogmem+exp+half+close+sub", "pattern"),
|
||||
"-regime": ("contra+half+close", "pattern"),
|
||||
"-close": ("contra+flatanalogmempt+exp+half+sub", "pattern"),
|
||||
"-sub": ("contra+flatanalogmempt+exp+half+close", "pattern"),
|
||||
}
|
||||
|
||||
|
||||
def build_grid_aprime(smoke):
|
||||
"""The paper's proposed rule A' on the paper's own protocol (published roots): under every
|
||||
safeguard (the guarantee claims) and, under the projection, with each component left out."""
|
||||
specs = []
|
||||
for gu in (("project", "credit") if smoke else GUARD_MODES):
|
||||
specs.append(("pure", "ratio", "derived3", gu, "x", APRIME, "recent", "distance",
|
||||
"none", "pattern"))
|
||||
for name, (pre, ext) in APRIME_ABLATIONS.items():
|
||||
if name == "A'":
|
||||
continue
|
||||
specs.append(("pure", "ratio", "derived3", "project", "x", pre, "recent", "distance",
|
||||
"none", ext))
|
||||
return specs
|
||||
|
||||
|
||||
def build_grid_extrap2(smoke):
|
||||
"""PREREG_extrapolation_gates_2026-09-21: the same cells as build_grid_extrap, every
|
||||
one in all five modes (none, A, G1, G2, G3)."""
|
||||
@@ -504,10 +533,12 @@ def main():
|
||||
ptmode = "--orderpt" in sys.argv # split at the point from the pair's order
|
||||
expmode = "--explimit" in sys.argv # the translation limit of the analogy, m = inf
|
||||
halfmode = "--half" in sys.argv # the half promised by the step vs the bracket
|
||||
apmode = "--aprime" in sys.argv # the paper's proposed rule A' and its ablations
|
||||
combomode = "--combo" in sys.argv # modes A and B together
|
||||
pointsmode = "--points" in sys.argv # mode A: triple by shape
|
||||
dirmode = "--direction" in sys.argv # mode B: far end by dissimilarity
|
||||
specs = (build_grid_half(smoke) if halfmode else
|
||||
specs = (build_grid_aprime(smoke) if apmode else
|
||||
build_grid_half(smoke) if halfmode else
|
||||
build_grid_explimit(smoke) if expmode else
|
||||
build_grid_orderpt(smoke) if ptmode else
|
||||
build_grid_defcontra(smoke) if dcmode else
|
||||
@@ -545,8 +576,9 @@ def main():
|
||||
|
||||
# the published sweep_full.json is what verify_article.py checks the paper
|
||||
# against, so the log grid never writes into it
|
||||
if halfmode or expmode or ptmode or dcmode or memmode or analogmode or judgemode or exitmode or stickmode or creepmode or flat2mode or flatmode or ext2mode or extmode or decmode or ordermode or capmode or combomode or pointsmode or dirmode:
|
||||
base = ("sweep_half" if halfmode else
|
||||
if apmode or halfmode or expmode or ptmode or dcmode or memmode or analogmode or judgemode or exitmode or stickmode or creepmode or flat2mode or flatmode or ext2mode or extmode or decmode or ordermode or capmode or combomode or pointsmode or dirmode:
|
||||
base = ("sweep_aprime" if apmode else
|
||||
"sweep_half" if halfmode else
|
||||
"sweep_explimit" if expmode else
|
||||
"sweep_orderpt" if ptmode else
|
||||
"sweep_defcontra" if dcmode else
|
||||
|
||||
Reference in New Issue
Block a user