PREREG_out_of_sample outcome (server). Variant A (G2 + contra + remembered analogy + order point + exp limit + half) TRANSFERS: new corpus 3492 vs Chandrupatla 3899, per problem 105/34/67 p=0.0046, guarantee held, solved 205 = 205. Variant B (defect + remembered + point + exp) does NOT (total 3587 below, but per problem 95/33/78 p=0.22; loses the exponential class 9/5/26). Per class the transfer is carried by the multiple/flat-root class C3 (1194 vs 1792, 37/0/3); on C0-C2 A is at par per problem and ~6% dearer in total, C4 ~ par, C5 (near-cancelling) 691 vs 574. Ablation: every component earns its place on at least one set (contra only on APS/CHA: +9/+22, NEW -8; G2 only on NEW: +82). Addendum: C0.05 is SciPy's random step function - seeded and its cache cleared before every run for reproducibility; two unseeded runs gave the same verdicts.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
+165
@@ -0,0 +1,165 @@
|
||||
"""PREREG_out_of_sample_2026-09-21: the two strongest variants, their leave-one-out ablations
|
||||
and the three library rivals, on APS, CHA and the out-of-sample corpus (corpus_new.py).
|
||||
|
||||
python -u sweep_oos.py run -> sweep_oos.json, then the verdicts
|
||||
python -u sweep_oos.py --report verdicts only, from sweep_oos.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from multiprocessing import Pool
|
||||
|
||||
from corpus_new import build as build_new
|
||||
from sweep_full import ACC, MAXFEV, XTOL, load_problems, make_method
|
||||
|
||||
P = "project"
|
||||
|
||||
|
||||
def spec(pre, ext="none"):
|
||||
return ("pure", "ratio", "derived3", P, "x", pre, "recent", "distance", "none", ext)
|
||||
|
||||
|
||||
CONFIGS = {
|
||||
"base": spec("none"),
|
||||
"A": spec("contra+flatanalogmempt+exp+half", "pattern"),
|
||||
"A-G2": spec("contra+flatanalogmempt+exp+half"),
|
||||
"A-contra": spec("flatanalogmempt+exp+half", "pattern"),
|
||||
"A-half": spec("contra+flatanalogmempt+exp", "pattern"),
|
||||
"A-exp": spec("contra+flatanalogmempt+half", "pattern"),
|
||||
"A-point": spec("contra+flatanalogmem+exp+half", "pattern"),
|
||||
"A-regime": spec("contra+half", "pattern"),
|
||||
"B": spec("defect+flatanalogmempt+exp"),
|
||||
"B-defect": spec("flatanalogmempt+exp"),
|
||||
"B-exp": spec("defect+flatanalogmempt"),
|
||||
"B-point": spec("defect+flatanalogmem+exp"),
|
||||
"B-regime": spec("defect"),
|
||||
"chandrupatla": ("base", "chandrupatla"),
|
||||
"brent": ("base", "brent"),
|
||||
"toms748": ("base", "toms748"),
|
||||
}
|
||||
|
||||
|
||||
def problems():
|
||||
out = [(s, pid, f, a, b, r) for s, pid, f, a, b, r in load_problems()]
|
||||
new, _ = build_new()
|
||||
out += [("NEW", pid, f, a, b, r) for _cls, pid, f, a, b, r in new]
|
||||
return out
|
||||
|
||||
|
||||
def _reset_random_problem():
|
||||
"""SciPy's original.05.00 (our C0.05) is a random step function: unseeded random() values
|
||||
kept in a module-level cache shared by every run in the process. Addendum to the PREREG
|
||||
(after two unseeded runs gave the same verdicts): clear the cache and seed before EVERY
|
||||
run, so each method meets the same function and the run is reproducible."""
|
||||
import random
|
||||
|
||||
import scipy.optimize._tstutils as T
|
||||
T._f6_cache.clear()
|
||||
random.seed(20260921)
|
||||
|
||||
|
||||
def evaluate(name):
|
||||
method = make_method(CONFIGS[name])
|
||||
res = {}
|
||||
for s, pid, f, a, b, root in problems():
|
||||
_reset_random_problem()
|
||||
try:
|
||||
x, n, conv = method(f, a, b, XTOL, MAXFEV)
|
||||
except Exception:
|
||||
x, n, conv = float("nan"), MAXFEV, False
|
||||
ok = bool(conv and math.isfinite(x) and abs(x - root) <= ACC)
|
||||
over = ok and int(n) > 2 * math.ceil(math.log2(abs(b - a) / XTOL)) + 4
|
||||
res.setdefault(s, {})[pid] = [int(n), ok, bool(over)]
|
||||
return name, res
|
||||
|
||||
|
||||
def total(r, s, keep=None):
|
||||
return sum((n if ok else MAXFEV) for pid, (n, ok, _o) in r[s].items()
|
||||
if keep is None or keep(pid))
|
||||
|
||||
|
||||
def h2h(r, o, s, keep=None):
|
||||
w = t = l = 0
|
||||
for pid, (n, ok, _o) in r[s].items():
|
||||
if keep is not None and not keep(pid):
|
||||
continue
|
||||
c1 = n if ok else MAXFEV
|
||||
n2, ok2, _ = o[s][pid]
|
||||
c2 = n2 if ok2 else MAXFEV
|
||||
w += c1 < c2
|
||||
t += c1 == c2
|
||||
l += c1 > c2
|
||||
return w, t, l
|
||||
|
||||
|
||||
def sign_p(w, l):
|
||||
n = w + l
|
||||
if n == 0:
|
||||
return 1.0
|
||||
k = min(w, l)
|
||||
return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n)
|
||||
|
||||
|
||||
def report(R):
|
||||
ch = R["chandrupatla"]
|
||||
print("TOTALS APS CHA NEW (NEW C0-C2 NEW C3-C5)")
|
||||
neutral = lambda pid: pid[:2] in ("C0", "C1", "C2")
|
||||
ours_home = lambda pid: pid[:2] in ("C3", "C4", "C5")
|
||||
for k in ("base", "A", "B", "chandrupatla", "brent", "toms748"):
|
||||
r = R[k]
|
||||
print(f" {k:<18} {total(r, 'APS'):6d} {total(r, 'CHA'):6d} {total(r, 'NEW'):6d}"
|
||||
f" ({total(r, 'NEW', neutral):6d} {total(r, 'NEW', ours_home):6d})")
|
||||
print("\nOUT OF SAMPLE (NEW) - verdicts")
|
||||
for v in ("A", "B"):
|
||||
r = R[v]
|
||||
o1 = total(r, "NEW") < total(ch, "NEW")
|
||||
w, t, l = h2h(r, ch, "NEW")
|
||||
o2 = w > l and sign_p(w, l) < 0.05
|
||||
viol = sum(o for _n, _ok, o in r["NEW"].values())
|
||||
solved = sum(ok for _n, ok, _o in r["NEW"].values())
|
||||
solved_ch = sum(ok for _n, ok, _o in ch["NEW"].values())
|
||||
o3 = viol == 0 and solved >= solved_ch
|
||||
o4 = total(r, "NEW") < total(R["base"], "NEW")
|
||||
print(f" {v}: O1 {'✓' if o1 else '✗'} ({total(r, 'NEW')} vs {total(ch, 'NEW')}) "
|
||||
f"O2 {'✓' if o2 else '✗'} ({w}/{t}/{l} p={sign_p(w, l):.2g}) "
|
||||
f"O3 {'✓' if o3 else '✗'} (viol {viol}, solved {solved} vs {solved_ch}) "
|
||||
f"O4 {'✓' if o4 else '✗'} -> "
|
||||
+ ("TRANSFERS" if o1 and o2 and o3 else "does NOT transfer"))
|
||||
for rn in ("brent", "toms748"):
|
||||
w2, t2, l2 = h2h(r, R[rn], "NEW")
|
||||
print(f" vs {rn:<8} total {total(r, 'NEW')} vs {total(R[rn], 'NEW')} "
|
||||
f"per problem {w2}/{t2}/{l2} p={sign_p(w2, l2):.2g}")
|
||||
print(" per class (ours / Chandrupatla, per problem w/t/l):")
|
||||
for cls in ("C0", "C1", "C2", "C3", "C4", "C5"):
|
||||
keep = lambda pid, c=cls: pid.startswith(c)
|
||||
w3, t3, l3 = h2h(r, ch, "NEW", keep)
|
||||
print(f" {cls}: {total(r, 'NEW', keep):5d} / {total(ch, 'NEW', keep):5d}"
|
||||
f" {w3}/{t3}/{l3} p={sign_p(w3, l3):.2g}")
|
||||
print("\nABLATION delta = (without) - (with); + = the component earns its place")
|
||||
for v, parts in (("A", ("G2", "contra", "half", "exp", "point", "regime")),
|
||||
("B", ("defect", "exp", "point", "regime"))):
|
||||
r = R[v]
|
||||
for p in parts:
|
||||
o = R[f"{v}-{p}"]
|
||||
d = [total(o, s) - total(r, s) for s in ("APS", "CHA", "NEW")]
|
||||
earned = any(x > 0 for x in d)
|
||||
print(f" {v} - {p:<7} APS {d[0]:+5d} CHA {d[1]:+5d} NEW {d[2]:+5d} "
|
||||
+ ("earned" if earned else "NOT earned"))
|
||||
|
||||
|
||||
def main():
|
||||
if "--report" not in sys.argv:
|
||||
with Pool(min(8, len(CONFIGS))) as pool:
|
||||
R = dict(pool.map(evaluate, list(CONFIGS)))
|
||||
with open("sweep_oos.json", "w", encoding="utf8") as fh:
|
||||
json.dump(R, fh)
|
||||
with open("sweep_oos.json", encoding="utf8") as fh:
|
||||
R = json.load(fh)
|
||||
report(R)
|
||||
print("oos done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user