vvsandClaude Opus 5
de90c7a089
What the tables' totals hid: problem by problem the scheduled projection beats Chandrupatla on APS 87/24/42, exact sign test p = 9e-5, while trailing by 0.7% in total; on CHA Chandrupatla is genuinely better (26 to 9, p = 0.006). The earned projection ties him on APS problems (79/3/71, p = 0.57). Benchmark not yet switched to the library baseline - awaiting Valeri's decision together with the framing of the proposed method. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
63 lines
2.2 KiB
Python
63 lines
2.2 KiB
Python
"""Audit: is "Chandrupatla is first on the canonical set" a result or a total?
|
|
|
|
Totals differ by 0.7% (2806 against 2787 with the tolerance corrected); the paper itself
|
|
reads 0.5% as a tie. Problem by problem, with an exact two-sided sign test on the
|
|
problems where the two differ, for the scheduled projection AND the earned one against
|
|
Chandrupatla at the corrected tolerance.
|
|
|
|
python -u audit_head_to_head.py
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
from audit_chandrupatla_tolerance import ours_half
|
|
from simroot.baselines import best_of, chandrupatla_scipy
|
|
from simroot.pure import pure_root
|
|
from stage12_validation import load, per_problem
|
|
|
|
|
|
def sign_test(w, l):
|
|
n = w + l
|
|
if n == 0:
|
|
return 1.0
|
|
k = min(w, l)
|
|
return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n)
|
|
|
|
|
|
def main():
|
|
probs = load()
|
|
ch = per_problem(best_of(ours_half, chandrupatla_scipy), probs, 200)
|
|
ours = {
|
|
"scheduled projection (c=2)": lambda f, a, b, xt, mx: pure_root(
|
|
f, a, b, xt, mx, similarity="ratio", scale="derived3", guard="project"),
|
|
"earned projection (credit)": lambda f, a, b, xt, mx: pure_root(
|
|
f, a, b, xt, mx, similarity="ratio", scale="derived3", guard="credit"),
|
|
}
|
|
sets = {s: [pid for s_, pid, *_ in probs if s_ == s] for s in ("APS", "CHA")}
|
|
out = {}
|
|
for name, m in ours.items():
|
|
p = per_problem(m, probs, 200)
|
|
for s, ids in sets.items():
|
|
w = t = l = 0
|
|
for pid in ids:
|
|
(no, oko), (nc, okc) = p[pid], ch[pid]
|
|
if not (oko and okc):
|
|
continue
|
|
if no < nc:
|
|
w += 1
|
|
elif no > nc:
|
|
l += 1
|
|
else:
|
|
t += 1
|
|
tot_o = sum(p[pid][0] for pid in ids)
|
|
tot_c = sum(ch[pid][0] for pid in ids)
|
|
out[(name, s)] = (w, t, l, sign_test(w, l), tot_o, tot_c)
|
|
print(f"{name:<28} {s}: wins/ties/losses {w}/{t}/{l} sign test p = "
|
|
f"{sign_test(w, l):.2g} totals {tot_o} vs Chandrupatla {tot_c}")
|
|
return out
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|