65 lines
2.1 KiB
Python
65 lines
2.1 KiB
Python
"""Leave-one-out ablation of A' on all six corpora (APS, CHA, new corpus 1, Roots.jl, applied
|
|
+ GSL, fourth corpus), uniform success criterion (a sign change within 1e-6), for the paper's
|
|
label-honesty table. Descriptive: every corpus has been seen by now.
|
|
|
|
python -u sweep_ablation_all.py run -> sweep_ablation_all.json, then the table
|
|
python -u sweep_ablation_all.py --report
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from multiprocessing import Pool
|
|
|
|
from sweep_fourth import corpus, evaluate as _eval_one
|
|
import sweep_fourth
|
|
from sweep_full import APRIME_ABLATIONS
|
|
|
|
SETS = ("APS", "CHA", "NEW", "RJL", "APP", "D")
|
|
|
|
|
|
def spec(pre, ext):
|
|
return ("pure", "ratio", "derived3", "project", "x", pre, "recent", "distance", "none", ext)
|
|
|
|
|
|
for _name, (_pre, _ext) in APRIME_ABLATIONS.items():
|
|
sweep_fourth.CONFIGS[f"abl{_name}"] = spec(_pre, _ext)
|
|
|
|
|
|
def evaluate(job):
|
|
name, which = job
|
|
return name, which, _eval_one((name, which))[1]
|
|
|
|
|
|
def cost(v):
|
|
return v[0] if v[1] else 200
|
|
|
|
|
|
def main():
|
|
names = [f"abl{n}" for n in APRIME_ABLATIONS]
|
|
if "--report" not in sys.argv:
|
|
jobs = [(n, w) for n in names for w in ("seen", "fourth")]
|
|
with Pool(min(12, len(jobs))) as pool:
|
|
out = pool.map(evaluate, jobs)
|
|
R = {}
|
|
for n, w, res in out:
|
|
R.setdefault(n, {}).update(res)
|
|
with open("sweep_ablation_all.json", "w", encoding="utf8") as fh:
|
|
json.dump(R, fh)
|
|
with open("sweep_ablation_all.json", encoding="utf8") as fh:
|
|
R = json.load(fh)
|
|
tot = {n: {s: sum(cost(v) for v in R[n][s].values()) for s in SETS} for n in R}
|
|
base = tot["ablA'"]
|
|
print("A' totals: " + " ".join(f"{s} {base[s]}" for s in SETS))
|
|
print("delta = (without) - (with); + = the component earns its place")
|
|
for n in names[1:]:
|
|
d = {s: tot[n][s] - base[s] for s in SETS}
|
|
earned = [s for s in SETS if d[s] > 0]
|
|
print(f" {n[3:]:<8} " + " ".join(f"{s} {d[s]:+5d}" for s in SETS)
|
|
+ f" sum {sum(d.values()):+6d} earned on {len(earned)}/6")
|
|
print("ablation done")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|