sim-numerics/numbers_uniform.py

"""Numbers the revised text needs beyond verify_article.py's existing checks, all under the
paper's single criterion (sweep_uniform.json, sweep_fourth_*.json, sweep_itp.json), plus the
two small fresh runs the re-scoring cannot supply.  Run on Linux; stdout is kept as
results_numbers_uniform.txt and every number quoted from it is checked by verify_article.py.

    python -u numbers_uniform.py
"""
from __future__ import annotations

import math
from collections import defaultdict

from rescore_uniform import load_uniform

PROJ = "pure.ratio.derived3.project"
CREDIT = "pure.ratio.derived3.credit"
APRIME = "pure+extP+contra+flatanalogmempt+exp+half+close+sub.ratio.derived3.project"


def cost(v):
    return v[0] if v[1] else 200


def sign_p(w, l):
    k, n = min(w, l), w + l
    return 1.0 if n == 0 else min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n)


def rank(rows, s):
    return [r["label"] for r in sorted(rows, key=lambda r: (r[s]["n"] - r[s]["solved"], r[s]["total"]))]


def family(pid):
    """APS: the function number (aps.NN.MM); CHA: funN; the unit of dependence."""
    return ".".join(pid.split(".")[:2]) if pid.startswith("aps") else pid.split(".")[0]


def families(a, b, s):
    """Family-level comparison: per family the total of a against the total of b."""
    ta, tb = defaultdict(int), defaultdict(int)
    for p in a[s]["per"]:
        ta[family(p)] += cost(a[s]["per"][p])
        tb[family(p)] += cost(b[s]["per"][p])
    w = sum(ta[k] < tb[k] for k in ta)
    l = sum(ta[k] > tb[k] for k in ta)
    return w, len(ta) - w - l, l, dict(ta), dict(tb)


def main():
    full = load_uniform("sweep_full.json")
    by = {r["label"]: r for r in full}
    for s in ("APS", "CHA"):
        rk = rank(full, s)
        print(f"{s} places: " + ", ".join(
            f"{lb} {rk.index(lb) + 1}" for lb in (
                "chandrupatla", "pure.ratio.derived3.none", PROJ, "toms748", "brent", "dekker",
                "itp_oracle", "ridders", CREDIT, "itp", "illinois", "bisection", "regula_falsi",
                "pure.exp_abs.bracket.both", "pure.exp_abs.bracket.bracket",
                "pure.exp_abs.bracket.credit")))
        for lb in ("dekker", "illinois", "regula_falsi", "brent", "toms748",
                   "pure.ratio.derived3.none", "pure.exp_abs.bracket.both"):
            r = by[lb][s]
            print(f"   {lb:<28} solved {r['solved']}/{r['n']} total {r['total']} over {r['over']}")
    # Corollary 2: best over scales, step guard (Table 4's individual ranking)
    best = {}
    for r in full:
        if r["spec"][0] == "pure" and r["spec"][3] == "step":
            best[r["spec"][1]] = min(best.get(r["spec"][1], 10 ** 9),
                                     r["APS"]["total"] + r["CHA"]["total"])
    print("best over scales (step guard):", dict(sorted(best.items(), key=lambda kv: kv[1])))

    # revision point 8: families as the unit
    ap = {r["label"]: r for r in load_uniform("sweep_aprime.json")}[APRIME]
    for name, a in (("base rule", by[PROJ]), ("A'", ap)):
        for s in ("APS", "CHA"):
            w, t, l, ta, tb = families(a, by["chandrupatla"], s)
            print(f"{name} vs Chandrupatla by family, {s}: {w}/{t}/{l} of {w + t + l} families,"
                  f" p={sign_p(w, l):.2g}")
            if name == "A'" and s == "CHA":
                print("   per family A' / Chandrupatla:",
                      {k: (ta[k], tb[k]) for k in sorted(ta)})
        for s in ("APS", "CHA"):
            ps = list(a[s]["per"])
            w = sum(cost(a[s]["per"][p]) < cost(by["chandrupatla"][s]["per"][p]) for p in ps)
            l = sum(cost(a[s]["per"][p]) > cost(by["chandrupatla"][s]["per"][p]) for p in ps)
            print(f"{name} vs Chandrupatla by problem, {s}: {w}/{len(ps) - w - l}/{l}"
                  f" p={sign_p(w, l):.2g}")
    # Table 6: the 60 paired (function, safeguard) cells of the far-end reading
    order = load_uniform("sweep_order.json")
    cells = {}
    for r in order:
        sim, sc, gu = r["spec"][1], r["spec"][2], r["spec"][3]
        ex = r["spec"][8] if len(r["spec"]) > 8 else "none"
        if sc == "far":
            cells[(sim, gu, ex)] = r["APS"]["total"] + r["CHA"]["total"]
    pairs = [(k[0], k[1]) for k in cells if k[2] == "none" and (k[0], k[1], "order") in cells]
    cheaper = sum(cells[(s_, g, "order")] < cells[(s_, g, "none")] for s_, g in pairs)
    print(f"far-end reading, measured order cheaper in {cheaper} of {len(pairs)} cells")
    fp = [r for r in order if r["spec"][1] == "rational" and r["spec"][2] == "far"
          and r["spec"][3] == "none"]
    for r in fp:
        print(f"   false position reading {r['spec'][8]}: APS solved {r['APS']['solved']} "
              f"total {r['APS']['total']}")
    # negative item 9: the capped scale
    cap = {r["label"]: r for r in load_uniform("sweep_cap.json")}
    dire = {r["label"]: r for r in load_uniform("sweep_direction.json")}
    for lb in ("pure+dissim+slope.ratio.derived3_cap.project",
               "pure+dissim+slope.ratio.derived3_cap.safe"):
        r = cap[lb]
        print(f"cap {lb}: {r['APS']['total']} / {r['CHA']['total']} solved "
              f"{r['APS']['solved'] + r['CHA']['solved']}")
    for lb in ("pure+dissim+slope.ratio.derived3.project", "pure+dissim+slope.ratio.derived3.safe"):
        r = dire[lb]
        print(f"control {lb}: {r['APS']['total']} / {r['CHA']['total']}")
    # Table 1a: which canonical problems the multiplier moves
    from simroot.pure import pure_root
    from stage12_validation import load, per_problem
    probs = [p for p in load() if p[0] == "APS"]
    runs = {}
    for fac in (1.25, 1.5, 2.0):
        runs[fac] = per_problem(lambda f, a, b, xt, mx, _f=fac: pure_root(
            f, a, b, xt, mx, similarity="ratio", scale="derived3", guard="project",
            budget_factor=_f), probs, 200, criterion="sign")
    moved = {p: (runs[1.25][p][0], runs[1.5][p][0], runs[2.0][p][0]) for p in runs[2.0]
             if len({runs[1.25][p][0], runs[1.5][p][0], runs[2.0][p][0]}) > 1}
    print("canonical problems moved by the multiplier (1.25, 1.5, 2):", moved)
    # negative item 5: the phase detector (scale derived3_m) on the full benchmark
    allp = load()
    for sc in ("derived3", "derived3_m"):
        pp = per_problem(lambda f, a, b, xt, mx, _s=sc: pure_root(
            f, a, b, xt, mx, similarity="ratio", scale=_s, guard="project"), allp, 200,
            criterion="sign")
        t = {"APS": 0, "CHA": 0}
        for s_, pid, *_ in allp:
            t[s_] += pp[pid][0]
        print(f"phase detector item: scale {sc}: APS {t['APS']} CHA {t['CHA']} "
              f"both {t['APS'] + t['CHA']}")
    print("numbers uniform done")


if __name__ == "__main__":
    main()