"""Audit: is "Chandrupatla is first on the canonical set" a result or a total?

Totals differ by 0.7% (2806 against 2787 with the tolerance corrected); the paper itself
reads 0.5% as a tie.  Problem by problem, with an exact two-sided sign test on the
problems where the two differ, for the scheduled projection AND the earned one against
Chandrupatla at the corrected tolerance.

    python -u audit_head_to_head.py
"""
from __future__ import annotations

import math

from audit_chandrupatla_tolerance import ours_half
from simroot.baselines import best_of, chandrupatla_scipy
from simroot.pure import pure_root
from stage12_validation import load, per_problem


def sign_test(w, l):
    n = w + l
    if n == 0:
        return 1.0
    k = min(w, l)
    return min(1.0, 2.0 * sum(math.comb(n, i) for i in range(k + 1)) / 2.0 ** n)


def main():
    probs = load()
    ch = per_problem(best_of(ours_half, chandrupatla_scipy), probs, 200)
    ours = {
        "scheduled projection (c=2)": lambda f, a, b, xt, mx: pure_root(
            f, a, b, xt, mx, similarity="ratio", scale="derived3", guard="project"),
        "earned projection (credit)": lambda f, a, b, xt, mx: pure_root(
            f, a, b, xt, mx, similarity="ratio", scale="derived3", guard="credit"),
    }
    sets = {s: [pid for s_, pid, *_ in probs if s_ == s] for s in ("APS", "CHA")}
    out = {}
    for name, m in ours.items():
        p = per_problem(m, probs, 200)
        for s, ids in sets.items():
            w = t = l = 0
            for pid in ids:
                (no, oko), (nc, okc) = p[pid], ch[pid]
                if not (oko and okc):
                    continue
                if no < nc:
                    w += 1
                elif no > nc:
                    l += 1
                else:
                    t += 1
            tot_o = sum(p[pid][0] for pid in ids)
            tot_c = sum(ch[pid][0] for pid in ids)
            out[(name, s)] = (w, t, l, sign_test(w, l), tot_o, tot_c)
            print(f"{name:<28} {s}: wins/ties/losses {w}/{t}/{l}   sign test p = "
                  f"{sign_test(w, l):.2g}   totals {tot_o} vs Chandrupatla {tot_c}")
    return out


if __name__ == "__main__":
    main()