sim-numerics/report_sweep.py

"""Read sweep_full.json and answer the questions the paper needs answered.

Everything printed here comes from the single server run; no number is
recomputed on a different machine.  (The test functions use libm, whose last
digit differs between Windows and Linux, so mixing platforms would move
evaluation counts by a unit or two - measured: scipy's toms748 scores 2819 on
Windows and 2820 on Linux for the same set.)
"""
from __future__ import annotations

import json
import sys

BASE = {"bisection", "regula_falsi", "illinois", "pegasus", "anderson_bjorck",
        "ridders", "brent", "toms748", "itp", "itp_oracle"}

# The configuration the paper proposes.  It is not the winner of a search over
# the grid - it is what the theory of section 2 dictates, fixed before the
# ranking is looked at:
#   similarity  a nonzero slope of (1 - S) at 0   -> the ratio form 1 - |z|
#   scale       the finest data-derived one        -> the three-point difference
#   safeguard   the one that carries a bound without destroying the order ->
#               the scheduled projection, twice the bisection budget
# Valeri's decision, 2026-09-21 (variant a): the projection is proposed with its
# multiplier DECLARED as the method's one free constant - on the canonical set the
# total is the same for every multiplier from 1.25 to 4, on the wide-bracket set it
# is not (measure_budget_factor.py).  The earned projection (credit) is kept as the
# constant-free variant; it was PROPOSED here for a while, a choice recorded in code
# but never taken by Valeri.
# The grid is evidence for the theory, not the method's selection procedure.
PROPOSED = "pure.ratio.derived3.project"
CREDIT = "pure.ratio.derived3.credit"     # the constant-free variant
TUNED = {"illinois", "pegasus", "anderson_bjorck", "brent", "toms748",
         "itp", "itp_oracle"}


def load(path="sweep_full.json"):
    with open(path, encoding="utf8") as fh:
        return json.load(fh)


def rank(rows, setname):
    return sorted(rows, key=lambda r: (r[setname]["n"] - r[setname]["solved"],
                                       r[setname]["total"]))


def place(rows, setname, lbl):
    for i, r in enumerate(rank(rows, setname), 1):
        if r["label"] == lbl:
            return i
    raise KeyError(f"no configuration labelled {lbl!r} in the sweep; "
                   "a mistyped name must not be reported as a missing place")


def best_pure(rows, setname):
    """The best configuration OF OURS.  The overall winner of a set may be a
    baseline - itp_oracle tops Chandrupatla - and comparing that against
    itself prints nothing of interest."""
    for r in rank(rows, setname):
        if r["label"].startswith("pure."):
            return r["label"]
    raise KeyError("no pure configuration in the sweep")


def head_to_head(rows, a, b, setname):
    """Wins/ties/losses of a against b over EVERY problem of the set, a failure charged the
    full budget - the charging of the totals.  Until 2026-09-22 only the problems both
    methods solved were counted, so a row of the paper summed to 153 of 154 (found on revision)."""
    ra = next(r for r in rows if r["label"] == a)[setname]
    rb = next(r for r in rows if r["label"] == b)[setname]
    win = tie = loss = both = 0
    for pid, (na, oka) in ra["per"].items():
        nb, okb = rb["per"][pid]
        na, nb = (na if oka else 200), (nb if okb else 200)
        both += 1
        if na < nb:
            win += 1
        elif na > nb:
            loss += 1
        else:
            tie += 1
    return win, tie, loss, both


def main():
    rows = load(sys.argv[1] if len(sys.argv) > 1 else "sweep_full.json")
    print(f"{len(rows)} configurations\n")

    for setname in ("APS", "CHA"):
        r = rank(rows, setname)
        n = r[0][setname]["n"]
        print("=" * 90)
        print(f"{setname}  ({n} problems)   top 12, with the best baseline "
              f"positions marked")
        print("=" * 90)
        print(f"{'#':>3s}  {'configuration':38s}{'tuning':>8s}{'solved':>9s}"
              f"{'total':>9s}{'over 2x':>9s}")
        print("-" * 90)
        for i, row in enumerate(r[:12], 1):
            d = row[setname]
            tuned = "yes" if row["label"] in TUNED else "no"
            star = " <-" if row["label"] not in BASE else ""
            print(f"{i:>3d}  {row['label']:38s}{tuned:>8s}"
                  f"{d['solved']:>6d}/{d['n']:<3d}{d['total']:>9d}"
                  f"{d['over']:>9d}{star}")
        print()
        for b in ("brent", "toms748", "itp", "itp_oracle", "bisection"):
            p = place(rows, setname, b)
            d = next(x for x in rows if x["label"] == b)[setname]
            print(f"      {b:38s}place {p:>3d}   total {d['total']}")
        print()

    print("=" * 90)
    print(f"THE PROPOSED CONFIGURATION  {PROPOSED}")
    print("  (dictated by the theory, not selected from the ranking)")
    print("=" * 90)
    for setname in ("APS", "CHA"):
        d = next(r for r in rows if r["label"] == PROPOSED)[setname]
        print(f"  {setname}: place {place(rows, setname, PROPOSED):>3d} of "
              f"{len(rows)}   solved {d['solved']}/{d['n']}   "
              f"total {d['total']}   over 2x bound {d['over']}")
        for opp in ("brent", "toms748", "itp", "itp_oracle", "bisection"):
            w, t, l, both = head_to_head(rows, PROPOSED, opp, setname)
            print(f"        vs {opp:12s}: {w} wins / {t} ties / {l} losses  "
                  f"(of {both})")
    print()

    print("=" * 90)
    print("HEAD TO HEAD, best pure configuration of each set")
    print("=" * 90)
    for setname in ("APS", "CHA"):
        best = best_pure(rows, setname)
        for opp in ("brent", "toms748", "itp", "itp_oracle"):
            w, t, l, both = head_to_head(rows, best, opp, setname)
            print(f"{setname}  {best} vs {opp:12s}: "
                  f"{w} wins / {t} ties / {l} losses  (of {both})")
        print()

    print("=" * 90)
    print("BEST CONFIGURATION THAT CARRIES THE PROVEN 2x BOUND")
    print("=" * 90)
    for setname in ("APS", "CHA"):
        sound = [x for x in rank(rows, setname)
                 if x["label"].startswith("pure.")
                 and x["label"].rsplit(".", 1)[1] in ("bracket", "both")]
        b = sound[0]
        print(f"{setname}: {b['label']:38s} total {b[setname]['total']:>7d}"
              f"   place {place(rows, setname, b['label'])}")

    print("\n" + "=" * 90)
    print("WHICH SIMILARITY, WHICH SCALE  (guard = step, both sets summed)")
    print("=" * 90)
    sims, scales = {}, {}
    for row in rows:
        if not row["label"].startswith("pure."):
            continue
        _, sim, scale, guard = row["label"].split(".")
        if guard != "step":
            continue
        tot = row["APS"]["total"] + row["CHA"]["total"]
        sims.setdefault(sim, []).append(tot)
        scales.setdefault(scale, []).append(tot)
    print("similarity (best over scales):")
    for sim, v in sorted(sims.items(), key=lambda kv: min(kv[1])):
        print(f"   {sim:12s} best {min(v):>7d}   worst {max(v):>7d}")
    print("scale (best over similarities):")
    for sc, v in sorted(scales.items(), key=lambda kv: min(kv[1])):
        print(f"   {sc:12s} best {min(v):>7d}   worst {max(v):>7d}")


if __name__ == "__main__":
    main()