166 lines
6.5 KiB
Python
166 lines
6.5 KiB
Python
"""Read sweep_full.json and answer the questions the paper needs answered.
|
|
|
|
Everything printed here comes from the single server run; no number is
|
|
recomputed on a different machine. (The test functions use libm, whose last
|
|
digit differs between Windows and Linux, so mixing platforms would move
|
|
evaluation counts by a unit or two - measured: scipy's toms748 scores 2819 on
|
|
Windows and 2820 on Linux for the same set.)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
|
|
BASE = {"bisection", "regula_falsi", "illinois", "pegasus", "anderson_bjorck",
|
|
"ridders", "brent", "toms748", "itp", "itp_oracle"}
|
|
|
|
# The configuration the paper proposes. It is not the winner of a search over
|
|
# the grid - it is what the theory of section 2 dictates, fixed before the
|
|
# ranking is looked at:
|
|
# similarity a nonzero slope of (1 - S) at 0 -> the ratio form 1 - |z|
|
|
# scale the finest data-derived one -> the three-point difference
|
|
# safeguard the one that carries a bound without destroying the order
|
|
# AND without a free constant -> the credit projection, whose
|
|
# endowment is a licence of one unconstrained step rather than
|
|
# a number (see simroot.pure.project_credit)
|
|
# The grid is evidence for the theory, not the method's selection procedure.
|
|
PROPOSED = "pure.ratio.derived3.credit"
|
|
TUNED = {"illinois", "pegasus", "anderson_bjorck", "brent", "toms748",
|
|
"itp", "itp_oracle"}
|
|
|
|
|
|
def load(path="sweep_full.json"):
|
|
with open(path, encoding="utf8") as fh:
|
|
return json.load(fh)
|
|
|
|
|
|
def rank(rows, setname):
|
|
return sorted(rows, key=lambda r: (r[setname]["n"] - r[setname]["solved"],
|
|
r[setname]["total"]))
|
|
|
|
|
|
def place(rows, setname, lbl):
|
|
for i, r in enumerate(rank(rows, setname), 1):
|
|
if r["label"] == lbl:
|
|
return i
|
|
raise KeyError(f"no configuration labelled {lbl!r} in the sweep; "
|
|
"a mistyped name must not be reported as a missing place")
|
|
|
|
|
|
def best_pure(rows, setname):
|
|
"""The best configuration OF OURS. The overall winner of a set may be a
|
|
baseline - itp_oracle tops Chandrupatla - and comparing that against
|
|
itself prints nothing of interest."""
|
|
for r in rank(rows, setname):
|
|
if r["label"].startswith("pure."):
|
|
return r["label"]
|
|
raise KeyError("no pure configuration in the sweep")
|
|
|
|
|
|
def head_to_head(rows, a, b, setname):
|
|
ra = next(r for r in rows if r["label"] == a)[setname]
|
|
rb = next(r for r in rows if r["label"] == b)[setname]
|
|
win = tie = loss = both = 0
|
|
for pid, (na, oka) in ra["per"].items():
|
|
nb, okb = rb["per"][pid]
|
|
if not (oka and okb):
|
|
continue
|
|
both += 1
|
|
if na < nb:
|
|
win += 1
|
|
elif na > nb:
|
|
loss += 1
|
|
else:
|
|
tie += 1
|
|
return win, tie, loss, both
|
|
|
|
|
|
def main():
|
|
rows = load(sys.argv[1] if len(sys.argv) > 1 else "sweep_full.json")
|
|
print(f"{len(rows)} configurations\n")
|
|
|
|
for setname in ("APS", "CHA"):
|
|
r = rank(rows, setname)
|
|
n = r[0][setname]["n"]
|
|
print("=" * 90)
|
|
print(f"{setname} ({n} problems) top 12, with the best baseline "
|
|
f"positions marked")
|
|
print("=" * 90)
|
|
print(f"{'#':>3s} {'configuration':38s}{'tuning':>8s}{'solved':>9s}"
|
|
f"{'total':>9s}{'over 2x':>9s}")
|
|
print("-" * 90)
|
|
for i, row in enumerate(r[:12], 1):
|
|
d = row[setname]
|
|
tuned = "yes" if row["label"] in TUNED else "no"
|
|
star = " <-" if row["label"] not in BASE else ""
|
|
print(f"{i:>3d} {row['label']:38s}{tuned:>8s}"
|
|
f"{d['solved']:>6d}/{d['n']:<3d}{d['total']:>9d}"
|
|
f"{d['over']:>9d}{star}")
|
|
print()
|
|
for b in ("brent", "toms748", "itp", "itp_oracle", "bisection"):
|
|
p = place(rows, setname, b)
|
|
d = next(x for x in rows if x["label"] == b)[setname]
|
|
print(f" {b:38s}place {p:>3d} total {d['total']}")
|
|
print()
|
|
|
|
print("=" * 90)
|
|
print(f"THE PROPOSED CONFIGURATION {PROPOSED}")
|
|
print(" (dictated by the theory, not selected from the ranking)")
|
|
print("=" * 90)
|
|
for setname in ("APS", "CHA"):
|
|
d = next(r for r in rows if r["label"] == PROPOSED)[setname]
|
|
print(f" {setname}: place {place(rows, setname, PROPOSED):>3d} of "
|
|
f"{len(rows)} solved {d['solved']}/{d['n']} "
|
|
f"total {d['total']} over 2x bound {d['over']}")
|
|
for opp in ("brent", "toms748", "itp", "itp_oracle", "bisection"):
|
|
w, t, l, both = head_to_head(rows, PROPOSED, opp, setname)
|
|
print(f" vs {opp:12s}: {w} wins / {t} ties / {l} losses "
|
|
f"(of {both})")
|
|
print()
|
|
|
|
print("=" * 90)
|
|
print("HEAD TO HEAD, best pure configuration of each set")
|
|
print("=" * 90)
|
|
for setname in ("APS", "CHA"):
|
|
best = best_pure(rows, setname)
|
|
for opp in ("brent", "toms748", "itp", "itp_oracle"):
|
|
w, t, l, both = head_to_head(rows, best, opp, setname)
|
|
print(f"{setname} {best} vs {opp:12s}: "
|
|
f"{w} wins / {t} ties / {l} losses (of {both})")
|
|
print()
|
|
|
|
print("=" * 90)
|
|
print("BEST CONFIGURATION THAT CARRIES THE PROVEN 2x BOUND")
|
|
print("=" * 90)
|
|
for setname in ("APS", "CHA"):
|
|
sound = [x for x in rank(rows, setname)
|
|
if x["label"].startswith("pure.")
|
|
and x["label"].rsplit(".", 1)[1] in ("bracket", "both")]
|
|
b = sound[0]
|
|
print(f"{setname}: {b['label']:38s} total {b[setname]['total']:>7d}"
|
|
f" place {place(rows, setname, b['label'])}")
|
|
|
|
print("\n" + "=" * 90)
|
|
print("WHICH SIMILARITY, WHICH SCALE (guard = step, both sets summed)")
|
|
print("=" * 90)
|
|
sims, scales = {}, {}
|
|
for row in rows:
|
|
if not row["label"].startswith("pure."):
|
|
continue
|
|
_, sim, scale, guard = row["label"].split(".")
|
|
if guard != "step":
|
|
continue
|
|
tot = row["APS"]["total"] + row["CHA"]["total"]
|
|
sims.setdefault(sim, []).append(tot)
|
|
scales.setdefault(scale, []).append(tot)
|
|
print("similarity (best over scales):")
|
|
for sim, v in sorted(sims.items(), key=lambda kv: min(kv[1])):
|
|
print(f" {sim:12s} best {min(v):>7d} worst {max(v):>7d}")
|
|
print("scale (best over similarities):")
|
|
for sc, v in sorted(scales.items(), key=lambda kv: min(kv[1])):
|
|
print(f" {sc:12s} best {min(v):>7d} worst {max(v):>7d}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|