baseline: the published sim-numerics artifact as it stands (490 tests, verify_article.py checks all 72 paper numbers against sweep_full.json)
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,275 @@
|
||||
"""Stage 7 - how many regimes are there?
|
||||
|
||||
On Chandrupatla the bracket scale wins where the three-point derived scale is
|
||||
the right choice on Alefeld-Potra-Shi. That could mean two regimes. It could
|
||||
equally mean more than two, in which case a rule that recognises them would
|
||||
win on problems we currently lose everywhere - not only on Chandrupatla.
|
||||
|
||||
The question is answered in three steps, and the first two need no new runs:
|
||||
the sweep already recorded the evaluation count of every configuration on
|
||||
every problem.
|
||||
|
||||
1. HOW MUCH IS ON THE TABLE. Compare our single fixed choice against an
|
||||
oracle that is handed the best scale for each problem separately. If the
|
||||
gap is small there is no regime worth finding and the line stops here.
|
||||
2. HOW MANY REGIMES. Count how often each scale is the per-problem winner,
|
||||
and by how much it wins. A scale that wins often but only by a hair is
|
||||
not a regime; one that wins rarely but by a landslide is.
|
||||
3. ARE THEY RECOGNISABLE. A regime is only useful if it can be told apart
|
||||
from the inside, from quantities the method can see before it commits -
|
||||
never from the answer. Step 3 reports what distinguishes them.
|
||||
|
||||
Nothing here selects a configuration for the paper. An oracle is an upper
|
||||
bound on what regime detection could buy, not a method.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
from scipy.optimize._tstutils import (_APS_TESTS_DICTS,
|
||||
_CHANDRUPATLA_TESTS_DICTS)
|
||||
|
||||
SCALES = ("start", "bracket", "derived", "derived3")
|
||||
SIMS = ("ratio", "sqrt", "cbrt", "rational", "ratio_soft", "exp_abs",
|
||||
"rat_sq", "exp_sq", "cosine")
|
||||
GUARD = "project" # the safeguard the paper proposes
|
||||
SIM = "ratio" # the similarity the paper proposes
|
||||
MAXFEV = 200
|
||||
|
||||
|
||||
def problem_facts():
|
||||
"""Properties of each problem that a method could in principle see, plus
|
||||
the bracket width. Nothing here uses the root."""
|
||||
out = {}
|
||||
for d in _APS_TESTS_DICTS:
|
||||
a, b = float(d["bracket"][0]), float(d["bracket"][1])
|
||||
args = d.get("args", ())
|
||||
f = d["f"]
|
||||
out[d["ID"]] = dict(set="APS", a=a, b=b, w=abs(b - a),
|
||||
fa=f(a, *args), fb=f(b, *args))
|
||||
for d in _CHANDRUPATLA_TESTS_DICTS:
|
||||
a, b = float(d["bracket"][0]), float(d["bracket"][1])
|
||||
f = d["f"]
|
||||
out[d["ID"]] = dict(set="CHA", a=a, b=b, w=abs(b - a),
|
||||
fa=f(a), fb=f(b))
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
rows = json.load(open("sweep_full.json", encoding="utf8"))
|
||||
by_label = {r["label"]: r for r in rows}
|
||||
facts = problem_facts()
|
||||
|
||||
def cost(sim, scale, pid):
|
||||
"""Evaluations on one problem, or the full budget if unsolved."""
|
||||
r = by_label[f"pure.{sim}.{scale}.{GUARD}"]
|
||||
d = r["APS"]["per"].get(pid) or r["CHA"]["per"].get(pid)
|
||||
return d[0] if d[1] else MAXFEV
|
||||
|
||||
pids = list(facts)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
print("=" * 78)
|
||||
print("1. HOW MUCH IS ON THE TABLE")
|
||||
print("=" * 78)
|
||||
print(f" similarity fixed at '{SIM}', safeguard '{GUARD}'; "
|
||||
f"only the scale varies\n")
|
||||
print(f"{'':26s}{'APS':>10s}{'CHA':>10s}{'both':>10s}")
|
||||
print("-" * 56)
|
||||
for scale in SCALES:
|
||||
tots = {"APS": 0, "CHA": 0}
|
||||
for pid in pids:
|
||||
tots[facts[pid]["set"]] += cost(SIM, scale, pid)
|
||||
mark = " <- ours" if scale == "derived3" else ""
|
||||
print(f"{'fixed: ' + scale:26s}{tots['APS']:>10d}{tots['CHA']:>10d}"
|
||||
f"{tots['APS'] + tots['CHA']:>10d}{mark}")
|
||||
orc = {"APS": 0, "CHA": 0}
|
||||
for pid in pids:
|
||||
orc[facts[pid]["set"]] += min(cost(SIM, s, pid) for s in SCALES)
|
||||
print(f"{'ORACLE over the 4 scales':26s}{orc['APS']:>10d}{orc['CHA']:>10d}"
|
||||
f"{orc['APS'] + orc['CHA']:>10d}")
|
||||
ours = sum(cost(SIM, "derived3", pid) for pid in pids)
|
||||
tot_orc = orc["APS"] + orc["CHA"]
|
||||
print(f"\n headroom if the regime were always recognised: "
|
||||
f"{ours - tot_orc} evaluations "
|
||||
f"({100.0 * (ours - tot_orc) / ours:.1f}% of ours)")
|
||||
|
||||
orc_all = 0
|
||||
for pid in pids:
|
||||
orc_all += min(cost(s, sc, pid) for s in SIMS for sc in SCALES)
|
||||
print(f" headroom if the SIMILARITY could also vary: "
|
||||
f"{ours - orc_all} evaluations "
|
||||
f"({100.0 * (ours - orc_all) / ours:.1f}%)")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
print()
|
||||
print("=" * 78)
|
||||
print("2. HOW MANY REGIMES (per-problem winner among the four scales)")
|
||||
print("=" * 78)
|
||||
wins = Counter()
|
||||
margins = defaultdict(list)
|
||||
detail = []
|
||||
for pid in pids:
|
||||
c = {s: cost(SIM, s, pid) for s in SCALES}
|
||||
best = min(c, key=lambda s: c[s])
|
||||
runner = min((v for s, v in c.items() if s != best), default=c[best])
|
||||
wins[(facts[pid]["set"], best)] += 1
|
||||
margins[best].append(c["derived3"] - c[best])
|
||||
detail.append((pid, facts[pid], c, best, c["derived3"] - c[best]))
|
||||
print(f"{'scale':12s}{'APS wins':>10s}{'CHA wins':>10s}"
|
||||
f"{'evals it saves over ours':>28s}")
|
||||
print("-" * 62)
|
||||
for s in SCALES:
|
||||
saved = sum(m for m in margins[s])
|
||||
print(f"{s:12s}{wins[('APS', s)]:>10d}{wins[('CHA', s)]:>10d}"
|
||||
f"{saved:>28d}")
|
||||
print("\n ties are credited to the first scale in the list order, so "
|
||||
"'start'\n is over-credited and the others under-credited; the "
|
||||
"saving column is\n the honest one.")
|
||||
|
||||
# where does the saving actually live?
|
||||
detail.sort(key=lambda t: t[4], reverse=True)
|
||||
print(f"\n the 12 problems with the most to gain:")
|
||||
print(f" {'problem':12s}{'set':>5s}{'width':>12s}"
|
||||
f"{'|fa/fb|':>12s}{'ours':>7s}{'best':>7s} best scale")
|
||||
print(" " + "-" * 72)
|
||||
for pid, fct, c, best, gain in detail[:12]:
|
||||
fa, fb = abs(fct["fa"]), abs(fct["fb"])
|
||||
ratio = fa / fb if fb else float("inf")
|
||||
print(f" {pid:12s}{fct['set']:>5s}{fct['w']:>12.3g}"
|
||||
f"{ratio:>12.3g}{c['derived3']:>7d}{c[best]:>7d} {best}")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
print()
|
||||
print("=" * 78)
|
||||
print("3. ARE THE REGIMES RECOGNISABLE FROM THE INSIDE?")
|
||||
print("=" * 78)
|
||||
print(" Two quantities a method can see before it commits: the bracket")
|
||||
print(" width, and how lopsided the two end values are. If the winners")
|
||||
print(" separate along either, a regime test exists; if they overlap,")
|
||||
print(" there is nothing to detect and the line closes.\n")
|
||||
# NB the column below is min/max, not fa/fb as printed in section 2:
|
||||
# min/max is symmetric and lives in [0,1], which is what a separation
|
||||
# test would use. Labelled accordingly so the two are not compared.
|
||||
print(f" {'best scale':12s}{'n':>5s}{'width: median':>16s}"
|
||||
f"{'min|f|/max|f|: med':>20s}{'log10 range of width':>24s}")
|
||||
print(" " + "-" * 74)
|
||||
for s in SCALES:
|
||||
grp = [d for d in detail if d[3] == s]
|
||||
if not grp:
|
||||
continue
|
||||
ws = sorted(g[1]["w"] for g in grp)
|
||||
rs = sorted(min(abs(g[1]["fa"]), abs(g[1]["fb"])) /
|
||||
max(abs(g[1]["fa"]), abs(g[1]["fb"]), 1e-300)
|
||||
for g in grp)
|
||||
lo = math.log10(ws[0]) if ws[0] > 0 else float("-inf")
|
||||
hi = math.log10(ws[-1]) if ws[-1] > 0 else float("-inf")
|
||||
print(f" {s:12s}{len(grp):>5d}{ws[len(ws) // 2]:>16.3g}"
|
||||
f"{rs[len(rs) // 2]:>20.3g}{f'{lo:.1f} .. {hi:.1f}':>24s}")
|
||||
print("\n Overlapping ranges mean the winners are NOT separated by these")
|
||||
print(" quantities, and a regime test built on them would be a guess.")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 4. The scale axis is worth 5.3%; the similarity axis is worth 25.9%.
|
||||
# So the regimes, if any, live in the similarity - which is the
|
||||
# opposite of where the Chandrupatla observation pointed.
|
||||
#
|
||||
# An oracle over nine similarities on 199 problems will find a gain
|
||||
# by luck alone, so the gain is not the evidence. The evidence is
|
||||
# STABILITY: the APS problems come in families (aps.NN.xx are the same
|
||||
# function at different parameters). If one similarity wins across a
|
||||
# whole family, that is structure a rule could find. If the winner
|
||||
# jumps about inside a family, the oracle is fitting noise and the
|
||||
# line closes.
|
||||
# ------------------------------------------------------------------
|
||||
print()
|
||||
print("=" * 78)
|
||||
print("4. THE SIMILARITY AXIS (scale fixed at derived3)")
|
||||
print("=" * 78)
|
||||
swins = Counter()
|
||||
sgain = defaultdict(int)
|
||||
per_pid_best = {}
|
||||
for pid in pids:
|
||||
c = {s: cost(s, "derived3", pid) for s in SIMS}
|
||||
best = min(c, key=lambda s: c[s])
|
||||
per_pid_best[pid] = best
|
||||
swins[best] += 1
|
||||
sgain[best] += c[SIM] - c[best]
|
||||
print(f"{'similarity':14s}{'problems won':>14s}{'evals saved over ratio':>26s}")
|
||||
print("-" * 56)
|
||||
for s, n in swins.most_common():
|
||||
print(f"{s:14s}{n:>14d}{sgain[s]:>26d}")
|
||||
|
||||
print("\n STABILITY WITHIN A FAMILY (aps.NN.* is one function, "
|
||||
"many parameters)")
|
||||
fam = defaultdict(list)
|
||||
for pid in pids:
|
||||
if pid.startswith("aps."):
|
||||
fam[pid.split(".")[1]].append(per_pid_best[pid])
|
||||
consistent = mixed = 0
|
||||
print(f" {'family':10s}{'members':>9s}{'distinct winners':>19s} winners")
|
||||
print(" " + "-" * 68)
|
||||
for k in sorted(fam):
|
||||
v = fam[k]
|
||||
if len(v) < 3:
|
||||
continue
|
||||
distinct = Counter(v)
|
||||
if len(distinct) == 1:
|
||||
consistent += 1
|
||||
else:
|
||||
mixed += 1
|
||||
top = ", ".join(f"{s}x{n}" for s, n in distinct.most_common(3))
|
||||
print(f" aps.{k:<7s}{len(v):>9d}{len(distinct):>19d} {top}")
|
||||
print(f"\n families with a single winner: {consistent} "
|
||||
f"families with a mixed winner: {mixed}")
|
||||
print(" A mixed family means the same function at a different parameter")
|
||||
print(" wants a different similarity - which is not a regime but noise,")
|
||||
print(" and no rule reading the problem could recover it.")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 5. Two suspicions to settle before the line is closed.
|
||||
# (a) cbrt wins only 15 problems but saves 678 evaluations, while
|
||||
# being one of the worst similarities overall. A large saving
|
||||
# from a bad method on few problems is the signature of a lucky
|
||||
# escape on problems where everything flails, not of a regime.
|
||||
# (b) the scale winner DID look family-consistent: every bracket-scale
|
||||
# win in the top twelve was an aps.15.* problem. If one family
|
||||
# accounts for the scale headroom, the regime is real but tiny.
|
||||
# ------------------------------------------------------------------
|
||||
print()
|
||||
print("=" * 78)
|
||||
print("5. SETTLING THE TWO SUSPICIONS")
|
||||
print("=" * 78)
|
||||
print(" (a) where cbrt's saving comes from")
|
||||
print(f" {'problem':12s}{'ours (ratio)':>14s}{'cbrt':>8s}{'saved':>8s}")
|
||||
print(" " + "-" * 44)
|
||||
cb = sorted(((pid, cost(SIM, "derived3", pid), cost("cbrt", "derived3", pid))
|
||||
for pid in pids if per_pid_best[pid] == "cbrt"),
|
||||
key=lambda t: t[1] - t[2], reverse=True)
|
||||
for pid, ours_c, cb_c in cb[:8]:
|
||||
print(f" {pid:12s}{ours_c:>14d}{cb_c:>8d}{ours_c - cb_c:>8d}")
|
||||
near_cap = sum(1 for _p, o, _c in cb if o >= 100)
|
||||
print(f"\n of the {len(cb)} problems cbrt wins, {near_cap} are ones "
|
||||
f"where our own run costs\n 100+ evaluations - i.e. problems on "
|
||||
f"which every method is struggling.")
|
||||
|
||||
print("\n (b) is the scale headroom one family?")
|
||||
fam_gain = defaultdict(int)
|
||||
for pid, fct, c, best, gain in detail:
|
||||
key = pid.rsplit(".", 1)[0] if pid.startswith("aps.") else "CHA"
|
||||
fam_gain[key] += gain
|
||||
tot = sum(fam_gain.values())
|
||||
print(f" {'family':12s}{'evals gained by the oracle':>28s}{'share':>9s}")
|
||||
print(" " + "-" * 50)
|
||||
for k, v in sorted(fam_gain.items(), key=lambda kv: kv[1], reverse=True)[:6]:
|
||||
if v == 0:
|
||||
continue
|
||||
print(f" {k:12s}{v:>28d}{100.0 * v / tot:>8.0f}%")
|
||||
print(f"\n total scale headroom {tot} evaluations out of {ours} "
|
||||
f"({100.0 * tot / ours:.1f}%)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user