diff --git a/audit_similarity_places.py b/audit_similarity_places.py new file mode 100644 index 0000000..d1f9633 --- /dev/null +++ b/audit_similarity_places.py @@ -0,0 +1,100 @@ +"""Where is the similarity function S actually evaluated, and does it earn the gains? + +Valeri, 2026-09-21: "check again in which places and which similarity functions are +used, because that is what gave the gain". + +Static reading of simroot/pure.py: S is EVALUATED only in the step length +(1 - S(z))*(far - x), in the 'simpair' decision S(|phi-xi|) < S(z), and in two refuted +modes (log space, progress gate). The readings behind the records - the slope-sign gate, +the triple chosen by shape, the far end by dissimilarity, the 'defect' decision - compare +quantities directly and never call S. + +The ablation, read off the existing Linux sweeps (no new run): for each reading and for +each of the nine functions S used in the step, the gain of the reading is + total(reading on) - total(reading off) +with the same S, scale and safeguard. If the gain is the same whatever S is, S does not +produce it; if it changes with S, S takes part. + + python -u audit_similarity_places.py +""" +from __future__ import annotations + +import json + +SIMS = ("ratio", "sqrt", "cbrt", "rational", "ratio_soft", "exp_abs", "rat_sq", "exp_sq", + "cosine") +NAME = {"ratio": "tent 1-|z|", "sqrt": "1-sqrt|z|", "cbrt": "1-|z|^1/3", + "rational": "1/(1+|z|)", "ratio_soft": "(1+|z|)^-2", "exp_abs": "e^-|z|", + "rat_sq": "1/(1+z^2)", "exp_sq": "e^-z^2", "cosine": "1/sqrt(1+z^2)"} + + +def load(path): + with open(path, encoding="utf8") as fh: + return json.load(fh) + + +def key(r): + s = r["spec"] + pad = list(s) + [None] * (9 - len(s)) + return (pad[1], pad[2], pad[3], pad[5] or "none", pad[6] or "recent", + pad[7] or "distance") + + +def index(*paths): + out = {} + for p in paths: + for r in load(p): + if r["spec"][0] == "pure": + out.setdefault(key(r), r) + return out + + +def both(r): + return r["APS"]["total"] + r["CHA"]["total"] + + +def main(): + cells = index("sweep_full.json", "sweep_slope.json", "sweep_points.json", + "sweep_direction.json", "sweep_decision.json") + # reading, its 'on' cell, its 'off' cell (same S, scale, guard), guard as in the record + readings = ( + ("slope-sign gate", ("slope", "recent", "distance"), ("none", "recent", "distance"), + "project"), + ("triple by shape", ("slope", "shape", "distance"), ("slope", "recent", "distance"), + "safe"), + ("far end by dissimilarity", ("slope", "recent", "dissimilar"), + ("slope", "recent", "distance"), "project"), + ("decision 'defect'", ("defect", "recent", "distance"), ("none", "recent", "distance"), + "project"), + ("decision 'simpair' (uses S)", ("simpair", "recent", "distance"), + ("none", "recent", "distance"), "project"), + ) + print("Gain of each reading (APS | CHA), by the similarity S used in the STEP." + " Negative = cheaper.\n") + header = f"{'S in the step':<16}" + "".join(f"{r[0]:>30}" for r in readings) + print(header) + spread = {} + for sim in SIMS: + row = f"{NAME[sim]:<16}" + for name, on, off, gu in readings: + a = cells.get((sim, "derived3", gu) + on) + b = cells.get((sim, "derived3", gu) + off) + if a is None or b is None: + row += f"{'—':>30}" + continue + da = a["APS"]["total"] - b["APS"]["total"] + dc = a["CHA"]["total"] - b["CHA"]["total"] + spread.setdefault(name, []).append((sim, da, dc)) + row += f"{da:>+14d} |{dc:>+7d} " + print(row) + print("\nDoes the gain depend on S? Range of the gain over the nine functions:") + for name, rows in spread.items(): + das = [r[1] for r in rows] + dcs = [r[2] for r in rows] + print(f" {name:<30} APS {min(das):+6d} .. {max(das):+6d} " + f"CHA {min(dcs):+6d} .. {max(dcs):+6d} (over {len(rows)} functions)") + return spread + + +if __name__ == "__main__": + main() diff --git a/diag_contraction_sequence.py b/diag_contraction_sequence.py new file mode 100644 index 0000000..371f3c4 --- /dev/null +++ b/diag_contraction_sequence.py @@ -0,0 +1,62 @@ +"""Why the contraction-ratio similarity came out inverted: the rho sequence itself. + +diag_contraction_similarity.py falsified "successive contraction ratios are similar on +the flat families": they are LESS similar there than on the canonical set. The picture +behind the hypothesis - rho constant at about (1 - 1/m)^m - holds for pure Newton steps on +x^m. The explanation to check is that our safeguarded run is not pure: the projection +interleaves forced moves, so rho alternates. Printed per step for one flat problem and +one canonical problem: rho_n, and whether the step was the similarity step as proposed or +was changed by the safeguard (x_new != x + raw). + + python -u diag_contraction_sequence.py +""" +from __future__ import annotations + +from scipy.optimize._tstutils import _APS_TESTS_DICTS, _CHANDRUPATLA_TESTS_DICTS + +from simroot.pure import pure_root + +CFG = dict(similarity="ratio", scale="derived3", guard="project") + + +def show(name, f, a, b): + evals = [] + + def g(t): + v = f(t) + evals.append((t, v)) + return v + st = {"trace": []} + pure_root(g, a, b, 1e-12, 200, stats=st, **CFG) + # map each traced state to the evaluation that followed it + order = [e[0] for e in evals] + print(f"\n== {name}: {len(evals)} evaluations") + print(f" {'k':>3} {'rho_n=|f_n/f_(n-1)|':>22} {'step as proposed?':>18}") + for k, s in enumerate(st["trace"][:40]): + proposed = s["x"] + s["raw"] + try: + i = order.index(s["x"]) + except ValueError: + continue + if i + 1 >= len(evals): + break + x_next, f_next = evals[i + 1] + rho = abs(f_next / s["fx"]) if s["fx"] != 0.0 else float("nan") + kept = "yes" if x_next == proposed else "CHANGED" + print(f" {k:3d} {rho:22.3e} {kept:>18}") + + +def main(): + cha = {str(d["ID"]): d for d in _CHANDRUPATLA_TESTS_DICTS} + d = cha["fun5.1"] + show("fun5.1 = x^9 (flat)", d["f"], *(float(v) for v in d["bracket"])) + d = cha["fun4.3"] + show("fun4.3 = 6(x-2)^5 on a wide bracket (flat)", d["f"], + *(float(v) for v in d["bracket"])) + d = _APS_TESTS_DICTS[10] + f = (lambda ff, aa: lambda t: ff(t, *aa))(d["f"], d.get("args", ())) + show(f"{d['ID']} (canonical)", f, *(float(v) for v in d["bracket"])) + + +if __name__ == "__main__": + main() diff --git a/diag_contraction_similarity.py b/diag_contraction_similarity.py new file mode 100644 index 0000000..52d77b1 --- /dev/null +++ b/diag_contraction_similarity.py @@ -0,0 +1,106 @@ +"""Is the flat regime DETECTABLE by the similarity of two successive contraction ratios? + +Valeri, 2026-09-21: "similarity should solve exactly this problem". The similarity of a +triple to a linear configuration, |phi - xi|, cannot: fun1 on a wide bracket and the +pure powers x^9, x^19 look equally non-linear, yet a quadratic model works on the first +and fails on the second. The flat regime shows in the HISTORY instead: with +rho_n = |f_n / f_{n-1}|, a superlinear run has rho falling from step to step (successive +ratios dissimilar), a linearly converging one has rho nearly constant (similar). So the +candidate similarity for this place is + + S = min(rho_n, rho_{n-1}) / max(rho_n, rho_{n-1}) (the SIM ratio form) + +Declared before the run, with no threshold anywhere: + * AUC of S, flat states (fun3-fun6) against each other group: the probability that a + random flat state has the higher S. 0.5 = no separation, 1.0 = perfect. + * the threshold-free rule "the contraction did not speed up", rho_n >= rho_{n-1}: how + often it fires in each group. The bar to beat is the three-point order estimate, + m >= 2: 0.2% of canonical states, 5.8% of flat ones (diag_flat_detectable.py). +Nothing in the method changes; states come from the behaviour-neutral trace hook. + + python -u diag_contraction_similarity.py +""" +from __future__ import annotations + +from scipy.optimize._tstutils import _APS_TESTS_DICTS, _CHANDRUPATLA_TESTS_DICTS + +from simroot.pure import pure_root + +XTOL, MAXFEV, ACC = 1e-12, 200, 1e-6 +CONFIGS = { + "proposed (projection)": dict(similarity="ratio", scale="derived3", guard="project"), + "reading (3), as for m>=2": dict(similarity="ratio", scale="derived3", guard="safe", + prestep="slope", direction="dissimilar"), +} +FLAT = ("fun3", "fun4", "fun5", "fun6") + + +def states(f, a, b, r, cfg): + """(S, contraction_did_not_speed_up) for every traced state with three nonzero f.""" + st = {"trace": []} + x, _n, conv = pure_root(f, a, b, XTOL, MAXFEV, stats=st, **cfg) + if not (conv and abs(float(x) - r) <= ACC): + return [] + out = [] + for s in st["trace"]: + if s["xp2"] is None or 0.0 in (s["fx"], s["fp"], s["fp2"]): + continue + rho_n = abs(s["fx"] / s["fp"]) + rho_p = abs(s["fp"] / s["fp2"]) + hi, lo = max(rho_n, rho_p), min(rho_n, rho_p) + if hi == 0.0 or hi != hi: + continue + out.append((lo / hi, rho_n >= rho_p)) + return out + + +def auc(pos, neg): + """P(random pos > random neg) + 0.5 P(tie), by ranks (Mann-Whitney).""" + if not pos or not neg: + return float("nan") + allv = sorted([(v, 1) for v in pos] + [(v, 0) for v in neg]) + rank_sum, i = 0.0, 0 + while i < len(allv): + j = i + while j < len(allv) and allv[j][0] == allv[i][0]: + j += 1 + mean_rank = (i + 1 + j) / 2.0 + rank_sum += mean_rank * sum(1 for k in range(i, j) if allv[k][1] == 1) + i = j + n1, n0 = len(pos), len(neg) + return (rank_sum - n1 * (n1 + 1) / 2.0) / (n1 * n0) + + +def main(): + out = {} + for cname, cfg in CONFIGS.items(): + groups = {"APS": [], "CHA fun1": [], "CHA flat fun3-6": [], "CHA other": []} + for d in _APS_TESTS_DICTS: + a, b = (float(v) for v in d["bracket"]) + f = (lambda ff, aa: lambda t: ff(t, *aa))(d["f"], d.get("args", ())) + groups["APS"] += states(f, a, b, float(d["root"]), cfg) + for d in _CHANDRUPATLA_TESTS_DICTS: + a, b = (float(v) for v in d["bracket"]) + fam = str(d["ID"]).split(".")[0] + g = ("CHA flat fun3-6" if fam in FLAT else "CHA fun1" if fam == "fun1" + else "CHA other") + groups[g] += states(d["f"], a, b, float(d["root"]), cfg) + print(f"\n== {cname}") + print(f" {'group':<18} {'states':>7} {'rho_n >= rho_(n-1)':>20}") + for g, v in groups.items(): + n = max(len(v), 1) + print(f" {g:<18} {len(v):7d} {100.0 * sum(x[1] for x in v) / n:19.1f}%") + flat = [x[0] for x in groups["CHA flat fun3-6"]] + print(" AUC of S, flat states against:") + res = {} + for g in ("APS", "CHA fun1", "CHA other"): + res[g] = auc(flat, [x[0] for x in groups[g]]) + print(f" {g:<14} {res[g]:.3f}") + out[cname] = {"groups": {g: (len(v), sum(x[1] for x in v)) for g, v in groups.items()}, + "auc": res} + print("\nbar to beat (three-point order estimate, m >= 2): 0.2% canonical, 5.8% flat") + return out + + +if __name__ == "__main__": + main() diff --git a/results_audit_similarity_places.txt b/results_audit_similarity_places.txt new file mode 100644 index 0000000..5d8bfe5 --- /dev/null +++ b/results_audit_similarity_places.txt @@ -0,0 +1,19 @@ +Gain of each reading (APS | CHA), by the similarity S used in the STEP. Negative = cheaper. + +S in the step slope-sign gate triple by shape far end by dissimilarity decision 'defect' decision 'simpair' (uses S) +tent 1-|z| +42 | -146 -493 | +205 -122 | -289 -283 | -10 +2515 | -310 +1-sqrt|z| +16 | +167 -47 | +9 -186 | -43 +195 | -27 +18 | -199 +1-|z|^1/3 +3 | +186 -36 | +139 -776 | -368 +215 | +12 +208 | -50 +1/(1+|z|) +165 | -87 +126 | +358 -118 | -327 +8 | -8 +3825 | -432 +(1+|z|)^-2 +29 | -158 -255 | +182 +29 | -217 +204 | -111 +49 | -670 +e^-|z| +64 | -203 -227 | +278 +30 | -382 -2 | -2 +3194 | -503 +1/(1+z^2) +2 | +1 +11 | +273 -9 | -223 +14 | -172 +18 | -853 +e^-z^2 +1 | -3 -224 | +303 -8 | -214 +14 | -179 +10 | -790 +1/sqrt(1+z^2) +0 | +14 +250 | +186 -12 | -211 -10 | -171 +4 | -844 + +Does the gain depend on S? Range of the gain over the nine functions: + slope-sign gate APS +0 .. +165 CHA -203 .. +186 (over 9 functions) + triple by shape APS -493 .. +250 CHA +9 .. +358 (over 9 functions) + far end by dissimilarity APS -776 .. +30 CHA -382 .. -43 (over 9 functions) + decision 'defect' APS -283 .. +215 CHA -179 .. +12 (over 9 functions) + decision 'simpair' (uses S) APS +4 .. +3825 CHA -853 .. -50 (over 9 functions) diff --git a/results_diag_contraction_sequence.txt b/results_diag_contraction_sequence.txt new file mode 100644 index 0000000..23e4aa1 --- /dev/null +++ b/results_diag_contraction_sequence.txt @@ -0,0 +1,99 @@ + +== fun5.1 = x^9 (flat): 88 evaluations + k rho_n=|f_n/f_(n-1)| step as proposed? + 0 2.621e+05 CHANGED + 1 3.550e-01 yes + 2 2.604e-01 yes + 3 3.585e-05 yes + 4 9.999e-01 yes + 5 3.470e-01 yes + 6 2.550e-01 yes + 7 6.018e+02 yes + 8 5.407e-07 yes + 9 1.000e+00 yes + 10 3.223e-01 yes + 11 2.359e-01 yes + 12 6.252e-06 yes + 13 1.000e+00 yes + 14 3.448e-01 yes + 15 2.535e-01 yes + 16 2.491e+05 yes + 17 1.122e-04 yes + 18 9.964e-01 yes + 19 2.933e-01 yes + 20 2.064e-01 yes + 21 1.091e-01 yes + 22 7.494e-01 yes + 23 2.743e-01 yes + 24 1.228e-01 yes + 25 6.374e-01 yes + 26 2.198e-01 yes + 27 1.111e-03 yes + 28 9.981e-01 yes + 29 3.403e-01 yes + 30 2.500e-01 yes + 31 6.771e+03 CHANGED + 32 1.632e-05 yes + 33 9.993e-01 yes + 34 3.077e-01 yes + 35 2.223e-01 yes + 36 1.674e-02 yes + 37 9.719e-01 yes + 38 3.284e-01 yes + 39 2.372e-01 yes + +== fun4.3 = 6(x-2)^5 on a wide bracket (flat): 113 evaluations + k rho_n=|f_n/f_(n-1)| step as proposed? + 0 3.280e-16 yes + 1 1.000e+00 yes + 2 3.276e-01 yes + 3 2.574e-01 yes + 4 2.487e-02 yes + 5 9.091e-01 yes + 6 2.960e-01 yes + 7 2.287e-01 yes + 8 1.209e-07 yes + 9 1.000e+00 yes + 10 3.270e-01 yes + 11 2.571e-01 yes + 12 2.414e-02 yes + 13 9.119e-01 yes + 14 2.966e-01 yes + 15 2.295e-01 yes + 16 1.122e-06 yes + 17 1.000e+00 yes + 18 3.265e-01 yes + 19 2.568e-01 yes + 20 2.363e-02 yes + 21 9.139e-01 yes + 22 2.971e-01 yes + 23 2.300e-01 yes + 24 3.255e-06 yes + 25 1.000e+00 yes + 26 3.262e-01 yes + 27 2.567e-01 yes + 28 2.327e-02 yes + 29 9.153e-01 yes + 30 2.974e-01 yes + 31 2.304e-01 yes + 32 5.987e-06 yes + 33 1.000e+00 yes + 34 3.264e-01 yes + 35 2.568e-01 yes + 36 2.347e-02 yes + 37 9.145e-01 yes + 38 2.972e-01 yes + 39 2.302e-01 yes + +== aps.02.09 (canonical): 12 evaluations + k rho_n=|f_n/f_(n-1)| step as proposed? + 0 1.284e+00 CHANGED + 1 1.219e+01 CHANGED + 2 2.208e-01 CHANGED + 3 3.952e-01 yes + 4 4.398e-01 yes + 5 9.225e-02 yes + 6 1.079e-02 yes + 7 4.440e-04 yes + 8 4.563e-07 yes + 9 4.333e-02 yes diff --git a/results_diag_contraction_similarity.txt b/results_diag_contraction_similarity.txt new file mode 100644 index 0000000..07d87ac --- /dev/null +++ b/results_diag_contraction_similarity.txt @@ -0,0 +1,24 @@ + +== proposed (projection) + group states rho_n >= rho_(n-1) + APS 1429 37.6% + CHA fun1 83 28.9% + CHA flat fun3-6 1763 28.2% + CHA other 237 25.3% + AUC of S, flat states against: + APS 0.422 + CHA fun1 0.409 + CHA other 0.386 + +== reading (3), as for m>=2 + group states rho_n >= rho_(n-1) + APS 990 27.5% + CHA fun1 36 27.8% + CHA flat fun3-6 659 31.3% + CHA other 204 16.7% + AUC of S, flat states against: + APS 0.284 + CHA fun1 0.345 + CHA other 0.229 + +bar to beat (three-point order estimate, m >= 2): 0.2% canonical, 5.8% flat