baseline: the published sim-numerics artifact as it stands (490 tests, verify_article.py checks all 72 paper numbers against sweep_full.json)

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-20 22:38:17 +01:00
commit 4a23aacf96
75 changed files with 12175 additions and 0 deletions
@@ -0,0 +1,77 @@
+"""A prediction registered before the run, and falsified by it.
+
+The hypothesis was that a method which wins in some places and loses in others
+must have a regime where it wins outright, and that the regime is where the
+quantity the constant approximates CHANGES during the run - because one
+constant cannot be right at two different noise levels while a measured value
+tracks.
+
+It was stated before measuring, with the falsification condition attached: the
+measured rule had to lose every constant-noise cell and win every
+varying-noise cell.  It lost all twelve.
+
+The first version of the test could not have confirmed it either way - noise
+reduction was scored on the second half of the run, by which time every ramp
+and step had already arrived, so the metric never saw a transition.  That flaw
+was found and fixed before the conclusion was drawn, and the conclusion did
+not change.
+
+This file exists so the line is not reopened without new evidence.
+"""
+from __future__ import annotations
+
+import math
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from anc.rules import Fixed, Measured
+from stage20_prediction import (FLAT_5, FLAT_30, STEP_25, SWING_25, sig_cabin,
+                                sig_tonal, score)
+
+SEEDS = 3
+
+
+def test_the_metric_window_sees_the_transition():
+    """The flaw that would have produced a false falsification.  A varying
+    profile must not score the same as a constant one at its final level."""
+    varying = score(Fixed(0.1), sig_tonal, SWING_25, seeds=SEEDS)
+    final_level = score(Fixed(0.1), sig_tonal, FLAT_5, seeds=SEEDS)
+    assert varying > final_level + 2.0, (varying, final_level)
+
+
+@pytest.mark.parametrize("signal", [sig_tonal, sig_cabin],
+                         ids=["tonal", "cabin"])
+@pytest.mark.parametrize("profile,kind", [(FLAT_30, "control"),
+                                          (FLAT_5, "control"),
+                                          (SWING_25, "prediction"),
+                                          (STEP_25, "prediction")],
+                         ids=["flat30", "flat5", "ramp", "step"])
+def test_an_oracle_constant_beats_the_measured_rule_everywhere(signal, profile,
+                                                               kind):
+    """The registered prediction required the measured rule to WIN the two
+    'prediction' cells.  It does not, on either signal.  A single constant,
+    chosen with knowledge of the scenario, is ahead even when the noise level
+    swings by 25 dB inside the run."""
+    ms = score(Measured(), signal, profile, seeds=SEEDS)
+    best = max(score(Fixed(mu), signal, profile, seeds=SEEDS)
+               for mu in (0.05, 0.1, 0.2, 0.5, 1.0))
+    assert best > ms, (kind, best, ms)
+
+
+def test_the_conclusion_this_licenses():
+    """Winning somewhere and losing elsewhere does not imply a regime of
+    outright victory exists; it is equally consistent with a middling
+    generalist.  On this evidence there is no such regime, and the claim the
+    work may make is the scoped one in test_modern_repairs.py."""
+    losses = 0
+    for signal in (sig_tonal, sig_cabin):
+        for profile in (FLAT_30, FLAT_5, SWING_25, STEP_25):
+            ms = score(Measured(), signal, profile, seeds=SEEDS)
+            best = max(score(Fixed(mu), signal, profile, seeds=SEEDS)
+                       for mu in (0.05, 0.2, 1.0))
+            losses += ms < best
+    assert losses == 8, losses