78 lines
3.4 KiB
Python
78 lines
3.4 KiB
Python
"""A prediction registered before the run, and falsified by it.
|
|
|
|
The hypothesis was that a method which wins in some places and loses in others
|
|
must have a regime where it wins outright, and that the regime is where the
|
|
quantity the constant approximates CHANGES during the run - because one
|
|
constant cannot be right at two different noise levels while a measured value
|
|
tracks.
|
|
|
|
It was stated before measuring, with the falsification condition attached: the
|
|
measured rule had to lose every constant-noise cell and win every
|
|
varying-noise cell. It lost all twelve.
|
|
|
|
The first version of the test could not have confirmed it either way - noise
|
|
reduction was scored on the second half of the run, by which time every ramp
|
|
and step had already arrived, so the metric never saw a transition. That flaw
|
|
was found and fixed before the conclusion was drawn, and the conclusion did
|
|
not change.
|
|
|
|
This file exists so the line is not reopened without new evidence.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
import os
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from anc.rules import Fixed, Measured
|
|
from stage20_prediction import (FLAT_5, FLAT_30, STEP_25, SWING_25, sig_cabin,
|
|
sig_tonal, score)
|
|
|
|
SEEDS = 3
|
|
|
|
|
|
def test_the_metric_window_sees_the_transition():
|
|
"""The flaw that would have produced a false falsification. A varying
|
|
profile must not score the same as a constant one at its final level."""
|
|
varying = score(Fixed(0.1), sig_tonal, SWING_25, seeds=SEEDS)
|
|
final_level = score(Fixed(0.1), sig_tonal, FLAT_5, seeds=SEEDS)
|
|
assert varying > final_level + 2.0, (varying, final_level)
|
|
|
|
|
|
@pytest.mark.parametrize("signal", [sig_tonal, sig_cabin],
|
|
ids=["tonal", "cabin"])
|
|
@pytest.mark.parametrize("profile,kind", [(FLAT_30, "control"),
|
|
(FLAT_5, "control"),
|
|
(SWING_25, "prediction"),
|
|
(STEP_25, "prediction")],
|
|
ids=["flat30", "flat5", "ramp", "step"])
|
|
def test_an_oracle_constant_beats_the_measured_rule_everywhere(signal, profile,
|
|
kind):
|
|
"""The registered prediction required the measured rule to WIN the two
|
|
'prediction' cells. It does not, on either signal. A single constant,
|
|
chosen with knowledge of the scenario, is ahead even when the noise level
|
|
swings by 25 dB inside the run."""
|
|
ms = score(Measured(), signal, profile, seeds=SEEDS)
|
|
best = max(score(Fixed(mu), signal, profile, seeds=SEEDS)
|
|
for mu in (0.05, 0.1, 0.2, 0.5, 1.0))
|
|
assert best > ms, (kind, best, ms)
|
|
|
|
|
|
def test_the_conclusion_this_licenses():
|
|
"""Winning somewhere and losing elsewhere does not imply a regime of
|
|
outright victory exists; it is equally consistent with a middling
|
|
generalist. On this evidence there is no such regime, and the claim the
|
|
work may make is the scoped one in test_modern_repairs.py."""
|
|
losses = 0
|
|
for signal in (sig_tonal, sig_cabin):
|
|
for profile in (FLAT_30, FLAT_5, SWING_25, STEP_25):
|
|
ms = score(Measured(), signal, profile, seeds=SEEDS)
|
|
best = max(score(Fixed(mu), signal, profile, seeds=SEEDS)
|
|
for mu in (0.05, 0.2, 1.0))
|
|
losses += ms < best
|
|
assert losses == 8, losses
|