49 lines
2.3 KiB
Python
49 lines
2.3 KiB
Python
"""verify_article.py checks the round-3 additions: the post hoc row of Table 12
|
|
(sweep_estimated_posthoc.json), A' against it on corpus 4, its fractional-free holds, and the
|
|
statement that the transferred detector loses more on simple roots than it gains on multiple ones.
|
|
|
|
python -u apply_verify_revision3.py
|
|
"""
|
|
PATH = "verify_article.py"
|
|
FUNC = '''
|
|
|
|
def verify_revision3():
|
|
"""Revision 3 (point 5): the post hoc branch m >= 2 and the reading of Table 12."""
|
|
print("\\nRevision 3:")
|
|
|
|
def cost(v):
|
|
return v[0] if v[1] else 200
|
|
|
|
def tot(r, keep=None):
|
|
return sum(cost(v) for p, v in r.items() if keep is None or keep(p))
|
|
sets = ("APS", "CHA", "NEW", "RJL", "APP", "D")
|
|
P = _load("sweep_estimated_posthoc.json")
|
|
K = _load("sweep_estimated.json")["k3_analogy"]
|
|
seen, four = _load("sweep_fourth_seen.json"), _load("sweep_fourth_fourth.json")
|
|
ap = dict(seen["Aprime"])
|
|
ap.update(four["Aprime"])
|
|
K1 = _load("sweep_controls.json")["K1"]
|
|
nonsimple = lambda m: m >= 2.0 or m <= 2.0 / 3.0 # noqa: E731
|
|
|
|
def group(r):
|
|
return sum(tot(r[s], lambda p, _m=K1[s]["mstar"]: nonsimple(_m[p])) for s in ("CHA", "NEW", "D"))
|
|
check("Table 12 post hoc row", (tot(P["D"]), sum(tot(P[s]) for s in sets), group(P), tot(P["APS"])),
|
|
(5568, 18372, 6540, 2960))
|
|
ps = list(ap["D"])
|
|
w = sum(cost(ap["D"][p]) < cost(P["D"][p]) for p in ps)
|
|
l = sum(cost(ap["D"][p]) > cost(P["D"][p]) for p in ps)
|
|
check("A' vs the post hoc variant on corpus 4", (w, len(ps) - w - l, l), (178, 34, 28))
|
|
check("post hoc violations", sum(v[2] for s in sets for v in P[s].values()), 0)
|
|
gain = group(ap) - group(K)
|
|
loss = (sum(tot(K[s]) for s in sets) - group(K)) - (sum(tot(ap[s]) for s in sets) - group(ap))
|
|
check("transferred detector: gain on multiple roots < loss elsewhere (vs A')", (gain, loss, gain < loss),
|
|
(1233, 3934, True))
|
|
'''
|
|
src = open(PATH, encoding="utf8").read()
|
|
call_old = " verify_revision2()\n\n print()"
|
|
marker = '\n\nif __name__ == "__main__":\n main()\n'
|
|
assert src.count(call_old) == 1 and src.count(marker) == 1
|
|
src = src.replace(call_old, " verify_revision2()\n verify_revision3()\n\n print()").replace(marker, FUNC + marker)
|
|
open(PATH, "w", encoding="utf8").write(src)
|
|
print("added verify_revision3")
|