Step 5: pre-training on the rules' outputs (pretrain.py, m2.py --pretrain); regressions as detectors/bench/compare.py; the reader from the tools repository

The detectors were simplified on 2026-11-16: the reader moved to aimsight-tools (scripts/data.py, put on the path by
categories.py) and a regression is now a Total losing more than 1 point, or a family of a station losing more than
5 points and more than 20 instants.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 09:51:56 +01:00
1 parent 7312a1c3ca
commit 5337dd0dac
6 files changed
+147 -51

No files matched your search

+21 -13
View File
@@ -5,7 +5,7 @@ predictions stored per instant, so that a model and the rules are scored on the
3 days of history) -> <cache>/rules/<STATION>.npz (yhat [instant, family]); checks that the counts are those
of bench/score.py family_scores, exactly.
score: the predictions of a folder (<STATION>.npz with yhat) against the labels, beside the rules, with the
regressions (a family of a station losing more than 2 points of found or correct, as detectors/CLAUDE.md).
regressions of detectors/bench/compare.py (Total: more than 1 point; a station: more than 5 points and 20 instants).
Usage: python evaluate.py rules --detectors <detectors> --data <data> --labels <labels folder> --cache <cache>
python evaluate.py score <predictions folder> --cache <cache>
@@ -18,7 +18,7 @@ import numpy as np
from dataset import FAMILIES
REGRESSION = 2 # points, detectors/CLAUDE.md
TOTAL_POINTS, POINTS, INSTANTS = 1, 5, 20 # a regression, as detectors/bench/compare.py
def counts(y, yhat):
@@ -44,15 +44,19 @@ def pct(both, n):
return 100 * both / n if n else None
def regressions(base, new):
"""{family: counts} before and after -> [(family, "found"|"correct", before %, after %)] losing > REGRESSION."""
def regressions(base, new, whole=False):
"""{family: counts} before and after -> [(family, "found"|"correct", before %, after %, lost instants)], as
detectors/bench/compare.py: a Total (whole) losing more than 1 point; a family of a station losing more than 5
points and more than 20 instants (labelled instants no longer flagged, or wrongly flagged instants added)."""
out = []
for f in sorted(set(base) | set(new)):
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
for name, i in (("found", 0), ("correct", 1)):
pb, pn = pct(b[2], b[i]), pct(n[2], n[i])
if pb is not None and pn is not None and pb - pn > REGRESSION: # nothing flagged: no correct to lose
out.append((f, name, pb, pn))
(l0, f0, b0), (l1, f1, b1) = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
for name, pb, pn, lost in (("found", pct(b0, l0), pct(b1, l1), b0 - b1),
("correct", pct(b0, f0), pct(b1, f1), (f1 - b1) - (f0 - b0))):
if pb is None or pn is None or round(pb) == round(pn): # nothing flagged: no correct to lose
continue
if pb - pn > TOTAL_POINTS if whole else pb - pn > POINTS and lost > INSTANTS:
out.append((f, name, pb, pn, lost))
return out
@@ -90,10 +94,12 @@ def report_time_split(cache, data, proba, families, threshold=0.5):
yhat[:, FAMILIES.index(f)] = proba[s][:, j] >= threshold
keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731
base[s], new[s] = keep(counts(d["y"][cut:], rules_yhat)), keep(counts(d["y"][cut:], yhat))
for f, name, pb, pn in regressions(base[s], new[s]):
print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %"
for f, name, pb, pn, lost in regressions(base[s], new[s]):
print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants"
f" (rules {base[s][f]}, model {new[s].get(f)}: labelled, flagged, both)")
show(f"Last {round(100 * (1 - CUT))} % of every station", total(base), total(new))
for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True):
print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %")
return base, new
@@ -129,10 +135,12 @@ def score_folder(a):
base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"])
new[path.stem] = counts(y, np.load(p)["yhat"])
show(path.stem, base[path.stem], new[path.stem])
for f, name, pb, pn in regressions(base[path.stem], new[path.stem]):
print(f" regression: {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}")
for f, name, pb, pn, lost in regressions(base[path.stem], new[path.stem]):
print(f" regression: {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants")
print()
show("Total", total(base), total(new))
for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True):
print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %")
if __name__ == "__main__":