Step 5: pre-training on the rules' outputs (pretrain.py, m2.py --pretrain); regressions as detectors/bench/compare.py; the reader from the tools repository
The detectors were simplified on 2026-11-16: the reader moved to aimsight-tools (scripts/data.py, put on the path by categories.py) and a regression is now a Total losing more than 1 point, or a family of a station losing more than 5 points and more than 20 instants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7312a1c3ca
commit
5337dd0dac
6 files changed
+147
-51
No files matched your search
+21
-13
@@ -5,7 +5,7 @@ predictions stored per instant, so that a model and the rules are scored on the
|
||||
3 days of history) -> <cache>/rules/<STATION>.npz (yhat [instant, family]); checks that the counts are those
|
||||
of bench/score.py family_scores, exactly.
|
||||
score: the predictions of a folder (<STATION>.npz with yhat) against the labels, beside the rules, with the
|
||||
regressions (a family of a station losing more than 2 points of found or correct, as detectors/CLAUDE.md).
|
||||
regressions of detectors/bench/compare.py (Total: more than 1 point; a station: more than 5 points and 20 instants).
|
||||
|
||||
Usage: python evaluate.py rules --detectors <detectors> --data <data> --labels <labels folder> --cache <cache>
|
||||
python evaluate.py score <predictions folder> --cache <cache>
|
||||
@@ -18,7 +18,7 @@ import numpy as np
|
||||
|
||||
from dataset import FAMILIES
|
||||
|
||||
REGRESSION = 2 # points, detectors/CLAUDE.md
|
||||
TOTAL_POINTS, POINTS, INSTANTS = 1, 5, 20 # a regression, as detectors/bench/compare.py
|
||||
|
||||
|
||||
def counts(y, yhat):
|
||||
@@ -44,15 +44,19 @@ def pct(both, n):
|
||||
return 100 * both / n if n else None
|
||||
|
||||
|
||||
def regressions(base, new):
|
||||
"""{family: counts} before and after -> [(family, "found"|"correct", before %, after %)] losing > REGRESSION."""
|
||||
def regressions(base, new, whole=False):
|
||||
"""{family: counts} before and after -> [(family, "found"|"correct", before %, after %, lost instants)], as
|
||||
detectors/bench/compare.py: a Total (whole) losing more than 1 point; a family of a station losing more than 5
|
||||
points and more than 20 instants (labelled instants no longer flagged, or wrongly flagged instants added)."""
|
||||
out = []
|
||||
for f in sorted(set(base) | set(new)):
|
||||
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
|
||||
for name, i in (("found", 0), ("correct", 1)):
|
||||
pb, pn = pct(b[2], b[i]), pct(n[2], n[i])
|
||||
if pb is not None and pn is not None and pb - pn > REGRESSION: # nothing flagged: no correct to lose
|
||||
out.append((f, name, pb, pn))
|
||||
(l0, f0, b0), (l1, f1, b1) = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
|
||||
for name, pb, pn, lost in (("found", pct(b0, l0), pct(b1, l1), b0 - b1),
|
||||
("correct", pct(b0, f0), pct(b1, f1), (f1 - b1) - (f0 - b0))):
|
||||
if pb is None or pn is None or round(pb) == round(pn): # nothing flagged: no correct to lose
|
||||
continue
|
||||
if pb - pn > TOTAL_POINTS if whole else pb - pn > POINTS and lost > INSTANTS:
|
||||
out.append((f, name, pb, pn, lost))
|
||||
return out
|
||||
|
||||
|
||||
@@ -90,10 +94,12 @@ def report_time_split(cache, data, proba, families, threshold=0.5):
|
||||
yhat[:, FAMILIES.index(f)] = proba[s][:, j] >= threshold
|
||||
keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731
|
||||
base[s], new[s] = keep(counts(d["y"][cut:], rules_yhat)), keep(counts(d["y"][cut:], yhat))
|
||||
for f, name, pb, pn in regressions(base[s], new[s]):
|
||||
print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %"
|
||||
for f, name, pb, pn, lost in regressions(base[s], new[s]):
|
||||
print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants"
|
||||
f" (rules {base[s][f]}, model {new[s].get(f)}: labelled, flagged, both)")
|
||||
show(f"Last {round(100 * (1 - CUT))} % of every station", total(base), total(new))
|
||||
for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True):
|
||||
print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %")
|
||||
return base, new
|
||||
|
||||
|
||||
@@ -129,10 +135,12 @@ def score_folder(a):
|
||||
base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"])
|
||||
new[path.stem] = counts(y, np.load(p)["yhat"])
|
||||
show(path.stem, base[path.stem], new[path.stem])
|
||||
for f, name, pb, pn in regressions(base[path.stem], new[path.stem]):
|
||||
print(f" regression: {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}")
|
||||
for f, name, pb, pn, lost in regressions(base[path.stem], new[path.stem]):
|
||||
print(f" regression: {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants")
|
||||
print()
|
||||
show("Total", total(base), total(new))
|
||||
for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True):
|
||||
print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user