Files
aimsight-models/evaluate.py
T
maximeandClaude Opus 5.5 5337dd0dac Step 5: pre-training on the rules' outputs (pretrain.py, m2.py --pretrain); regressions as detectors/bench/compare.py; the reader from the tools repository
The detectors were simplified on 2026-11-16: the reader moved to aimsight-tools (scripts/data.py, put on the path by
categories.py) and a regression is now a Total losing more than 1 point, or a family of a station losing more than
5 points and more than 20 instants.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 09:51:56 +01:00

157 lines
7.2 KiB
Python

"""The evaluator of the pilot: the measure of detectors/bench/score.py (found, correct, per family and per instant) on
predictions stored per instant, so that a model and the rules are scored on the same grid.
rules: the detectors' events of each station of the cache (bench/score.py events(), the range classified with its
3 days of history) -> <cache>/rules/<STATION>.npz (yhat [instant, family]); checks that the counts are those
of bench/score.py family_scores, exactly.
score: the predictions of a folder (<STATION>.npz with yhat) against the labels, beside the rules, with the
regressions of detectors/bench/compare.py (Total: more than 1 point; a station: more than 5 points and 20 instants).
Usage: python evaluate.py rules --detectors <detectors> --data <data> --labels <labels folder> --cache <cache>
python evaluate.py score <predictions folder> --cache <cache>
"""
import argparse
import sys
from pathlib import Path
import numpy as np
from dataset import FAMILIES
TOTAL_POINTS, POINTS, INSTANTS = 1, 5, 20 # a regression, as detectors/bench/compare.py
def counts(y, yhat):
"""Labelled and predicted [T, K] -> {family: [labelled, flagged, both]}, the families present on either side."""
y, yhat = y.astype(bool), yhat.astype(bool)
return {f: [int(y[:, k].sum()), int(yhat[:, k].sum()), int((y[:, k] & yhat[:, k]).sum())]
for k, f in enumerate(FAMILIES) if y[:, k].any() or yhat[:, k].any()}
def events_to_yhat(events, instants):
"""Episodes (start, end iso; category or kind = the family) -> yhat [T, K]: an instant within the bounds, included,
as bench/score.py."""
arr = np.array(instants)
yhat = np.zeros((len(arr), len(FAMILIES)), np.uint8)
for v in events:
family = v.get("category") or v.get("kind")
if family in FAMILIES:
yhat[(arr >= v["start"]) & (arr <= v["end"]), FAMILIES.index(family)] = 1
return yhat
def pct(both, n):
return 100 * both / n if n else None
def regressions(base, new, whole=False):
"""{family: counts} before and after -> [(family, "found"|"correct", before %, after %, lost instants)], as
detectors/bench/compare.py: a Total (whole) losing more than 1 point; a family of a station losing more than 5
points and more than 20 instants (labelled instants no longer flagged, or wrongly flagged instants added)."""
out = []
for f in sorted(set(base) | set(new)):
(l0, f0, b0), (l1, f1, b1) = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
for name, pb, pn, lost in (("found", pct(b0, l0), pct(b1, l1), b0 - b1),
("correct", pct(b0, f0), pct(b1, f1), (f1 - b1) - (f0 - b0))):
if pb is None or pn is None or round(pb) == round(pn): # nothing flagged: no correct to lose
continue
if pb - pn > TOTAL_POINTS if whole else pb - pn > POINTS and lost > INSTANTS:
out.append((f, name, pb, pn, lost))
return out
def total(per_station):
t = {}
for c in per_station.values():
for f, v in c.items():
t[f] = [a + b for a, b in zip(t.get(f, [0, 0, 0]), v)]
return t
def show(title, base, new):
def cell(c, i):
return f"{c[2]}/{c[i]} ({round(100 * c[2] / c[i])} %)" if c[i] else "—"
print(title)
print(f" {'':<22}{'found: rules':>21}{'model':>21}{'correct: rules':>21}{'model':>21}")
for f in FAMILIES:
if f in base or f in new:
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
print(f" {f:<22}{cell(b, 0):>21}{cell(n, 0):>21}{cell(b, 1):>21}{cell(n, 1):>21}")
CUT = 0.7 # the criterion: trained on the first 70 % of every station, tested on the last 30 %
def report_time_split(cache, data, proba, families, threshold=0.5):
"""The criterion: {station: proba [last 30 %, pilot family]} against the rules on the same instants -> the
regressions station by station and the total, printed; returns (rules, model) counts per station."""
base, new = {}, {}
for s, d in data.items():
cut = int(CUT * len(d["y"]))
rules_yhat = np.load(Path(cache) / "rules" / f"{s}.npz")["yhat"][cut:]
yhat = rules_yhat.copy()
for j, f in enumerate(families):
yhat[:, FAMILIES.index(f)] = proba[s][:, j] >= threshold
keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731
base[s], new[s] = keep(counts(d["y"][cut:], rules_yhat)), keep(counts(d["y"][cut:], yhat))
for f, name, pb, pn, lost in regressions(base[s], new[s]):
print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants"
f" (rules {base[s][f]}, model {new[s].get(f)}: labelled, flagged, both)")
show(f"Last {round(100 * (1 - CUT))} % of every station", total(base), total(new))
for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True):
print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %")
return base, new
def rules(a):
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
import score # noqa: E402
from data import load # noqa: E402
out = Path(a.cache) / "rules"
out.mkdir(exist_ok=True)
wrong = 0
for path in sorted(Path(a.cache).glob("*.npz")):
station, d = path.stem, np.load(path)
instants, start, end, folder = list(d["instants"]), d["start"].item(), d["end"].item(), str(Path(a.data) / station)
bench, _ = score.family_scores(folder, str(Path(a.labels) / f"{station}.jsonl"), start, end)
evts = score.events(load(folder), start, end, folder)
yhat = events_to_yhat(evts, instants)
np.savez_compressed(out / f"{station}.npz", yhat=yhat)
mine = counts(d["y"], yhat)
bench = {f: v for f, v in bench.items() if f in FAMILIES}
ok = mine == bench
wrong += not ok
print(f"{station}: {'same counts as bench/score.py' if ok else 'DIFFERENT'}" + ("" if ok else f"\n bench {bench}\n mine {mine}"))
sys.exit(1 if wrong else 0)
def score_folder(a):
base, new = {}, {}
for path in sorted(Path(a.cache).glob("*.npz")):
p = Path(a.predictions) / path.name
if not p.exists():
continue
y = np.load(path)["y"]
base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"])
new[path.stem] = counts(y, np.load(p)["yhat"])
show(path.stem, base[path.stem], new[path.stem])
for f, name, pb, pn, lost in regressions(base[path.stem], new[path.stem]):
print(f" regression: {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants")
print()
show("Total", total(base), total(new))
for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True):
print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %")
if __name__ == "__main__":
p = argparse.ArgumentParser(usage=__doc__)
sub = p.add_subparsers(dest="cmd", required=True)
r = sub.add_parser("rules")
for k in ("detectors", "data", "labels", "cache"):
r.add_argument("--" + k, required=True)
s = sub.add_parser("score")
s.add_argument("predictions")
s.add_argument("--cache", required=True)
a = p.parse_args()
rules(a) if a.cmd == "rules" else score_folder(a)