2026-11-16 16:25:03 +01:00
|
|
|
"""The evaluator of the pilot: the measure of detectors/bench/score.py (found, correct, per family and per instant) on
|
|
|
|
|
predictions stored per instant, so that a model and the rules are scored on the same grid.
|
|
|
|
|
|
|
|
|
|
rules: the detectors' events of each station of the cache (bench/score.py events(), the range classified with its
|
|
|
|
|
3 days of history) -> <cache>/rules/<STATION>.npz (yhat [instant, family]); checks that the counts are those
|
|
|
|
|
of bench/score.py family_scores, exactly.
|
|
|
|
|
score: the predictions of a folder (<STATION>.npz with yhat) against the labels, beside the rules, with the
|
|
|
|
|
regressions (a family of a station losing more than 2 points of found or correct, as detectors/CLAUDE.md).
|
|
|
|
|
|
|
|
|
|
Usage: python evaluate.py rules --detectors <detectors> --data <data> --labels <labels folder> --cache <cache>
|
|
|
|
|
python evaluate.py score <predictions folder> --cache <cache>
|
|
|
|
|
"""
|
|
|
|
|
import argparse
|
|
|
|
|
import sys
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
|
|
|
|
|
|
|
from dataset import FAMILIES
|
|
|
|
|
|
|
|
|
|
REGRESSION = 2 # points, detectors/CLAUDE.md
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def counts(y, yhat):
|
|
|
|
|
"""Labelled and predicted [T, K] -> {family: [labelled, flagged, both]}, the families present on either side."""
|
|
|
|
|
y, yhat = y.astype(bool), yhat.astype(bool)
|
|
|
|
|
return {f: [int(y[:, k].sum()), int(yhat[:, k].sum()), int((y[:, k] & yhat[:, k]).sum())]
|
|
|
|
|
for k, f in enumerate(FAMILIES) if y[:, k].any() or yhat[:, k].any()}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def events_to_yhat(events, instants):
|
|
|
|
|
"""Episodes (start, end iso; category or kind = the family) -> yhat [T, K]: an instant within the bounds, included,
|
|
|
|
|
as bench/score.py."""
|
|
|
|
|
arr = np.array(instants)
|
|
|
|
|
yhat = np.zeros((len(arr), len(FAMILIES)), np.uint8)
|
|
|
|
|
for v in events:
|
|
|
|
|
family = v.get("category") or v.get("kind")
|
|
|
|
|
if family in FAMILIES:
|
|
|
|
|
yhat[(arr >= v["start"]) & (arr <= v["end"]), FAMILIES.index(family)] = 1
|
|
|
|
|
return yhat
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def pct(both, n):
|
|
|
|
|
return 100 * both / n if n else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def regressions(base, new):
|
|
|
|
|
"""{family: counts} before and after -> [(family, "found"|"correct", before %, after %)] losing > REGRESSION."""
|
|
|
|
|
out = []
|
|
|
|
|
for f in sorted(set(base) | set(new)):
|
|
|
|
|
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
|
|
|
|
|
for name, i in (("found", 0), ("correct", 1)):
|
|
|
|
|
pb, pn = pct(b[2], b[i]), pct(n[2], n[i])
|
2026-11-16 17:59:06 +01:00
|
|
|
if pb is not None and pn is not None and pb - pn > REGRESSION: # nothing flagged: no correct to lose
|
2026-11-16 16:25:03 +01:00
|
|
|
out.append((f, name, pb, pn))
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def total(per_station):
|
|
|
|
|
t = {}
|
|
|
|
|
for c in per_station.values():
|
|
|
|
|
for f, v in c.items():
|
|
|
|
|
t[f] = [a + b for a, b in zip(t.get(f, [0, 0, 0]), v)]
|
|
|
|
|
return t
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def show(title, base, new):
|
|
|
|
|
def cell(c, i):
|
|
|
|
|
return f"{c[2]}/{c[i]} ({round(100 * c[2] / c[i])} %)" if c[i] else "—"
|
|
|
|
|
print(title)
|
|
|
|
|
print(f" {'':<22}{'found: rules':>21}{'model':>21}{'correct: rules':>21}{'model':>21}")
|
|
|
|
|
for f in FAMILIES:
|
|
|
|
|
if f in base or f in new:
|
|
|
|
|
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
|
|
|
|
|
print(f" {f:<22}{cell(b, 0):>21}{cell(n, 0):>21}{cell(b, 1):>21}{cell(n, 1):>21}")
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 19:22:53 +01:00
|
|
|
CUT = 0.7 # the criterion: trained on the first 70 % of every station, tested on the last 30 %
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def report_time_split(cache, data, proba, families, threshold=0.5):
|
|
|
|
|
"""The criterion: {station: proba [last 30 %, pilot family]} against the rules on the same instants -> the
|
|
|
|
|
regressions station by station and the total, printed; returns (rules, model) counts per station."""
|
|
|
|
|
base, new = {}, {}
|
|
|
|
|
for s, d in data.items():
|
|
|
|
|
cut = int(CUT * len(d["y"]))
|
|
|
|
|
rules_yhat = np.load(Path(cache) / "rules" / f"{s}.npz")["yhat"][cut:]
|
|
|
|
|
yhat = rules_yhat.copy()
|
|
|
|
|
for j, f in enumerate(families):
|
|
|
|
|
yhat[:, FAMILIES.index(f)] = proba[s][:, j] >= threshold
|
|
|
|
|
keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731
|
|
|
|
|
base[s], new[s] = keep(counts(d["y"][cut:], rules_yhat)), keep(counts(d["y"][cut:], yhat))
|
|
|
|
|
for f, name, pb, pn in regressions(base[s], new[s]):
|
|
|
|
|
print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %"
|
|
|
|
|
f" (rules {base[s][f]}, model {new[s].get(f)}: labelled, flagged, both)")
|
|
|
|
|
show(f"Last {round(100 * (1 - CUT))} % of every station", total(base), total(new))
|
|
|
|
|
return base, new
|
|
|
|
|
|
|
|
|
|
|
2026-11-16 16:25:03 +01:00
|
|
|
def rules(a):
|
|
|
|
|
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
|
|
|
|
|
import score # noqa: E402
|
|
|
|
|
from data import load # noqa: E402
|
|
|
|
|
out = Path(a.cache) / "rules"
|
|
|
|
|
out.mkdir(exist_ok=True)
|
|
|
|
|
wrong = 0
|
|
|
|
|
for path in sorted(Path(a.cache).glob("*.npz")):
|
|
|
|
|
station, d = path.stem, np.load(path)
|
|
|
|
|
instants, start, end, folder = list(d["instants"]), d["start"].item(), d["end"].item(), str(Path(a.data) / station)
|
|
|
|
|
bench, _ = score.family_scores(folder, str(Path(a.labels) / f"{station}.jsonl"), start, end)
|
|
|
|
|
evts = score.events(load(folder), start, end, folder)
|
|
|
|
|
yhat = events_to_yhat(evts, instants)
|
|
|
|
|
np.savez_compressed(out / f"{station}.npz", yhat=yhat)
|
|
|
|
|
mine = counts(d["y"], yhat)
|
|
|
|
|
bench = {f: v for f, v in bench.items() if f in FAMILIES}
|
|
|
|
|
ok = mine == bench
|
|
|
|
|
wrong += not ok
|
|
|
|
|
print(f"{station}: {'same counts as bench/score.py' if ok else 'DIFFERENT'}" + ("" if ok else f"\n bench {bench}\n mine {mine}"))
|
|
|
|
|
sys.exit(1 if wrong else 0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def score_folder(a):
|
|
|
|
|
base, new = {}, {}
|
|
|
|
|
for path in sorted(Path(a.cache).glob("*.npz")):
|
|
|
|
|
p = Path(a.predictions) / path.name
|
|
|
|
|
if not p.exists():
|
|
|
|
|
continue
|
|
|
|
|
y = np.load(path)["y"]
|
|
|
|
|
base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"])
|
|
|
|
|
new[path.stem] = counts(y, np.load(p)["yhat"])
|
|
|
|
|
show(path.stem, base[path.stem], new[path.stem])
|
|
|
|
|
for f, name, pb, pn in regressions(base[path.stem], new[path.stem]):
|
|
|
|
|
print(f" regression: {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}")
|
|
|
|
|
print()
|
|
|
|
|
show("Total", total(base), total(new))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
p = argparse.ArgumentParser(usage=__doc__)
|
|
|
|
|
sub = p.add_subparsers(dest="cmd", required=True)
|
|
|
|
|
r = sub.add_parser("rules")
|
|
|
|
|
for k in ("detectors", "data", "labels", "cache"):
|
|
|
|
|
r.add_argument("--" + k, required=True)
|
|
|
|
|
s = sub.add_parser("score")
|
|
|
|
|
s.add_argument("predictions")
|
|
|
|
|
s.add_argument("--cache", required=True)
|
|
|
|
|
a = p.parse_args()
|
|
|
|
|
rules(a) if a.cmd == "rules" else score_folder(a)
|