"""The evaluator of the pilot: the measure of detectors/bench/score.py (found, correct, per family and per instant) on predictions stored per instant, so that a model and the rules are scored on the same grid. rules: the detectors' events of each station of the cache (bench/score.py events(), the range classified with its 3 days of history) -> /rules/.npz (yhat [instant, family]); checks that the counts are those of bench/score.py family_scores, exactly. score: the predictions of a folder (.npz with yhat) against the labels, beside the rules, with the regressions (a family of a station losing more than 2 points of found or correct, as detectors/CLAUDE.md). Usage: python evaluate.py rules --detectors --data --labels --cache python evaluate.py score --cache """ import argparse import sys from pathlib import Path import numpy as np from dataset import FAMILIES REGRESSION = 2 # points, detectors/CLAUDE.md def counts(y, yhat): """Labelled and predicted [T, K] -> {family: [labelled, flagged, both]}, the families present on either side.""" y, yhat = y.astype(bool), yhat.astype(bool) return {f: [int(y[:, k].sum()), int(yhat[:, k].sum()), int((y[:, k] & yhat[:, k]).sum())] for k, f in enumerate(FAMILIES) if y[:, k].any() or yhat[:, k].any()} def events_to_yhat(events, instants): """Episodes (start, end iso; category or kind = the family) -> yhat [T, K]: an instant within the bounds, included, as bench/score.py.""" arr = np.array(instants) yhat = np.zeros((len(arr), len(FAMILIES)), np.uint8) for v in events: family = v.get("category") or v.get("kind") if family in FAMILIES: yhat[(arr >= v["start"]) & (arr <= v["end"]), FAMILIES.index(family)] = 1 return yhat def pct(both, n): return 100 * both / n if n else None def regressions(base, new): """{family: counts} before and after -> [(family, "found"|"correct", before %, after %)] losing > REGRESSION.""" out = [] for f in sorted(set(base) | set(new)): b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0]) for name, i in (("found", 0), ("correct", 1)): pb, pn = pct(b[2], b[i]), pct(n[2], n[i]) if pb is not None and pn is not None and pb - pn > REGRESSION: # nothing flagged: no correct to lose out.append((f, name, pb, pn)) return out def total(per_station): t = {} for c in per_station.values(): for f, v in c.items(): t[f] = [a + b for a, b in zip(t.get(f, [0, 0, 0]), v)] return t def show(title, base, new): def cell(c, i): return f"{c[2]}/{c[i]} ({round(100 * c[2] / c[i])} %)" if c[i] else "—" print(title) print(f" {'':<22}{'found: rules':>21}{'model':>21}{'correct: rules':>21}{'model':>21}") for f in FAMILIES: if f in base or f in new: b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0]) print(f" {f:<22}{cell(b, 0):>21}{cell(n, 0):>21}{cell(b, 1):>21}{cell(n, 1):>21}") def rules(a): sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")] import score # noqa: E402 from data import load # noqa: E402 out = Path(a.cache) / "rules" out.mkdir(exist_ok=True) wrong = 0 for path in sorted(Path(a.cache).glob("*.npz")): station, d = path.stem, np.load(path) instants, start, end, folder = list(d["instants"]), d["start"].item(), d["end"].item(), str(Path(a.data) / station) bench, _ = score.family_scores(folder, str(Path(a.labels) / f"{station}.jsonl"), start, end) evts = score.events(load(folder), start, end, folder) yhat = events_to_yhat(evts, instants) np.savez_compressed(out / f"{station}.npz", yhat=yhat) mine = counts(d["y"], yhat) bench = {f: v for f, v in bench.items() if f in FAMILIES} ok = mine == bench wrong += not ok print(f"{station}: {'same counts as bench/score.py' if ok else 'DIFFERENT'}" + ("" if ok else f"\n bench {bench}\n mine {mine}")) sys.exit(1 if wrong else 0) def score_folder(a): base, new = {}, {} for path in sorted(Path(a.cache).glob("*.npz")): p = Path(a.predictions) / path.name if not p.exists(): continue y = np.load(path)["y"] base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"]) new[path.stem] = counts(y, np.load(p)["yhat"]) show(path.stem, base[path.stem], new[path.stem]) for f, name, pb, pn in regressions(base[path.stem], new[path.stem]): print(f" regression: {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}") print() show("Total", total(base), total(new)) if __name__ == "__main__": p = argparse.ArgumentParser(usage=__doc__) sub = p.add_subparsers(dest="cmd", required=True) r = sub.add_parser("rules") for k in ("detectors", "data", "labels", "cache"): r.add_argument("--" + k, required=True) s = sub.add_parser("score") s.add_argument("predictions") s.add_argument("--cache", required=True) a = p.parse_args() rules(a) if a.cmd == "rules" else score_folder(a)