From c04cbc90bba8bbadfe114b418c5298d8509b4b34 Mon Sep 17 00:00:00 2001 From: maxime Date: Mon, 16 Nov 2026 16:25:03 +0100 Subject: [PATCH] Step 2: the evaluator (the measure of bench/score.py on predictions per instant; the rules reproduce it exactly) Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +++ dataset.py | 2 +- evaluate.py | 126 +++++++++++++++++++++++++++++++++++++++++++++++ test_evaluate.py | 18 +++++++ 4 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 evaluate.py create mode 100644 test_evaluate.py diff --git a/README.md b/README.md index 48e0ff4..b7b57b8 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,14 @@ uv venv .venv && uv pip install -p .venv -r requirements.txt `dataset.py` (step 1, ~2 min): one `cache/.npz` per labelled station, over its labelled range as the benchmark reads it: `xs` [instant, sensor, feature], `xc` [instant, feature], `y` [instant, family] with the conventions of `bench/score.py` (its labelled instants per family are the benchmark's, exactly). NaN where a value does not exist (a 1 h window at a 4 h step, no temperature, no image quality outside LENZSPITZE). +```bash +.venv/bin/python test_evaluate.py +.venv/bin/python evaluate.py rules --detectors ~/aimsight-detectors --data ~/aimsight-data/data --labels ~/aimsight-labeling/auto --cache cache +.venv/bin/python evaluate.py score --cache cache +``` + +`evaluate.py rules` (step 2, ~20 min, once per change of the detectors): the rules' predictions per instant in `cache/rules/`, and the check that their counts are those of `bench/score.py`, station by station (exit 1 otherwise). `evaluate.py score`: a folder of `.npz` with `yhat` [instant, family] against the labels, beside the rules, with the regressions of `detectors/CLAUDE.md` (more than 2 points of found or correct lost on a family of a station). + ## Contracts - Reads the measurements through `detectors/scripts/data.py`, the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments. diff --git a/dataset.py b/dataset.py index e633ff5..baed5cf 100644 --- a/dataset.py +++ b/dataset.py @@ -170,7 +170,7 @@ def build(folder, labels, start, end, data): c_names, xc = station_features(f, kinds, grid, data.load_camera(folder), data.load_temperature(folder), measurements) keep = np.array([start <= data.iso(t) < end for t in grid]) instants = [data.iso(t) for t in grid[keep]] - return {"instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())), + return {"start": np.array(start), "end": np.array(end), "instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())), "xs": xs[keep].astype(np.float32), "xs_names": np.array(s_names), "xc": xc[keep].astype(np.float32), "xc_names": np.array(c_names), "y": label_families(labels, instants), "families": np.array(FAMILIES)} diff --git a/evaluate.py b/evaluate.py new file mode 100644 index 0000000..10d0442 --- /dev/null +++ b/evaluate.py @@ -0,0 +1,126 @@ +"""The evaluator of the pilot: the measure of detectors/bench/score.py (found, correct, per family and per instant) on +predictions stored per instant, so that a model and the rules are scored on the same grid. + + rules: the detectors' events of each station of the cache (bench/score.py events(), the range classified with its + 3 days of history) -> /rules/.npz (yhat [instant, family]); checks that the counts are those + of bench/score.py family_scores, exactly. + score: the predictions of a folder (.npz with yhat) against the labels, beside the rules, with the + regressions (a family of a station losing more than 2 points of found or correct, as detectors/CLAUDE.md). + +Usage: python evaluate.py rules --detectors --data --labels --cache + python evaluate.py score --cache +""" +import argparse +import sys +from pathlib import Path + +import numpy as np + +from dataset import FAMILIES + +REGRESSION = 2 # points, detectors/CLAUDE.md + + +def counts(y, yhat): + """Labelled and predicted [T, K] -> {family: [labelled, flagged, both]}, the families present on either side.""" + y, yhat = y.astype(bool), yhat.astype(bool) + return {f: [int(y[:, k].sum()), int(yhat[:, k].sum()), int((y[:, k] & yhat[:, k]).sum())] + for k, f in enumerate(FAMILIES) if y[:, k].any() or yhat[:, k].any()} + + +def events_to_yhat(events, instants): + """Episodes (start, end iso; category or kind = the family) -> yhat [T, K]: an instant within the bounds, included, + as bench/score.py.""" + arr = np.array(instants) + yhat = np.zeros((len(arr), len(FAMILIES)), np.uint8) + for v in events: + family = v.get("category") or v.get("kind") + if family in FAMILIES: + yhat[(arr >= v["start"]) & (arr <= v["end"]), FAMILIES.index(family)] = 1 + return yhat + + +def pct(both, n): + return 100 * both / n if n else None + + +def regressions(base, new): + """{family: counts} before and after -> [(family, "found"|"correct", before %, after %)] losing > REGRESSION.""" + out = [] + for f in sorted(set(base) | set(new)): + b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0]) + for name, i in (("found", 0), ("correct", 1)): + pb, pn = pct(b[2], b[i]), pct(n[2], n[i]) + if pb is not None and (pn is None or pb - pn > REGRESSION): + out.append((f, name, pb, pn)) + return out + + +def total(per_station): + t = {} + for c in per_station.values(): + for f, v in c.items(): + t[f] = [a + b for a, b in zip(t.get(f, [0, 0, 0]), v)] + return t + + +def show(title, base, new): + def cell(c, i): + return f"{c[2]}/{c[i]} ({round(100 * c[2] / c[i])} %)" if c[i] else "—" + print(title) + print(f" {'':<22}{'found: rules':>21}{'model':>21}{'correct: rules':>21}{'model':>21}") + for f in FAMILIES: + if f in base or f in new: + b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0]) + print(f" {f:<22}{cell(b, 0):>21}{cell(n, 0):>21}{cell(b, 1):>21}{cell(n, 1):>21}") + + +def rules(a): + sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")] + import score # noqa: E402 + from data import load # noqa: E402 + out = Path(a.cache) / "rules" + out.mkdir(exist_ok=True) + wrong = 0 + for path in sorted(Path(a.cache).glob("*.npz")): + station, d = path.stem, np.load(path) + instants, start, end, folder = list(d["instants"]), d["start"].item(), d["end"].item(), str(Path(a.data) / station) + bench, _ = score.family_scores(folder, str(Path(a.labels) / f"{station}.jsonl"), start, end) + evts = score.events(load(folder), start, end, folder) + yhat = events_to_yhat(evts, instants) + np.savez_compressed(out / f"{station}.npz", yhat=yhat) + mine = counts(d["y"], yhat) + bench = {f: v for f, v in bench.items() if f in FAMILIES} + ok = mine == bench + wrong += not ok + print(f"{station}: {'same counts as bench/score.py' if ok else 'DIFFERENT'}" + ("" if ok else f"\n bench {bench}\n mine {mine}")) + sys.exit(1 if wrong else 0) + + +def score_folder(a): + base, new = {}, {} + for path in sorted(Path(a.cache).glob("*.npz")): + p = Path(a.predictions) / path.name + if not p.exists(): + continue + y = np.load(path)["y"] + base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"]) + new[path.stem] = counts(y, np.load(p)["yhat"]) + show(path.stem, base[path.stem], new[path.stem]) + for f, name, pb, pn in regressions(base[path.stem], new[path.stem]): + print(f" regression: {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}") + print() + show("Total", total(base), total(new)) + + +if __name__ == "__main__": + p = argparse.ArgumentParser(usage=__doc__) + sub = p.add_subparsers(dest="cmd", required=True) + r = sub.add_parser("rules") + for k in ("detectors", "data", "labels", "cache"): + r.add_argument("--" + k, required=True) + s = sub.add_parser("score") + s.add_argument("predictions") + s.add_argument("--cache", required=True) + a = p.parse_args() + rules(a) if a.cmd == "rules" else score_folder(a) diff --git a/test_evaluate.py b/test_evaluate.py new file mode 100644 index 0000000..912fa13 --- /dev/null +++ b/test_evaluate.py @@ -0,0 +1,18 @@ +"""Checks of evaluate.py: python test_evaluate.py (the check against the benchmark itself is `evaluate.py rules`).""" +import numpy as np + +import evaluate + +instants = ["2026-10-01T00:00Z", "2026-10-01T00:15Z", "2026-10-01T00:30Z"] +yhat = evaluate.events_to_yhat([{"start": "2026-10-01T00:15Z", "end": "2026-10-01T00:30Z", "category": "Obstruction"}, + {"start": "2026-10-01T00:00Z", "end": "2026-10-01T00:00Z", "kind": "Visibility"}], + instants) +assert yhat[:, 1].tolist() == [0, 1, 1] and yhat[:, 0].tolist() == [1, 0, 0] # bounds included +y = np.zeros_like(yhat) +y[1:, 1] = 1 +y[2, 0] = 1 +c = evaluate.counts(y, yhat) +assert c == {"Visibility": [1, 1, 0], "Obstruction": [2, 2, 2]}, c +assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}) == [("Obstruction", "found", 50, 40)] +assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 10, 5]}) == [] +print("ok")