Step 2: the evaluator (the measure of bench/score.py on predictions per instant; the rules reproduce it exactly)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 16:25:03 +01:00
1 parent b2d3734152
commit c04cbc90bb
4 files changed
+153 -1

No files matched your search

+8
View File
@@ -21,6 +21,14 @@ uv venv .venv && uv pip install -p .venv -r requirements.txt
`dataset.py` (step 1, ~2 min): one `cache/<STATION>.npz` per labelled station, over its labelled range as the benchmark reads it: `xs` [instant, sensor, feature], `xc` [instant, feature], `y` [instant, family] with the conventions of `bench/score.py` (its labelled instants per family are the benchmark's, exactly). NaN where a value does not exist (a 1 h window at a 4 h step, no temperature, no image quality outside LENZSPITZE).
```bash
.venv/bin/python test_evaluate.py
.venv/bin/python evaluate.py rules --detectors ~/aimsight-detectors --data ~/aimsight-data/data --labels ~/aimsight-labeling/auto --cache cache
.venv/bin/python evaluate.py score <predictions folder> --cache cache
```
`evaluate.py rules` (step 2, ~20 min, once per change of the detectors): the rules' predictions per instant in `cache/rules/`, and the check that their counts are those of `bench/score.py`, station by station (exit 1 otherwise). `evaluate.py score`: a folder of `<STATION>.npz` with `yhat` [instant, family] against the labels, beside the rules, with the regressions of `detectors/CLAUDE.md` (more than 2 points of found or correct lost on a family of a station).
## Contracts
- Reads the measurements through `detectors/scripts/data.py`, the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments.
+1 -1
View File
@@ -170,7 +170,7 @@ def build(folder, labels, start, end, data):
c_names, xc = station_features(f, kinds, grid, data.load_camera(folder), data.load_temperature(folder), measurements)
keep = np.array([start <= data.iso(t) < end for t in grid])
instants = [data.iso(t) for t in grid[keep]]
return {"instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())),
return {"start": np.array(start), "end": np.array(end), "instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())),
"xs": xs[keep].astype(np.float32), "xs_names": np.array(s_names),
"xc": xc[keep].astype(np.float32), "xc_names": np.array(c_names),
"y": label_families(labels, instants), "families": np.array(FAMILIES)}
+126
View File
@@ -0,0 +1,126 @@
"""The evaluator of the pilot: the measure of detectors/bench/score.py (found, correct, per family and per instant) on
predictions stored per instant, so that a model and the rules are scored on the same grid.
rules: the detectors' events of each station of the cache (bench/score.py events(), the range classified with its
3 days of history) -> <cache>/rules/<STATION>.npz (yhat [instant, family]); checks that the counts are those
of bench/score.py family_scores, exactly.
score: the predictions of a folder (<STATION>.npz with yhat) against the labels, beside the rules, with the
regressions (a family of a station losing more than 2 points of found or correct, as detectors/CLAUDE.md).
Usage: python evaluate.py rules --detectors <detectors> --data <data> --labels <labels folder> --cache <cache>
python evaluate.py score <predictions folder> --cache <cache>
"""
import argparse
import sys
from pathlib import Path
import numpy as np
from dataset import FAMILIES
REGRESSION = 2 # points, detectors/CLAUDE.md
def counts(y, yhat):
"""Labelled and predicted [T, K] -> {family: [labelled, flagged, both]}, the families present on either side."""
y, yhat = y.astype(bool), yhat.astype(bool)
return {f: [int(y[:, k].sum()), int(yhat[:, k].sum()), int((y[:, k] & yhat[:, k]).sum())]
for k, f in enumerate(FAMILIES) if y[:, k].any() or yhat[:, k].any()}
def events_to_yhat(events, instants):
"""Episodes (start, end iso; category or kind = the family) -> yhat [T, K]: an instant within the bounds, included,
as bench/score.py."""
arr = np.array(instants)
yhat = np.zeros((len(arr), len(FAMILIES)), np.uint8)
for v in events:
family = v.get("category") or v.get("kind")
if family in FAMILIES:
yhat[(arr >= v["start"]) & (arr <= v["end"]), FAMILIES.index(family)] = 1
return yhat
def pct(both, n):
return 100 * both / n if n else None
def regressions(base, new):
"""{family: counts} before and after -> [(family, "found"|"correct", before %, after %)] losing > REGRESSION."""
out = []
for f in sorted(set(base) | set(new)):
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
for name, i in (("found", 0), ("correct", 1)):
pb, pn = pct(b[2], b[i]), pct(n[2], n[i])
if pb is not None and (pn is None or pb - pn > REGRESSION):
out.append((f, name, pb, pn))
return out
def total(per_station):
t = {}
for c in per_station.values():
for f, v in c.items():
t[f] = [a + b for a, b in zip(t.get(f, [0, 0, 0]), v)]
return t
def show(title, base, new):
def cell(c, i):
return f"{c[2]}/{c[i]} ({round(100 * c[2] / c[i])} %)" if c[i] else "—"
print(title)
print(f" {'':<22}{'found: rules':>21}{'model':>21}{'correct: rules':>21}{'model':>21}")
for f in FAMILIES:
if f in base or f in new:
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
print(f" {f:<22}{cell(b, 0):>21}{cell(n, 0):>21}{cell(b, 1):>21}{cell(n, 1):>21}")
def rules(a):
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
import score # noqa: E402
from data import load # noqa: E402
out = Path(a.cache) / "rules"
out.mkdir(exist_ok=True)
wrong = 0
for path in sorted(Path(a.cache).glob("*.npz")):
station, d = path.stem, np.load(path)
instants, start, end, folder = list(d["instants"]), d["start"].item(), d["end"].item(), str(Path(a.data) / station)
bench, _ = score.family_scores(folder, str(Path(a.labels) / f"{station}.jsonl"), start, end)
evts = score.events(load(folder), start, end, folder)
yhat = events_to_yhat(evts, instants)
np.savez_compressed(out / f"{station}.npz", yhat=yhat)
mine = counts(d["y"], yhat)
bench = {f: v for f, v in bench.items() if f in FAMILIES}
ok = mine == bench
wrong += not ok
print(f"{station}: {'same counts as bench/score.py' if ok else 'DIFFERENT'}" + ("" if ok else f"\n bench {bench}\n mine {mine}"))
sys.exit(1 if wrong else 0)
def score_folder(a):
base, new = {}, {}
for path in sorted(Path(a.cache).glob("*.npz")):
p = Path(a.predictions) / path.name
if not p.exists():
continue
y = np.load(path)["y"]
base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"])
new[path.stem] = counts(y, np.load(p)["yhat"])
show(path.stem, base[path.stem], new[path.stem])
for f, name, pb, pn in regressions(base[path.stem], new[path.stem]):
print(f" regression: {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}")
print()
show("Total", total(base), total(new))
if __name__ == "__main__":
p = argparse.ArgumentParser(usage=__doc__)
sub = p.add_subparsers(dest="cmd", required=True)
r = sub.add_parser("rules")
for k in ("detectors", "data", "labels", "cache"):
r.add_argument("--" + k, required=True)
s = sub.add_parser("score")
s.add_argument("predictions")
s.add_argument("--cache", required=True)
a = p.parse_args()
rules(a) if a.cmd == "rules" else score_folder(a)
+18
View File
@@ -0,0 +1,18 @@
"""Checks of evaluate.py: python test_evaluate.py (the check against the benchmark itself is `evaluate.py rules`)."""
import numpy as np
import evaluate
instants = ["2026-10-01T00:00Z", "2026-10-01T00:15Z", "2026-10-01T00:30Z"]
yhat = evaluate.events_to_yhat([{"start": "2026-10-01T00:15Z", "end": "2026-10-01T00:30Z", "category": "Obstruction"},
{"start": "2026-10-01T00:00Z", "end": "2026-10-01T00:00Z", "kind": "Visibility"}],
instants)
assert yhat[:, 1].tolist() == [0, 1, 1] and yhat[:, 0].tolist() == [1, 0, 0] # bounds included
y = np.zeros_like(yhat)
y[1:, 1] = 1
y[2, 0] = 1
c = evaluate.counts(y, yhat)
assert c == {"Visibility": [1, 1, 0], "Obstruction": [2, 2, 2]}, c
assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}) == [("Obstruction", "found", 50, 40)]
assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 10, 5]}) == []
print("ok")