Step 2: the evaluator (the measure of bench/score.py on predictions per instant; the rules reproduce it exactly)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b2d3734152
commit
c04cbc90bb
4 files changed
+153
-1
No files matched your search
@@ -21,6 +21,14 @@ uv venv .venv && uv pip install -p .venv -r requirements.txt
|
||||
|
||||
`dataset.py` (step 1, ~2 min): one `cache/<STATION>.npz` per labelled station, over its labelled range as the benchmark reads it: `xs` [instant, sensor, feature], `xc` [instant, feature], `y` [instant, family] with the conventions of `bench/score.py` (its labelled instants per family are the benchmark's, exactly). NaN where a value does not exist (a 1 h window at a 4 h step, no temperature, no image quality outside LENZSPITZE).
|
||||
|
||||
```bash
|
||||
.venv/bin/python test_evaluate.py
|
||||
.venv/bin/python evaluate.py rules --detectors ~/aimsight-detectors --data ~/aimsight-data/data --labels ~/aimsight-labeling/auto --cache cache
|
||||
.venv/bin/python evaluate.py score <predictions folder> --cache cache
|
||||
```
|
||||
|
||||
`evaluate.py rules` (step 2, ~20 min, once per change of the detectors): the rules' predictions per instant in `cache/rules/`, and the check that their counts are those of `bench/score.py`, station by station (exit 1 otherwise). `evaluate.py score`: a folder of `<STATION>.npz` with `yhat` [instant, family] against the labels, beside the rules, with the regressions of `detectors/CLAUDE.md` (more than 2 points of found or correct lost on a family of a station).
|
||||
|
||||
## Contracts
|
||||
|
||||
- Reads the measurements through `detectors/scripts/data.py`, the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments.
|
||||
|
||||
+1
-1
@@ -170,7 +170,7 @@ def build(folder, labels, start, end, data):
|
||||
c_names, xc = station_features(f, kinds, grid, data.load_camera(folder), data.load_temperature(folder), measurements)
|
||||
keep = np.array([start <= data.iso(t) < end for t in grid])
|
||||
instants = [data.iso(t) for t in grid[keep]]
|
||||
return {"instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())),
|
||||
return {"start": np.array(start), "end": np.array(end), "instants": np.array(instants), "sensors": np.array(list(kinds)), "kinds": np.array(list(kinds.values())),
|
||||
"xs": xs[keep].astype(np.float32), "xs_names": np.array(s_names),
|
||||
"xc": xc[keep].astype(np.float32), "xc_names": np.array(c_names),
|
||||
"y": label_families(labels, instants), "families": np.array(FAMILIES)}
|
||||
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
"""The evaluator of the pilot: the measure of detectors/bench/score.py (found, correct, per family and per instant) on
|
||||
predictions stored per instant, so that a model and the rules are scored on the same grid.
|
||||
|
||||
rules: the detectors' events of each station of the cache (bench/score.py events(), the range classified with its
|
||||
3 days of history) -> <cache>/rules/<STATION>.npz (yhat [instant, family]); checks that the counts are those
|
||||
of bench/score.py family_scores, exactly.
|
||||
score: the predictions of a folder (<STATION>.npz with yhat) against the labels, beside the rules, with the
|
||||
regressions (a family of a station losing more than 2 points of found or correct, as detectors/CLAUDE.md).
|
||||
|
||||
Usage: python evaluate.py rules --detectors <detectors> --data <data> --labels <labels folder> --cache <cache>
|
||||
python evaluate.py score <predictions folder> --cache <cache>
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from dataset import FAMILIES
|
||||
|
||||
REGRESSION = 2 # points, detectors/CLAUDE.md
|
||||
|
||||
|
||||
def counts(y, yhat):
|
||||
"""Labelled and predicted [T, K] -> {family: [labelled, flagged, both]}, the families present on either side."""
|
||||
y, yhat = y.astype(bool), yhat.astype(bool)
|
||||
return {f: [int(y[:, k].sum()), int(yhat[:, k].sum()), int((y[:, k] & yhat[:, k]).sum())]
|
||||
for k, f in enumerate(FAMILIES) if y[:, k].any() or yhat[:, k].any()}
|
||||
|
||||
|
||||
def events_to_yhat(events, instants):
|
||||
"""Episodes (start, end iso; category or kind = the family) -> yhat [T, K]: an instant within the bounds, included,
|
||||
as bench/score.py."""
|
||||
arr = np.array(instants)
|
||||
yhat = np.zeros((len(arr), len(FAMILIES)), np.uint8)
|
||||
for v in events:
|
||||
family = v.get("category") or v.get("kind")
|
||||
if family in FAMILIES:
|
||||
yhat[(arr >= v["start"]) & (arr <= v["end"]), FAMILIES.index(family)] = 1
|
||||
return yhat
|
||||
|
||||
|
||||
def pct(both, n):
|
||||
return 100 * both / n if n else None
|
||||
|
||||
|
||||
def regressions(base, new):
|
||||
"""{family: counts} before and after -> [(family, "found"|"correct", before %, after %)] losing > REGRESSION."""
|
||||
out = []
|
||||
for f in sorted(set(base) | set(new)):
|
||||
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
|
||||
for name, i in (("found", 0), ("correct", 1)):
|
||||
pb, pn = pct(b[2], b[i]), pct(n[2], n[i])
|
||||
if pb is not None and (pn is None or pb - pn > REGRESSION):
|
||||
out.append((f, name, pb, pn))
|
||||
return out
|
||||
|
||||
|
||||
def total(per_station):
|
||||
t = {}
|
||||
for c in per_station.values():
|
||||
for f, v in c.items():
|
||||
t[f] = [a + b for a, b in zip(t.get(f, [0, 0, 0]), v)]
|
||||
return t
|
||||
|
||||
|
||||
def show(title, base, new):
|
||||
def cell(c, i):
|
||||
return f"{c[2]}/{c[i]} ({round(100 * c[2] / c[i])} %)" if c[i] else "—"
|
||||
print(title)
|
||||
print(f" {'':<22}{'found: rules':>21}{'model':>21}{'correct: rules':>21}{'model':>21}")
|
||||
for f in FAMILIES:
|
||||
if f in base or f in new:
|
||||
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
|
||||
print(f" {f:<22}{cell(b, 0):>21}{cell(n, 0):>21}{cell(b, 1):>21}{cell(n, 1):>21}")
|
||||
|
||||
|
||||
def rules(a):
|
||||
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
|
||||
import score # noqa: E402
|
||||
from data import load # noqa: E402
|
||||
out = Path(a.cache) / "rules"
|
||||
out.mkdir(exist_ok=True)
|
||||
wrong = 0
|
||||
for path in sorted(Path(a.cache).glob("*.npz")):
|
||||
station, d = path.stem, np.load(path)
|
||||
instants, start, end, folder = list(d["instants"]), d["start"].item(), d["end"].item(), str(Path(a.data) / station)
|
||||
bench, _ = score.family_scores(folder, str(Path(a.labels) / f"{station}.jsonl"), start, end)
|
||||
evts = score.events(load(folder), start, end, folder)
|
||||
yhat = events_to_yhat(evts, instants)
|
||||
np.savez_compressed(out / f"{station}.npz", yhat=yhat)
|
||||
mine = counts(d["y"], yhat)
|
||||
bench = {f: v for f, v in bench.items() if f in FAMILIES}
|
||||
ok = mine == bench
|
||||
wrong += not ok
|
||||
print(f"{station}: {'same counts as bench/score.py' if ok else 'DIFFERENT'}" + ("" if ok else f"\n bench {bench}\n mine {mine}"))
|
||||
sys.exit(1 if wrong else 0)
|
||||
|
||||
|
||||
def score_folder(a):
|
||||
base, new = {}, {}
|
||||
for path in sorted(Path(a.cache).glob("*.npz")):
|
||||
p = Path(a.predictions) / path.name
|
||||
if not p.exists():
|
||||
continue
|
||||
y = np.load(path)["y"]
|
||||
base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"])
|
||||
new[path.stem] = counts(y, np.load(p)["yhat"])
|
||||
show(path.stem, base[path.stem], new[path.stem])
|
||||
for f, name, pb, pn in regressions(base[path.stem], new[path.stem]):
|
||||
print(f" regression: {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}")
|
||||
print()
|
||||
show("Total", total(base), total(new))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
p = argparse.ArgumentParser(usage=__doc__)
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
r = sub.add_parser("rules")
|
||||
for k in ("detectors", "data", "labels", "cache"):
|
||||
r.add_argument("--" + k, required=True)
|
||||
s = sub.add_parser("score")
|
||||
s.add_argument("predictions")
|
||||
s.add_argument("--cache", required=True)
|
||||
a = p.parse_args()
|
||||
rules(a) if a.cmd == "rules" else score_folder(a)
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Checks of evaluate.py: python test_evaluate.py (the check against the benchmark itself is `evaluate.py rules`)."""
|
||||
import numpy as np
|
||||
|
||||
import evaluate
|
||||
|
||||
instants = ["2026-10-01T00:00Z", "2026-10-01T00:15Z", "2026-10-01T00:30Z"]
|
||||
yhat = evaluate.events_to_yhat([{"start": "2026-10-01T00:15Z", "end": "2026-10-01T00:30Z", "category": "Obstruction"},
|
||||
{"start": "2026-10-01T00:00Z", "end": "2026-10-01T00:00Z", "kind": "Visibility"}],
|
||||
instants)
|
||||
assert yhat[:, 1].tolist() == [0, 1, 1] and yhat[:, 0].tolist() == [1, 0, 0] # bounds included
|
||||
y = np.zeros_like(yhat)
|
||||
y[1:, 1] = 1
|
||||
y[2, 0] = 1
|
||||
c = evaluate.counts(y, yhat)
|
||||
assert c == {"Visibility": [1, 1, 0], "Obstruction": [2, 2, 2]}, c
|
||||
assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}) == [("Obstruction", "found", 50, 40)]
|
||||
assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 10, 5]}) == []
|
||||
print("ok")
|
||||
Reference in new issue
Block a user