diff --git a/README.md b/README.md index 95ca71f..f491084 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ Learned detectors for AIMsight: models trained on the reviewed labels to tell, a - **Model**: multiple-instance learning, since a label names a station and a range, not its sensors, and each station has its own number of sensors: one small MLP shared by every sensor, attention pooling to a score per family for the station; the attention weights give the sensors. - **Inputs**: each sensor against its own usual level (median and MAD over the 3 days before), on windows in hours (the step goes from 5 min to 4 h); the station context (share of high and rejected sensors, `sigma0`, references, camera pose, housing temperature, common mode by PCA); the image quality metrics when present, with a mask. - **Training**: pre-training on the outputs of the current detectors over all the data, then fine-tuning on the reviewed labels of `labeling/auto/`. -- **Evaluation**: leave one station out; the measure of `detectors/bench/score.py` (found, correct, per family and per instant), against the rules on the same stations. +- **Evaluation**: the measure of `detectors/bench/score.py` (found, correct, per family and per instant), against the rules on the same instants. +- **The criterion** (decided 2026-11-16): on the stations seen, the way the model will run (every station followed keeps being labelled in the app): trained on the start of every station, tested on its end (a cut in time, never at random). The model beats the rules by at least 5 points of found and of correct, in total, for Visibility and for Obstruction, and loses nowhere more than 2 points on a family of a station (the regression of `detectors/CLAUDE.md`). A new station stays under the rules until it has a few weeks of reviewed labels; the score with a station left out is followed at every step, as an indicator, not a gate. - **Steps**: dataset, evaluator (reproducing the rules' figures first), gradient boosting baseline, the PyTorch model, pre-training compared with and without, decoding into episodes. ## Use @@ -44,7 +45,7 @@ uv venv .venv && uv pip install -p .venv -r requirements.txt | rules, last 30 % | 49 / 77 % | 15 / 41 % | | M1, last 30 % (`--split-time`) | 58 / 87 % | 39 / 75 % | -The information is in the features: with every station seen, M1 beats the rules; moving to a new station is what fails (Obstruction is labelled on three stations only). +The information is in the features: with every station seen, M1 beats the rules; moving to a new station is what fails (Obstruction is labelled on three stations only). Against the criterion, M1 passes the totals but not the regressions: on the last 30 %, Obstruction found at BREITHORN (90 → 57 %, 21 instants), LENZSPITZE (15 → 9 %, 33) and SNCF01 (100 → 56 %, 9 instants, where the rules flag 272 for 9 right); Visibility at LENZSPITZE (found 64 → 58 %, correct 98 → 81 %), SNCF01 and WILDSTRUBEL (correct 96 → 89 % and 97 → 93 %, with found 10 → 68 % and 86 → 99 %). ## Contracts diff --git a/evaluate.py b/evaluate.py index 10d0442..d84a75f 100644 --- a/evaluate.py +++ b/evaluate.py @@ -51,7 +51,7 @@ def regressions(base, new): b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0]) for name, i in (("found", 0), ("correct", 1)): pb, pn = pct(b[2], b[i]), pct(n[2], n[i]) - if pb is not None and (pn is None or pb - pn > REGRESSION): + if pb is not None and pn is not None and pb - pn > REGRESSION: # nothing flagged: no correct to lose out.append((f, name, pb, pn)) return out diff --git a/m1.py b/m1.py index 8c9dac3..db2ad1a 100644 --- a/m1.py +++ b/m1.py @@ -49,7 +49,7 @@ def fit(x, y, w=None): def split_time(a, data, rows, families): """Every station seen in training: does the information lie in the features, apart from moving to a new station?""" - from evaluate import counts, show, total + from evaluate import counts, regressions, show, total cut = {s: int(0.7 * len(r)) for s, r in rows.items()} x = np.concatenate([rows[s][:cut[s]] for s in data]) base, new = {}, {} @@ -66,6 +66,10 @@ def split_time(a, data, rows, families): base[s] = counts(y, np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:]) new[s] = counts(y, yhat[s]) keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731 + for s in data: + for f, name, pb, pn in regressions(keep(base[s]), keep(new[s])): + print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}" + f" (rules {base[s][f]}, model {new[s].get(f)}: labelled, flagged, both)") show("Last 30 % of every station", keep(total(base)), keep(total(new))) diff --git a/test_evaluate.py b/test_evaluate.py index 912fa13..f33f161 100644 --- a/test_evaluate.py +++ b/test_evaluate.py @@ -15,4 +15,5 @@ c = evaluate.counts(y, yhat) assert c == {"Visibility": [1, 1, 0], "Obstruction": [2, 2, 2]}, c assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}) == [("Obstruction", "found", 50, 40)] assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 10, 5]}) == [] +assert evaluate.regressions({"Obstruction": [0, 15, 0]}, {}) == [] # false alarms gone: not a regression print("ok")