From f93cdf3cd15f9d9765e17dfed77d85401e3d1e26 Mon Sep 17 00:00:00 2001 From: maxime Date: Mon, 16 Nov 2026 17:48:11 +0100 Subject: [PATCH] Step 3: M1, gradient boosting on station aggregates (leave one station out, and a time split) Co-Authored-By: Claude Opus 5.5 --- README.md | 17 ++++++++ m1.py | 108 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 126 insertions(+) create mode 100644 m1.py diff --git a/README.md b/README.md index b7b57b8..95ca71f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,23 @@ uv venv .venv && uv pip install -p .venv -r requirements.txt `evaluate.py rules` (step 2, ~20 min, once per change of the detectors): the rules' predictions per instant in `cache/rules/`, and the check that their counts are those of `bench/score.py`, station by station (exit 1 otherwise). `evaluate.py score`: a folder of `.npz` with `yhat` [instant, family] against the labels, beside the rules, with the regressions of `detectors/CLAUDE.md` (more than 2 points of found or correct lost on a family of a station). +```bash +.venv/bin/python m1.py --cache cache --out runs/m1 --balance && .venv/bin/python evaluate.py score runs/m1 --cache cache +.venv/bin/python m1.py --cache cache --out - --split-time --balance +``` + +`m1.py` (step 3, a few minutes): gradient boosting on the station's features and the aggregates of its sensors', Visibility and Obstruction, leaving one station out; `--split-time`, the diagnosis on the last 30 % of every station, all stations seen in training. First results (2026-11-16, found / correct, total): + +| | Visibility | Obstruction | +|---|---|---| +| rules, all stations | 47 / 77 % | 26 / 28 % | +| M1, station left out | 44 / 62 % | 9 / 7 % | +| M1, station left out, `--balance` | 46 / 70 % | 21 / 23 % | +| rules, last 30 % | 49 / 77 % | 15 / 41 % | +| M1, last 30 % (`--split-time`) | 58 / 87 % | 39 / 75 % | + +The information is in the features: with every station seen, M1 beats the rules; moving to a new station is what fails (Obstruction is labelled on three stations only). + ## Contracts - Reads the measurements through `detectors/scripts/data.py`, the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments. diff --git a/m1.py b/m1.py new file mode 100644 index 0000000..8c9dac3 --- /dev/null +++ b/m1.py @@ -0,0 +1,108 @@ +"""M1, the first rung of the pilot: gradient boosting on what the station does at each instant (its features and the +aggregates of its sensors' features), one classifier per family of the pilot, leaving one station out. + +Each station is predicted by the classifier trained on the others; the families outside the pilot keep the rules' +predictions (cache/rules/), so that `evaluate.py score` compares the pilot families only. No sensor attribution: that is +M2. One .npz per station in --out, with yhat [instant, family] and proba [instant, pilot family]. +Usage: python m1.py --cache --out [--families Visibility,Obstruction] [--threshold 0.5] +""" +import argparse +import warnings +from pathlib import Path + +import numpy as np +from sklearn.ensemble import HistGradientBoostingClassifier + +from dataset import FAMILIES + +STATS = ("median", "q90", "min", "max") + + +def station_rows(d): + """A station of the cache -> (X [T, F], names): its station features, then each sensor feature aggregated over + the targets and over the references of the compensation.""" + xs, kinds = d["xs"], d["kinds"] + cols, names = [d["xc"]], [f"station.{n}" for n in d["xc_names"]] + with warnings.catch_warnings(): + warnings.simplefilter("ignore", RuntimeWarning) # all-NaN slices: the aggregate is NaN, which the model reads + for group, mask in (("targets", kinds != "global_ref"), ("refs", kinds == "global_ref")): + g = xs[:, mask, :] + for stat in STATS: + if not mask.any(): + v = np.full((len(xs), xs.shape[2]), np.nan, np.float32) + elif stat == "q90": + v = np.nanquantile(g, 0.9, axis=1) + else: + v = getattr(np, "nan" + stat)(g, axis=1) + cols.append(v) + names += [f"{group}.{stat}.{n}" for n in d["xs_names"]] + x = np.concatenate(cols, axis=1).astype(np.float32) + x[~np.isfinite(x)] = np.nan # inf of a log ratio: missing + return x, names + + +def fit(x, y, w=None): + model = HistGradientBoostingClassifier(max_iter=300, learning_rate=0.05, max_leaf_nodes=15, l2_regularization=1.0, + random_state=0) + return model.fit(x, y, sample_weight=None if w is None else w * len(w) / w.sum()) + + +def split_time(a, data, rows, families): + """Every station seen in training: does the information lie in the features, apart from moving to a new station?""" + from evaluate import counts, show, total + cut = {s: int(0.7 * len(r)) for s, r in rows.items()} + x = np.concatenate([rows[s][:cut[s]] for s in data]) + base, new = {}, {} + yhat = {s: np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:].copy() for s in data} + for f in families: + k = FAMILIES.index(f) + y = np.concatenate([data[s]["y"][:cut[s], k] for s in data]) + w = np.concatenate([np.full(cut[s], 1 / cut[s]) for s in data]) if a.balance else None + model = fit(x, y, w) + for s in data: + yhat[s][:, k] = model.predict_proba(rows[s][cut[s]:])[:, 1] >= a.threshold + for s in data: + y = data[s]["y"][cut[s]:] + base[s] = counts(y, np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:]) + new[s] = counts(y, yhat[s]) + keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731 + show("Last 30 % of every station", keep(total(base)), keep(total(new))) + + +def main(): + p = argparse.ArgumentParser(usage=__doc__) + p.add_argument("--cache", required=True) + p.add_argument("--out", required=True) + p.add_argument("--families", default="Visibility,Obstruction") + p.add_argument("--threshold", type=float, default=0.5) + p.add_argument("--balance", action="store_true", help="each station weighs the same in training") + p.add_argument("--split-time", action="store_true", + help="diagnosis: train on the first 70 %% of every station, score the last 30 %% (nothing saved)") + a = p.parse_args() + families = a.families.split(",") + paths = sorted(Path(a.cache).glob("*.npz")) + data = {q.stem: np.load(q) for q in paths} + rows = {s: station_rows(d)[0] for s, d in data.items()} + if a.split_time: + return split_time(a, data, rows, families) + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + for test in data: + train = [s for s in data if s != test] + x = np.concatenate([rows[s] for s in train]) + keep = ~np.isnan(x).all(axis=0) # the image quality, when the only station that has it is the tested one + x, xt = x[:, keep], rows[test][:, keep] + yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy() + proba = np.zeros((len(xt), len(families)), np.float32) + for j, f in enumerate(families): + k = FAMILIES.index(f) + y = np.concatenate([data[s]["y"][:, k] for s in train]) + w = np.concatenate([np.full(len(rows[s]), 1 / len(rows[s])) for s in train]) if a.balance else None + proba[:, j] = fit(x, y, w).predict_proba(xt)[:, 1] + yhat[:, k] = proba[:, j] >= a.threshold + np.savez_compressed(out / f"{test}.npz", yhat=yhat, proba=proba, families=np.array(families)) + print(test, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index 5da331c..ad5443d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ numpy pandas +scikit-learn