Step 3: M1, gradient boosting on station aggregates (leave one station out, and a time split)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 17:48:11 +01:00
1 parent c04cbc90bb
commit f93cdf3cd1
3 files changed
+126

No files matched your search

+17
View File
@@ -29,6 +29,23 @@ uv venv .venv && uv pip install -p .venv -r requirements.txt
`evaluate.py rules` (step 2, ~20 min, once per change of the detectors): the rules' predictions per instant in `cache/rules/`, and the check that their counts are those of `bench/score.py`, station by station (exit 1 otherwise). `evaluate.py score`: a folder of `<STATION>.npz` with `yhat` [instant, family] against the labels, beside the rules, with the regressions of `detectors/CLAUDE.md` (more than 2 points of found or correct lost on a family of a station).
```bash
.venv/bin/python m1.py --cache cache --out runs/m1 --balance && .venv/bin/python evaluate.py score runs/m1 --cache cache
.venv/bin/python m1.py --cache cache --out - --split-time --balance
```
`m1.py` (step 3, a few minutes): gradient boosting on the station's features and the aggregates of its sensors', Visibility and Obstruction, leaving one station out; `--split-time`, the diagnosis on the last 30 % of every station, all stations seen in training. First results (2026-11-16, found / correct, total):
| | Visibility | Obstruction |
|---|---|---|
| rules, all stations | 47 / 77 % | 26 / 28 % |
| M1, station left out | 44 / 62 % | 9 / 7 % |
| M1, station left out, `--balance` | 46 / 70 % | 21 / 23 % |
| rules, last 30 % | 49 / 77 % | 15 / 41 % |
| M1, last 30 % (`--split-time`) | 58 / 87 % | 39 / 75 % |
The information is in the features: with every station seen, M1 beats the rules; moving to a new station is what fails (Obstruction is labelled on three stations only).
## Contracts
- Reads the measurements through `detectors/scripts/data.py`, the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments.
+108
View File
@@ -0,0 +1,108 @@
"""M1, the first rung of the pilot: gradient boosting on what the station does at each instant (its features and the
aggregates of its sensors' features), one classifier per family of the pilot, leaving one station out.
Each station is predicted by the classifier trained on the others; the families outside the pilot keep the rules'
predictions (cache/rules/), so that `evaluate.py score` compares the pilot families only. No sensor attribution: that is
M2. One <STATION>.npz per station in --out, with yhat [instant, family] and proba [instant, pilot family].
Usage: python m1.py --cache <cache> --out <folder> [--families Visibility,Obstruction] [--threshold 0.5]
"""
import argparse
import warnings
from pathlib import Path
import numpy as np
from sklearn.ensemble import HistGradientBoostingClassifier
from dataset import FAMILIES
STATS = ("median", "q90", "min", "max")
def station_rows(d):
"""A station of the cache -> (X [T, F], names): its station features, then each sensor feature aggregated over
the targets and over the references of the compensation."""
xs, kinds = d["xs"], d["kinds"]
cols, names = [d["xc"]], [f"station.{n}" for n in d["xc_names"]]
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning) # all-NaN slices: the aggregate is NaN, which the model reads
for group, mask in (("targets", kinds != "global_ref"), ("refs", kinds == "global_ref")):
g = xs[:, mask, :]
for stat in STATS:
if not mask.any():
v = np.full((len(xs), xs.shape[2]), np.nan, np.float32)
elif stat == "q90":
v = np.nanquantile(g, 0.9, axis=1)
else:
v = getattr(np, "nan" + stat)(g, axis=1)
cols.append(v)
names += [f"{group}.{stat}.{n}" for n in d["xs_names"]]
x = np.concatenate(cols, axis=1).astype(np.float32)
x[~np.isfinite(x)] = np.nan # inf of a log ratio: missing
return x, names
def fit(x, y, w=None):
model = HistGradientBoostingClassifier(max_iter=300, learning_rate=0.05, max_leaf_nodes=15, l2_regularization=1.0,
random_state=0)
return model.fit(x, y, sample_weight=None if w is None else w * len(w) / w.sum())
def split_time(a, data, rows, families):
"""Every station seen in training: does the information lie in the features, apart from moving to a new station?"""
from evaluate import counts, show, total
cut = {s: int(0.7 * len(r)) for s, r in rows.items()}
x = np.concatenate([rows[s][:cut[s]] for s in data])
base, new = {}, {}
yhat = {s: np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:].copy() for s in data}
for f in families:
k = FAMILIES.index(f)
y = np.concatenate([data[s]["y"][:cut[s], k] for s in data])
w = np.concatenate([np.full(cut[s], 1 / cut[s]) for s in data]) if a.balance else None
model = fit(x, y, w)
for s in data:
yhat[s][:, k] = model.predict_proba(rows[s][cut[s]:])[:, 1] >= a.threshold
for s in data:
y = data[s]["y"][cut[s]:]
base[s] = counts(y, np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:])
new[s] = counts(y, yhat[s])
keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731
show("Last 30 % of every station", keep(total(base)), keep(total(new)))
def main():
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("--cache", required=True)
p.add_argument("--out", required=True)
p.add_argument("--families", default="Visibility,Obstruction")
p.add_argument("--threshold", type=float, default=0.5)
p.add_argument("--balance", action="store_true", help="each station weighs the same in training")
p.add_argument("--split-time", action="store_true",
help="diagnosis: train on the first 70 %% of every station, score the last 30 %% (nothing saved)")
a = p.parse_args()
families = a.families.split(",")
paths = sorted(Path(a.cache).glob("*.npz"))
data = {q.stem: np.load(q) for q in paths}
rows = {s: station_rows(d)[0] for s, d in data.items()}
if a.split_time:
return split_time(a, data, rows, families)
out = Path(a.out)
out.mkdir(parents=True, exist_ok=True)
for test in data:
train = [s for s in data if s != test]
x = np.concatenate([rows[s] for s in train])
keep = ~np.isnan(x).all(axis=0) # the image quality, when the only station that has it is the tested one
x, xt = x[:, keep], rows[test][:, keep]
yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy()
proba = np.zeros((len(xt), len(families)), np.float32)
for j, f in enumerate(families):
k = FAMILIES.index(f)
y = np.concatenate([data[s]["y"][:, k] for s in train])
w = np.concatenate([np.full(len(rows[s]), 1 / len(rows[s])) for s in train]) if a.balance else None
proba[:, j] = fit(x, y, w).predict_proba(xt)[:, 1]
yhat[:, k] = proba[:, j] >= a.threshold
np.savez_compressed(out / f"{test}.npz", yhat=yhat, proba=proba, families=np.array(families))
print(test, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True)
if __name__ == "__main__":
main()
+1
View File
@@ -1,2 +1,3 @@
numpy
pandas
scikit-learn