Step 3: M1, gradient boosting on station aggregates (leave one station out, and a time split)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c04cbc90bb
commit
f93cdf3cd1
3 files changed
+126
No files matched your search
@@ -29,6 +29,23 @@ uv venv .venv && uv pip install -p .venv -r requirements.txt
|
||||
|
||||
`evaluate.py rules` (step 2, ~20 min, once per change of the detectors): the rules' predictions per instant in `cache/rules/`, and the check that their counts are those of `bench/score.py`, station by station (exit 1 otherwise). `evaluate.py score`: a folder of `<STATION>.npz` with `yhat` [instant, family] against the labels, beside the rules, with the regressions of `detectors/CLAUDE.md` (more than 2 points of found or correct lost on a family of a station).
|
||||
|
||||
```bash
|
||||
.venv/bin/python m1.py --cache cache --out runs/m1 --balance && .venv/bin/python evaluate.py score runs/m1 --cache cache
|
||||
.venv/bin/python m1.py --cache cache --out - --split-time --balance
|
||||
```
|
||||
|
||||
`m1.py` (step 3, a few minutes): gradient boosting on the station's features and the aggregates of its sensors', Visibility and Obstruction, leaving one station out; `--split-time`, the diagnosis on the last 30 % of every station, all stations seen in training. First results (2026-11-16, found / correct, total):
|
||||
|
||||
| | Visibility | Obstruction |
|
||||
|---|---|---|
|
||||
| rules, all stations | 47 / 77 % | 26 / 28 % |
|
||||
| M1, station left out | 44 / 62 % | 9 / 7 % |
|
||||
| M1, station left out, `--balance` | 46 / 70 % | 21 / 23 % |
|
||||
| rules, last 30 % | 49 / 77 % | 15 / 41 % |
|
||||
| M1, last 30 % (`--split-time`) | 58 / 87 % | 39 / 75 % |
|
||||
|
||||
The information is in the features: with every station seen, M1 beats the rules; moving to a new station is what fails (Obstruction is labelled on three stations only).
|
||||
|
||||
## Contracts
|
||||
|
||||
- Reads the measurements through `detectors/scripts/data.py`, the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments.
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
"""M1, the first rung of the pilot: gradient boosting on what the station does at each instant (its features and the
|
||||
aggregates of its sensors' features), one classifier per family of the pilot, leaving one station out.
|
||||
|
||||
Each station is predicted by the classifier trained on the others; the families outside the pilot keep the rules'
|
||||
predictions (cache/rules/), so that `evaluate.py score` compares the pilot families only. No sensor attribution: that is
|
||||
M2. One <STATION>.npz per station in --out, with yhat [instant, family] and proba [instant, pilot family].
|
||||
Usage: python m1.py --cache <cache> --out <folder> [--families Visibility,Obstruction] [--threshold 0.5]
|
||||
"""
|
||||
import argparse
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from sklearn.ensemble import HistGradientBoostingClassifier
|
||||
|
||||
from dataset import FAMILIES
|
||||
|
||||
STATS = ("median", "q90", "min", "max")
|
||||
|
||||
|
||||
def station_rows(d):
|
||||
"""A station of the cache -> (X [T, F], names): its station features, then each sensor feature aggregated over
|
||||
the targets and over the references of the compensation."""
|
||||
xs, kinds = d["xs"], d["kinds"]
|
||||
cols, names = [d["xc"]], [f"station.{n}" for n in d["xc_names"]]
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", RuntimeWarning) # all-NaN slices: the aggregate is NaN, which the model reads
|
||||
for group, mask in (("targets", kinds != "global_ref"), ("refs", kinds == "global_ref")):
|
||||
g = xs[:, mask, :]
|
||||
for stat in STATS:
|
||||
if not mask.any():
|
||||
v = np.full((len(xs), xs.shape[2]), np.nan, np.float32)
|
||||
elif stat == "q90":
|
||||
v = np.nanquantile(g, 0.9, axis=1)
|
||||
else:
|
||||
v = getattr(np, "nan" + stat)(g, axis=1)
|
||||
cols.append(v)
|
||||
names += [f"{group}.{stat}.{n}" for n in d["xs_names"]]
|
||||
x = np.concatenate(cols, axis=1).astype(np.float32)
|
||||
x[~np.isfinite(x)] = np.nan # inf of a log ratio: missing
|
||||
return x, names
|
||||
|
||||
|
||||
def fit(x, y, w=None):
|
||||
model = HistGradientBoostingClassifier(max_iter=300, learning_rate=0.05, max_leaf_nodes=15, l2_regularization=1.0,
|
||||
random_state=0)
|
||||
return model.fit(x, y, sample_weight=None if w is None else w * len(w) / w.sum())
|
||||
|
||||
|
||||
def split_time(a, data, rows, families):
|
||||
"""Every station seen in training: does the information lie in the features, apart from moving to a new station?"""
|
||||
from evaluate import counts, show, total
|
||||
cut = {s: int(0.7 * len(r)) for s, r in rows.items()}
|
||||
x = np.concatenate([rows[s][:cut[s]] for s in data])
|
||||
base, new = {}, {}
|
||||
yhat = {s: np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:].copy() for s in data}
|
||||
for f in families:
|
||||
k = FAMILIES.index(f)
|
||||
y = np.concatenate([data[s]["y"][:cut[s], k] for s in data])
|
||||
w = np.concatenate([np.full(cut[s], 1 / cut[s]) for s in data]) if a.balance else None
|
||||
model = fit(x, y, w)
|
||||
for s in data:
|
||||
yhat[s][:, k] = model.predict_proba(rows[s][cut[s]:])[:, 1] >= a.threshold
|
||||
for s in data:
|
||||
y = data[s]["y"][cut[s]:]
|
||||
base[s] = counts(y, np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:])
|
||||
new[s] = counts(y, yhat[s])
|
||||
keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731
|
||||
show("Last 30 % of every station", keep(total(base)), keep(total(new)))
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(usage=__doc__)
|
||||
p.add_argument("--cache", required=True)
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--families", default="Visibility,Obstruction")
|
||||
p.add_argument("--threshold", type=float, default=0.5)
|
||||
p.add_argument("--balance", action="store_true", help="each station weighs the same in training")
|
||||
p.add_argument("--split-time", action="store_true",
|
||||
help="diagnosis: train on the first 70 %% of every station, score the last 30 %% (nothing saved)")
|
||||
a = p.parse_args()
|
||||
families = a.families.split(",")
|
||||
paths = sorted(Path(a.cache).glob("*.npz"))
|
||||
data = {q.stem: np.load(q) for q in paths}
|
||||
rows = {s: station_rows(d)[0] for s, d in data.items()}
|
||||
if a.split_time:
|
||||
return split_time(a, data, rows, families)
|
||||
out = Path(a.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
for test in data:
|
||||
train = [s for s in data if s != test]
|
||||
x = np.concatenate([rows[s] for s in train])
|
||||
keep = ~np.isnan(x).all(axis=0) # the image quality, when the only station that has it is the tested one
|
||||
x, xt = x[:, keep], rows[test][:, keep]
|
||||
yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy()
|
||||
proba = np.zeros((len(xt), len(families)), np.float32)
|
||||
for j, f in enumerate(families):
|
||||
k = FAMILIES.index(f)
|
||||
y = np.concatenate([data[s]["y"][:, k] for s in train])
|
||||
w = np.concatenate([np.full(len(rows[s]), 1 / len(rows[s])) for s in train]) if a.balance else None
|
||||
proba[:, j] = fit(x, y, w).predict_proba(xt)[:, 1]
|
||||
yhat[:, k] = proba[:, j] >= a.threshold
|
||||
np.savez_compressed(out / f"{test}.npz", yhat=yhat, proba=proba, families=np.array(families))
|
||||
print(test, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,2 +1,3 @@
|
||||
numpy
|
||||
pandas
|
||||
scikit-learn
|
||||
Reference in new issue
Block a user