Step 3: M1, gradient boosting on station aggregates (leave one station out, and a time split)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c04cbc90bb
commit
f93cdf3cd1
3 files changed
+126
No files matched your search
@@ -29,6 +29,23 @@ uv venv .venv && uv pip install -p .venv -r requirements.txt
|
|||||||
|
|
||||||
`evaluate.py rules` (step 2, ~20 min, once per change of the detectors): the rules' predictions per instant in `cache/rules/`, and the check that their counts are those of `bench/score.py`, station by station (exit 1 otherwise). `evaluate.py score`: a folder of `<STATION>.npz` with `yhat` [instant, family] against the labels, beside the rules, with the regressions of `detectors/CLAUDE.md` (more than 2 points of found or correct lost on a family of a station).
|
`evaluate.py rules` (step 2, ~20 min, once per change of the detectors): the rules' predictions per instant in `cache/rules/`, and the check that their counts are those of `bench/score.py`, station by station (exit 1 otherwise). `evaluate.py score`: a folder of `<STATION>.npz` with `yhat` [instant, family] against the labels, beside the rules, with the regressions of `detectors/CLAUDE.md` (more than 2 points of found or correct lost on a family of a station).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
.venv/bin/python m1.py --cache cache --out runs/m1 --balance && .venv/bin/python evaluate.py score runs/m1 --cache cache
|
||||||
|
.venv/bin/python m1.py --cache cache --out - --split-time --balance
|
||||||
|
```
|
||||||
|
|
||||||
|
`m1.py` (step 3, a few minutes): gradient boosting on the station's features and the aggregates of its sensors', Visibility and Obstruction, leaving one station out; `--split-time`, the diagnosis on the last 30 % of every station, all stations seen in training. First results (2026-11-16, found / correct, total):
|
||||||
|
|
||||||
|
| | Visibility | Obstruction |
|
||||||
|
|---|---|---|
|
||||||
|
| rules, all stations | 47 / 77 % | 26 / 28 % |
|
||||||
|
| M1, station left out | 44 / 62 % | 9 / 7 % |
|
||||||
|
| M1, station left out, `--balance` | 46 / 70 % | 21 / 23 % |
|
||||||
|
| rules, last 30 % | 49 / 77 % | 15 / 41 % |
|
||||||
|
| M1, last 30 % (`--split-time`) | 58 / 87 % | 39 / 75 % |
|
||||||
|
|
||||||
|
The information is in the features: with every station seen, M1 beats the rules; moving to a new station is what fails (Obstruction is labelled on three stations only).
|
||||||
|
|
||||||
## Contracts
|
## Contracts
|
||||||
|
|
||||||
- Reads the measurements through `detectors/scripts/data.py`, the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments.
|
- Reads the measurements through `detectors/scripts/data.py`, the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments.
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
"""M1, the first rung of the pilot: gradient boosting on what the station does at each instant (its features and the
|
||||||
|
aggregates of its sensors' features), one classifier per family of the pilot, leaving one station out.
|
||||||
|
|
||||||
|
Each station is predicted by the classifier trained on the others; the families outside the pilot keep the rules'
|
||||||
|
predictions (cache/rules/), so that `evaluate.py score` compares the pilot families only. No sensor attribution: that is
|
||||||
|
M2. One <STATION>.npz per station in --out, with yhat [instant, family] and proba [instant, pilot family].
|
||||||
|
Usage: python m1.py --cache <cache> --out <folder> [--families Visibility,Obstruction] [--threshold 0.5]
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import warnings
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from sklearn.ensemble import HistGradientBoostingClassifier
|
||||||
|
|
||||||
|
from dataset import FAMILIES
|
||||||
|
|
||||||
|
STATS = ("median", "q90", "min", "max")
|
||||||
|
|
||||||
|
|
||||||
|
def station_rows(d):
|
||||||
|
"""A station of the cache -> (X [T, F], names): its station features, then each sensor feature aggregated over
|
||||||
|
the targets and over the references of the compensation."""
|
||||||
|
xs, kinds = d["xs"], d["kinds"]
|
||||||
|
cols, names = [d["xc"]], [f"station.{n}" for n in d["xc_names"]]
|
||||||
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter("ignore", RuntimeWarning) # all-NaN slices: the aggregate is NaN, which the model reads
|
||||||
|
for group, mask in (("targets", kinds != "global_ref"), ("refs", kinds == "global_ref")):
|
||||||
|
g = xs[:, mask, :]
|
||||||
|
for stat in STATS:
|
||||||
|
if not mask.any():
|
||||||
|
v = np.full((len(xs), xs.shape[2]), np.nan, np.float32)
|
||||||
|
elif stat == "q90":
|
||||||
|
v = np.nanquantile(g, 0.9, axis=1)
|
||||||
|
else:
|
||||||
|
v = getattr(np, "nan" + stat)(g, axis=1)
|
||||||
|
cols.append(v)
|
||||||
|
names += [f"{group}.{stat}.{n}" for n in d["xs_names"]]
|
||||||
|
x = np.concatenate(cols, axis=1).astype(np.float32)
|
||||||
|
x[~np.isfinite(x)] = np.nan # inf of a log ratio: missing
|
||||||
|
return x, names
|
||||||
|
|
||||||
|
|
||||||
|
def fit(x, y, w=None):
|
||||||
|
model = HistGradientBoostingClassifier(max_iter=300, learning_rate=0.05, max_leaf_nodes=15, l2_regularization=1.0,
|
||||||
|
random_state=0)
|
||||||
|
return model.fit(x, y, sample_weight=None if w is None else w * len(w) / w.sum())
|
||||||
|
|
||||||
|
|
||||||
|
def split_time(a, data, rows, families):
|
||||||
|
"""Every station seen in training: does the information lie in the features, apart from moving to a new station?"""
|
||||||
|
from evaluate import counts, show, total
|
||||||
|
cut = {s: int(0.7 * len(r)) for s, r in rows.items()}
|
||||||
|
x = np.concatenate([rows[s][:cut[s]] for s in data])
|
||||||
|
base, new = {}, {}
|
||||||
|
yhat = {s: np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:].copy() for s in data}
|
||||||
|
for f in families:
|
||||||
|
k = FAMILIES.index(f)
|
||||||
|
y = np.concatenate([data[s]["y"][:cut[s], k] for s in data])
|
||||||
|
w = np.concatenate([np.full(cut[s], 1 / cut[s]) for s in data]) if a.balance else None
|
||||||
|
model = fit(x, y, w)
|
||||||
|
for s in data:
|
||||||
|
yhat[s][:, k] = model.predict_proba(rows[s][cut[s]:])[:, 1] >= a.threshold
|
||||||
|
for s in data:
|
||||||
|
y = data[s]["y"][cut[s]:]
|
||||||
|
base[s] = counts(y, np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:])
|
||||||
|
new[s] = counts(y, yhat[s])
|
||||||
|
keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731
|
||||||
|
show("Last 30 % of every station", keep(total(base)), keep(total(new)))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
p = argparse.ArgumentParser(usage=__doc__)
|
||||||
|
p.add_argument("--cache", required=True)
|
||||||
|
p.add_argument("--out", required=True)
|
||||||
|
p.add_argument("--families", default="Visibility,Obstruction")
|
||||||
|
p.add_argument("--threshold", type=float, default=0.5)
|
||||||
|
p.add_argument("--balance", action="store_true", help="each station weighs the same in training")
|
||||||
|
p.add_argument("--split-time", action="store_true",
|
||||||
|
help="diagnosis: train on the first 70 %% of every station, score the last 30 %% (nothing saved)")
|
||||||
|
a = p.parse_args()
|
||||||
|
families = a.families.split(",")
|
||||||
|
paths = sorted(Path(a.cache).glob("*.npz"))
|
||||||
|
data = {q.stem: np.load(q) for q in paths}
|
||||||
|
rows = {s: station_rows(d)[0] for s, d in data.items()}
|
||||||
|
if a.split_time:
|
||||||
|
return split_time(a, data, rows, families)
|
||||||
|
out = Path(a.out)
|
||||||
|
out.mkdir(parents=True, exist_ok=True)
|
||||||
|
for test in data:
|
||||||
|
train = [s for s in data if s != test]
|
||||||
|
x = np.concatenate([rows[s] for s in train])
|
||||||
|
keep = ~np.isnan(x).all(axis=0) # the image quality, when the only station that has it is the tested one
|
||||||
|
x, xt = x[:, keep], rows[test][:, keep]
|
||||||
|
yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy()
|
||||||
|
proba = np.zeros((len(xt), len(families)), np.float32)
|
||||||
|
for j, f in enumerate(families):
|
||||||
|
k = FAMILIES.index(f)
|
||||||
|
y = np.concatenate([data[s]["y"][:, k] for s in train])
|
||||||
|
w = np.concatenate([np.full(len(rows[s]), 1 / len(rows[s])) for s in train]) if a.balance else None
|
||||||
|
proba[:, j] = fit(x, y, w).predict_proba(xt)[:, 1]
|
||||||
|
yhat[:, k] = proba[:, j] >= a.threshold
|
||||||
|
np.savez_compressed(out / f"{test}.npz", yhat=yhat, proba=proba, families=np.array(families))
|
||||||
|
print(test, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -1,2 +1,3 @@
|
|||||||
numpy
|
numpy
|
||||||
pandas
|
pandas
|
||||||
|
scikit-learn
|
||||||
Reference in new issue
Block a user