"""M1, the first rung of the pilot: gradient boosting on what the station does at each instant (its features and the aggregates of its sensors' features), one classifier per family of the pilot, leaving one station out. Each station is predicted by the classifier trained on the others; the families outside the pilot keep the rules' predictions (cache/rules/), so that `evaluate.py score` compares the pilot families only. No sensor attribution: that is M2. One .npz per station in --out, with yhat [instant, family] and proba [instant, pilot family]. Usage: python m1.py --cache --out [--families Visibility,Obstruction] [--threshold 0.5] """ import argparse import warnings from pathlib import Path import numpy as np from sklearn.ensemble import HistGradientBoostingClassifier from dataset import FAMILIES STATS = ("median", "q90", "min", "max") def station_rows(d): """A station of the cache -> (X [T, F], names): its station features, then each sensor feature aggregated over the targets and over the references of the compensation.""" xs, kinds = d["xs"], d["kinds"] cols, names = [d["xc"]], [f"station.{n}" for n in d["xc_names"]] with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) # all-NaN slices: the aggregate is NaN, which the model reads for group, mask in (("targets", kinds != "global_ref"), ("refs", kinds == "global_ref")): g = xs[:, mask, :] for stat in STATS: if not mask.any(): v = np.full((len(xs), xs.shape[2]), np.nan, np.float32) elif stat == "q90": v = np.nanquantile(g, 0.9, axis=1) else: v = getattr(np, "nan" + stat)(g, axis=1) cols.append(v) names += [f"{group}.{stat}.{n}" for n in d["xs_names"]] x = np.concatenate(cols, axis=1).astype(np.float32) x[~np.isfinite(x)] = np.nan # inf of a log ratio: missing return x, names def fit(x, y, w=None): model = HistGradientBoostingClassifier(max_iter=300, learning_rate=0.05, max_leaf_nodes=15, l2_regularization=1.0, random_state=0) return model.fit(x, y, sample_weight=None if w is None else w * len(w) / w.sum()) def split_time(a, data, rows, families): """Every station seen in training: does the information lie in the features, apart from moving to a new station?""" from evaluate import counts, regressions, show, total cut = {s: int(0.7 * len(r)) for s, r in rows.items()} x = np.concatenate([rows[s][:cut[s]] for s in data]) base, new = {}, {} yhat = {s: np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:].copy() for s in data} for f in families: k = FAMILIES.index(f) y = np.concatenate([data[s]["y"][:cut[s], k] for s in data]) w = np.concatenate([np.full(cut[s], 1 / cut[s]) for s in data]) if a.balance else None model = fit(x, y, w) for s in data: yhat[s][:, k] = model.predict_proba(rows[s][cut[s]:])[:, 1] >= a.threshold for s in data: y = data[s]["y"][cut[s]:] base[s] = counts(y, np.load(Path(a.cache) / "rules" / f"{s}.npz")["yhat"][cut[s]:]) new[s] = counts(y, yhat[s]) keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731 for s in data: for f, name, pb, pn in regressions(keep(base[s]), keep(new[s])): print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}" f" (rules {base[s][f]}, model {new[s].get(f)}: labelled, flagged, both)") show("Last 30 % of every station", keep(total(base)), keep(total(new))) def main(): p = argparse.ArgumentParser(usage=__doc__) p.add_argument("--cache", required=True) p.add_argument("--out", required=True) p.add_argument("--families", default="Visibility,Obstruction") p.add_argument("--threshold", type=float, default=0.5) p.add_argument("--balance", action="store_true", help="each station weighs the same in training") p.add_argument("--split-time", action="store_true", help="diagnosis: train on the first 70 %% of every station, score the last 30 %% (nothing saved)") a = p.parse_args() families = a.families.split(",") paths = sorted(Path(a.cache).glob("*.npz")) data = {q.stem: np.load(q) for q in paths} rows = {s: station_rows(d)[0] for s, d in data.items()} if a.split_time: return split_time(a, data, rows, families) out = Path(a.out) out.mkdir(parents=True, exist_ok=True) for test in data: train = [s for s in data if s != test] x = np.concatenate([rows[s] for s in train]) keep = ~np.isnan(x).all(axis=0) # the image quality, when the only station that has it is the tested one x, xt = x[:, keep], rows[test][:, keep] yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy() proba = np.zeros((len(xt), len(families)), np.float32) for j, f in enumerate(families): k = FAMILIES.index(f) y = np.concatenate([data[s]["y"][:, k] for s in train]) w = np.concatenate([np.full(len(rows[s]), 1 / len(rows[s])) for s in train]) if a.balance else None proba[:, j] = fit(x, y, w).predict_proba(xt)[:, 1] yhat[:, k] = proba[:, j] >= a.threshold np.savez_compressed(out / f"{test}.npz", yhat=yhat, proba=proba, families=np.array(families)) print(test, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True) if __name__ == "__main__": main()