From 5337dd0dac940c5d75c23b5d306bd36d017ccb5e Mon Sep 17 00:00:00 2001 From: maxime Date: Tue, 17 Nov 2026 09:51:56 +0100 Subject: [PATCH] Step 5: pre-training on the rules' outputs (pretrain.py, m2.py --pretrain); regressions as detectors/bench/compare.py; the reader from the tools repository The detectors were simplified on 2026-11-16: the reader moved to aimsight-tools (scripts/data.py, put on the path by categories.py) and a regression is now a Total losing more than 1 point, or a family of a station losing more than 5 points and more than 20 instants. Co-Authored-By: Claude Opus 5.5 --- dataset.py | 11 +++--- evaluate.py | 34 +++++++++++------- m2.py | 92 ++++++++++++++++++++++++++++++++---------------- pretrain.py | 53 ++++++++++++++++++++++++++++ test_dataset.py | 1 + test_evaluate.py | 7 ++-- 6 files changed, 147 insertions(+), 51 deletions(-) create mode 100644 pretrain.py diff --git a/dataset.py b/dataset.py index baed5cf..c3b1d81 100644 --- a/dataset.py +++ b/dataset.py @@ -155,9 +155,10 @@ def station_features(f, kinds, grid, camera, temperature, measurements): return names, np.stack([np.asarray(c[n], float) for n in names], axis=-1) -def build(folder, labels, start, end, data): - """A station folder, its labels and the range [start, end) (iso) -> dict of arrays.""" - measurements = data.load(folder) +def build(folder, labels, start, end, data, measurements=None): + """A station folder, its labels and the range [start, end) (iso) -> dict of arrays. measurements: the station + already read (data.load), when several ranges are built from it.""" + measurements = measurements or data.load(folder) kinds = {s: ("global_ref" if data.is_reference(v) else "ref" if next((m["pattern_type"] for m in v if m["pattern_type"]), "") == "ref" else "target") for s, v in measurements.items()} @@ -178,7 +179,7 @@ def build(folder, labels, start, end, data): def main(): p = argparse.ArgumentParser(usage=__doc__) - p.add_argument("--detectors", required=True, help="the detectors repository (scripts/data.py, bench/)") + p.add_argument("--detectors", required=True, help="the detectors repository (scripts/, bench/; the reader is tools/scripts/data.py)") p.add_argument("--data", required=True, help="folder of the station folders") p.add_argument("--labels", required=True, help="folder of the .jsonl") p.add_argument("--out", required=True) @@ -187,9 +188,9 @@ def main(): p.add_argument("--to", dest="end") a = p.parse_args() sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")] + from score import read_labels # noqa: E402 (puts the tools repository's reader on the path) import data # noqa: E402 the reading of the benchmark from regression import label_range # noqa: E402 the labelled range, as the benchmark - from score import read_labels # noqa: E402 out = Path(a.out) out.mkdir(parents=True, exist_ok=True) for path in sorted(Path(a.labels).glob("*.jsonl")): diff --git a/evaluate.py b/evaluate.py index 303c9a6..281a60c 100644 --- a/evaluate.py +++ b/evaluate.py @@ -5,7 +5,7 @@ predictions stored per instant, so that a model and the rules are scored on the 3 days of history) -> /rules/.npz (yhat [instant, family]); checks that the counts are those of bench/score.py family_scores, exactly. score: the predictions of a folder (.npz with yhat) against the labels, beside the rules, with the - regressions (a family of a station losing more than 2 points of found or correct, as detectors/CLAUDE.md). + regressions of detectors/bench/compare.py (Total: more than 1 point; a station: more than 5 points and 20 instants). Usage: python evaluate.py rules --detectors --data --labels --cache python evaluate.py score --cache @@ -18,7 +18,7 @@ import numpy as np from dataset import FAMILIES -REGRESSION = 2 # points, detectors/CLAUDE.md +TOTAL_POINTS, POINTS, INSTANTS = 1, 5, 20 # a regression, as detectors/bench/compare.py def counts(y, yhat): @@ -44,15 +44,19 @@ def pct(both, n): return 100 * both / n if n else None -def regressions(base, new): - """{family: counts} before and after -> [(family, "found"|"correct", before %, after %)] losing > REGRESSION.""" +def regressions(base, new, whole=False): + """{family: counts} before and after -> [(family, "found"|"correct", before %, after %, lost instants)], as + detectors/bench/compare.py: a Total (whole) losing more than 1 point; a family of a station losing more than 5 + points and more than 20 instants (labelled instants no longer flagged, or wrongly flagged instants added).""" out = [] for f in sorted(set(base) | set(new)): - b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0]) - for name, i in (("found", 0), ("correct", 1)): - pb, pn = pct(b[2], b[i]), pct(n[2], n[i]) - if pb is not None and pn is not None and pb - pn > REGRESSION: # nothing flagged: no correct to lose - out.append((f, name, pb, pn)) + (l0, f0, b0), (l1, f1, b1) = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0]) + for name, pb, pn, lost in (("found", pct(b0, l0), pct(b1, l1), b0 - b1), + ("correct", pct(b0, f0), pct(b1, f1), (f1 - b1) - (f0 - b0))): + if pb is None or pn is None or round(pb) == round(pn): # nothing flagged: no correct to lose + continue + if pb - pn > TOTAL_POINTS if whole else pb - pn > POINTS and lost > INSTANTS: + out.append((f, name, pb, pn, lost)) return out @@ -90,10 +94,12 @@ def report_time_split(cache, data, proba, families, threshold=0.5): yhat[:, FAMILIES.index(f)] = proba[s][:, j] >= threshold keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731 base[s], new[s] = keep(counts(d["y"][cut:], rules_yhat)), keep(counts(d["y"][cut:], yhat)) - for f, name, pb, pn in regressions(base[s], new[s]): - print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %" + for f, name, pb, pn, lost in regressions(base[s], new[s]): + print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants" f" (rules {base[s][f]}, model {new[s].get(f)}: labelled, flagged, both)") show(f"Last {round(100 * (1 - CUT))} % of every station", total(base), total(new)) + for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True): + print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %") return base, new @@ -129,10 +135,12 @@ def score_folder(a): base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"]) new[path.stem] = counts(y, np.load(p)["yhat"]) show(path.stem, base[path.stem], new[path.stem]) - for f, name, pb, pn in regressions(base[path.stem], new[path.stem]): - print(f" regression: {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}") + for f, name, pb, pn, lost in regressions(base[path.stem], new[path.stem]): + print(f" regression: {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants") print() show("Total", total(base), total(new)) + for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True): + print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %") if __name__ == "__main__": diff --git a/m2.py b/m2.py index 97a94d8..e1caa35 100644 --- a/m2.py +++ b/m2.py @@ -4,12 +4,15 @@ station's; per family, an attention over the sensors pools their scores into the the sensors (step 6). Inputs: every feature through sign(x) log(1 + |x|) (jumps reach thousands of noises), NaN set to 0 with a presence mask. -Training: binary cross-entropy per family on the reviewed labels (fine-tuning only: the pre-training is step 5), each -station drawn as often as the others. +Training: binary cross-entropy per family on the reviewed labels, each station drawn as often as the others. +--pretrain : first the six families the rules flag, on every station and month, labelled or not; +then the same fine-tuning on the reviewed labels. Never on what is tested: time split, the instants from each labelled +station's cut on; station left out, the whole station. default: the criterion, trained on the first 70 % of every station, scored on the last 30 % (evaluate.CUT); --leave-out: the indicator, each station predicted by the model trained on the others; one .npz per station in --out, with yhat [instant, family] (the rules outside the pilot families), proba and attention [instant, sensor, family]. -Usage: python m2.py --cache [--leave-out --out ] [--families Visibility,Obstruction] [--epochs 200 --seeds 5] +Usage: python m2.py --cache [--pretrain ] [--leave-out --out ] [--families Visibility,Obstruction] + [--epochs 200 --seeds 5] """ import argparse from pathlib import Path @@ -25,11 +28,12 @@ BATCHES, BATCH = 8, 128 # per station and per epoch: every station weighs the s def prepare(d): - """A station of the cache -> tensors xs [T, S, 2 Fs], xc [T, 2 Fc]: the squashed features and their presence.""" + """A station of the cache -> tensors xs [T, S, 2 Fs], xc [T, 2 Fc]: the squashed features and their presence, in + float16 to hold every month in memory (the squashed values stay below 20); cast per batch.""" def squash(x): m = np.isfinite(x) v = np.where(m, x, 0) - return np.concatenate([np.sign(v) * np.log1p(np.abs(v)), m], axis=-1).astype(np.float32) + return np.concatenate([np.sign(v) * np.log1p(np.abs(v)), m], axis=-1).astype(np.float16) return torch.from_numpy(squash(d["xs"])), torch.from_numpy(squash(d["xc"])) @@ -48,34 +52,54 @@ class MIL(nn.Module): return (attention * self.score(h)).sum(dim=1), attention -def train(parts, k_index, epochs, seed=0, proportional=False): - """parts: [(xs, xc, y [T, K])], one per station -> the trained model. Each station drawn as often as the others, or - in proportion to its instants.""" +def train(parts, k_index, epochs, seed=0, weights=None, model=None): + """parts: [(xs, xc, y [T, K])] -> the model trained on the families k_index, from model if given (fine-tuning). + weights: the chance of each part (default: each part as often as the others); epochs: BATCHES batches per part.""" torch.manual_seed(seed) rng = np.random.default_rng(seed) - sizes = np.array([len(p[2]) for p in parts], float) - model = MIL(parts[0][0].shape[-1], parts[0][1].shape[-1], len(k_index)) + p = np.full(len(parts), 1 / len(parts)) if weights is None else np.asarray(weights) / np.sum(weights) + model = model or MIL(parts[0][0].shape[-1], parts[0][1].shape[-1], len(FAMILIES)) opt = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4) loss = nn.BCEWithLogitsLoss() model.train() - for _ in range(epochs): - for _ in range(BATCHES): - order = (rng.choice(len(parts), len(parts), p=sizes / sizes.sum()) if proportional - else rng.permutation(len(parts))) - for i in order: - xs, xc, y = parts[i] - b = torch.from_numpy(rng.integers(0, len(y), BATCH)) - logits, _ = model(xs[b], xc[b]) - opt.zero_grad() - loss(logits, y[b][:, k_index]).backward() - opt.step() + for i in rng.choice(len(parts), epochs * BATCHES * len(parts), p=p): + xs, xc, y = parts[i] + b = torch.from_numpy(rng.integers(0, len(y), BATCH)) + logits, _ = model(xs[b].float(), xc[b].float()) + opt.zero_grad() + loss(logits[:, k_index], y[b][:, k_index].float()).backward() + opt.step() return model.eval() @torch.no_grad() -def predict(model, xs, xc): - out = [model(xs[i:i + 1024], xc[i:i + 1024]) for i in range(0, len(xs), 1024)] - return torch.sigmoid(torch.cat([o[0] for o in out])).numpy(), torch.cat([o[1] for o in out]).numpy() +def predict(model, xs, xc, k_index): + out = [model(xs[i:i + 1024].float(), xc[i:i + 1024].float()) for i in range(0, len(xs), 1024)] + return (torch.sigmoid(torch.cat([o[0] for o in out]))[:, k_index].numpy(), + torch.cat([o[1] for o in out])[:, :, k_index].numpy()) + + +def pretraining_parts(folder, keep): + """The chunks of pretrain.py -> [(station, parts)]; keep(station, instants) -> the boolean mask of usable instants.""" + out = {} + for q in sorted(Path(folder).glob("*.npz")): + d = np.load(q) + station = q.stem.rsplit("_", 1)[0] + m = keep(station, d["instants"]) + if m.any(): + xs, xc = prepare(d) + out.setdefault(station, []).append((xs[m], xc[m], torch.from_numpy(d["yhat"][m]))) + return out + + +def pretrain_then_train(pre, parts, k_index, epochs, seed): + """pre: {station: [chunk parts]}, each station drawn as often as the others, its chunks by their size.""" + model = None + if pre: + chunks = [c for cs in pre.values() for c in cs] + weights = [len(c[2]) / sum(len(x[2]) for x in cs) for cs in pre.values() for c in cs] + model = train(chunks, torch.arange(len(FAMILIES)), max(1, epochs * len(parts) // len(chunks)), seed, weights) + return train(parts, k_index, epochs, seed, model=model) def main(): @@ -85,32 +109,38 @@ def main(): p.add_argument("--epochs", type=int, default=200) p.add_argument("--seeds", type=int, default=5, help="models averaged: one run alone varies by 10 to 20 points") p.add_argument("--threshold", type=float, default=0.5) - p.add_argument("--proportional", action="store_true", help="stations drawn in proportion to their instants") + p.add_argument("--pretrain", help="the folder of pretrain.py: pre-training on the rules' outputs first") p.add_argument("--leave-out", action="store_true") p.add_argument("--out") a = p.parse_args() families = a.families.split(",") k_index = torch.tensor([FAMILIES.index(f) for f in families]) data = {q.stem: np.load(q) for q in sorted(Path(a.cache).glob("*.npz"))} - tensors = {s: (*prepare(d), torch.from_numpy(d["y"].astype(np.float32))) for s, d in data.items()} + tensors = {s: (*prepare(d), torch.from_numpy(d["y"])) for s, d in data.items()} - def ensemble(parts): - return [train(parts, k_index, a.epochs, seed, a.proportional) for seed in range(a.seeds)] + def ensemble(parts, pre): + return [pretrain_then_train(pre, parts, k_index, a.epochs, seed) for seed in range(a.seeds)] def averaged(models, xs, xc): - out = [predict(m, xs, xc) for m in models] + out = [predict(m, xs, xc, k_index) for m in models] return np.mean([o[0] for o in out], axis=0), np.mean([o[1] for o in out], axis=0) + def pre(keep): + return pretraining_parts(a.pretrain, keep) if a.pretrain else {} + if not a.leave_out: cut = {s: int(CUT * len(d["y"])) for s, d in data.items()} - models = ensemble([tuple(t[:cut[s]] for t in tensors[s]) for s in data]) + first_tested = {s: data[s]["instants"][cut[s]] for s in data} + models = ensemble([tuple(t[:cut[s]] for t in tensors[s]) for s in data], + pre(lambda s, inst: inst < first_tested[s] if s in first_tested else np.ones(len(inst), bool))) proba = {s: averaged(models, tensors[s][0][cut[s]:], tensors[s][1][cut[s]:])[0] for s in data} report_time_split(a.cache, data, proba, families, a.threshold) return out = Path(a.out) out.mkdir(parents=True, exist_ok=True) for test in data: - proba, attention = averaged(ensemble([tensors[s] for s in data if s != test]), *tensors[test][:2]) + models = ensemble([tensors[s] for s in data if s != test], pre(lambda s, inst: np.full(len(inst), s != test))) + proba, attention = averaged(models, *tensors[test][:2]) yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy() for j, f in enumerate(families): yhat[:, FAMILIES.index(f)] = proba[:, j] >= a.threshold diff --git a/pretrain.py b/pretrain.py new file mode 100644 index 0000000..450101c --- /dev/null +++ b/pretrain.py @@ -0,0 +1,53 @@ +"""The pre-training set (step 5): every station of the data folder, labelled or not, over its whole range, in chunks of +two weeks: the features of dataset.py and, as targets, the families the rules flag at each instant (bench/score.py +events(), each chunk classified with its 3 days of history). No label is read. +-> /_.npz with instants, kinds, xs, xc, yhat [instant, family]; an existing chunk is kept. +Usage: python pretrain.py --detectors --data --out [--stations A,B] +""" +import argparse +import sys +from datetime import timedelta +from pathlib import Path + +import numpy as np + +from dataset import HISTORY, build +from evaluate import events_to_yhat + +CHUNK = timedelta(days=14) + + +def main(): + p = argparse.ArgumentParser(usage=__doc__) + for k in ("detectors", "data", "out"): + p.add_argument("--" + k, required=True) + p.add_argument("--stations") + a = p.parse_args() + sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")] + import score # noqa: E402 (puts the tools repository's reader on the path) + import data # noqa: E402 + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + for folder in sorted(Path(a.data).iterdir()): + if not (folder / "displacement").is_dir() or (a.stations and folder.name not in a.stations.split(",")): + continue + measurements = data.load(str(folder)) + times = [m["t"] for v in measurements.values() for m in v] + t = min(times).replace(hour=0, minute=0, second=0, microsecond=0) + HISTORY # the first days: history only + while t <= max(times): + path = out / f"{folder.name}_{t:%Y%m%d}.npz" + start, end = data.iso(t), data.iso(t + CHUNK) + t += CHUNK + if path.exists(): + continue + d = build(str(folder), [], start, end, data, measurements) + if not len(d["instants"]): + continue + yhat = events_to_yhat(score.events(measurements, start, end, str(folder)), list(d["instants"])) + np.savez_compressed(path, instants=d["instants"], kinds=d["kinds"], xs=d["xs"], xc=d["xc"], yhat=yhat) + print(f"{path.stem}: {len(d['instants'])} instants, {len(d['kinds'])} sensors, flagged " + f"{dict(zip(d['families'].tolist(), yhat.sum(0).tolist()))}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/test_dataset.py b/test_dataset.py index 7ee1054..3de0d08 100644 --- a/test_dataset.py +++ b/test_dataset.py @@ -6,6 +6,7 @@ from pathlib import Path import numpy as np sys.path.insert(0, str(Path(sys.argv[1]) / "scripts")) +import categories # noqa: E402,F401 puts the tools repository's reader on the path import data # noqa: E402 import dataset # noqa: E402 diff --git a/test_evaluate.py b/test_evaluate.py index f33f161..7298bcf 100644 --- a/test_evaluate.py +++ b/test_evaluate.py @@ -13,7 +13,10 @@ y[1:, 1] = 1 y[2, 0] = 1 c = evaluate.counts(y, yhat) assert c == {"Visibility": [1, 1, 0], "Obstruction": [2, 2, 2]}, c -assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}) == [("Obstruction", "found", 50, 40)] -assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 10, 5]}) == [] +assert evaluate.regressions({"Obstruction": [100, 100, 50]}, {"Obstruction": [100, 30, 30]}) == [] # 20 instants: not more +assert evaluate.regressions({"Obstruction": [100, 100, 50]}, {"Obstruction": [100, 29, 29]}) == [("Obstruction", "found", 50, 29, 21)] +assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}) == [] # 10 points, 1 instant: noise +assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}, whole=True) == [("Obstruction", "found", 50, 40, 1)] +assert evaluate.regressions({"Visibility": [100, 50, 50]}, {"Visibility": [100, 100, 71]}) == [("Visibility", "correct", 100, 71, 29)] assert evaluate.regressions({"Obstruction": [0, 15, 0]}, {}) == [] # false alarms gone: not a regression print("ok")