Step 5: pre-training on the rules' outputs (pretrain.py, m2.py --pretrain); regressions as detectors/bench/compare.py; the reader from the tools repository

The detectors were simplified on 2026-11-16: the reader moved to aimsight-tools (scripts/data.py, put on the path by
categories.py) and a regression is now a Total losing more than 1 point, or a family of a station losing more than
5 points and more than 20 instants.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 09:51:56 +01:00
1 parent 7312a1c3ca
commit 5337dd0dac
6 files changed
+147 -51

No files matched your search

+6 -5
View File
@@ -155,9 +155,10 @@ def station_features(f, kinds, grid, camera, temperature, measurements):
return names, np.stack([np.asarray(c[n], float) for n in names], axis=-1) return names, np.stack([np.asarray(c[n], float) for n in names], axis=-1)
def build(folder, labels, start, end, data): def build(folder, labels, start, end, data, measurements=None):
"""A station folder, its labels and the range [start, end) (iso) -> dict of arrays.""" """A station folder, its labels and the range [start, end) (iso) -> dict of arrays. measurements: the station
measurements = data.load(folder) already read (data.load), when several ranges are built from it."""
measurements = measurements or data.load(folder)
kinds = {s: ("global_ref" if data.is_reference(v) else kinds = {s: ("global_ref" if data.is_reference(v) else
"ref" if next((m["pattern_type"] for m in v if m["pattern_type"]), "") == "ref" else "target") "ref" if next((m["pattern_type"] for m in v if m["pattern_type"]), "") == "ref" else "target")
for s, v in measurements.items()} for s, v in measurements.items()}
@@ -178,7 +179,7 @@ def build(folder, labels, start, end, data):
def main(): def main():
p = argparse.ArgumentParser(usage=__doc__) p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("--detectors", required=True, help="the detectors repository (scripts/data.py, bench/)") p.add_argument("--detectors", required=True, help="the detectors repository (scripts/, bench/; the reader is tools/scripts/data.py)")
p.add_argument("--data", required=True, help="folder of the station folders") p.add_argument("--data", required=True, help="folder of the station folders")
p.add_argument("--labels", required=True, help="folder of the <STATION>.jsonl") p.add_argument("--labels", required=True, help="folder of the <STATION>.jsonl")
p.add_argument("--out", required=True) p.add_argument("--out", required=True)
@@ -187,9 +188,9 @@ def main():
p.add_argument("--to", dest="end") p.add_argument("--to", dest="end")
a = p.parse_args() a = p.parse_args()
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")] sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
from score import read_labels # noqa: E402 (puts the tools repository's reader on the path)
import data # noqa: E402 the reading of the benchmark import data # noqa: E402 the reading of the benchmark
from regression import label_range # noqa: E402 the labelled range, as the benchmark from regression import label_range # noqa: E402 the labelled range, as the benchmark
from score import read_labels # noqa: E402
out = Path(a.out) out = Path(a.out)
out.mkdir(parents=True, exist_ok=True) out.mkdir(parents=True, exist_ok=True)
for path in sorted(Path(a.labels).glob("*.jsonl")): for path in sorted(Path(a.labels).glob("*.jsonl")):
+21 -13
View File
@@ -5,7 +5,7 @@ predictions stored per instant, so that a model and the rules are scored on the
3 days of history) -> <cache>/rules/<STATION>.npz (yhat [instant, family]); checks that the counts are those 3 days of history) -> <cache>/rules/<STATION>.npz (yhat [instant, family]); checks that the counts are those
of bench/score.py family_scores, exactly. of bench/score.py family_scores, exactly.
score: the predictions of a folder (<STATION>.npz with yhat) against the labels, beside the rules, with the score: the predictions of a folder (<STATION>.npz with yhat) against the labels, beside the rules, with the
regressions (a family of a station losing more than 2 points of found or correct, as detectors/CLAUDE.md). regressions of detectors/bench/compare.py (Total: more than 1 point; a station: more than 5 points and 20 instants).
Usage: python evaluate.py rules --detectors <detectors> --data <data> --labels <labels folder> --cache <cache> Usage: python evaluate.py rules --detectors <detectors> --data <data> --labels <labels folder> --cache <cache>
python evaluate.py score <predictions folder> --cache <cache> python evaluate.py score <predictions folder> --cache <cache>
@@ -18,7 +18,7 @@ import numpy as np
from dataset import FAMILIES from dataset import FAMILIES
REGRESSION = 2 # points, detectors/CLAUDE.md TOTAL_POINTS, POINTS, INSTANTS = 1, 5, 20 # a regression, as detectors/bench/compare.py
def counts(y, yhat): def counts(y, yhat):
@@ -44,15 +44,19 @@ def pct(both, n):
return 100 * both / n if n else None return 100 * both / n if n else None
def regressions(base, new): def regressions(base, new, whole=False):
"""{family: counts} before and after -> [(family, "found"|"correct", before %, after %)] losing > REGRESSION.""" """{family: counts} before and after -> [(family, "found"|"correct", before %, after %, lost instants)], as
detectors/bench/compare.py: a Total (whole) losing more than 1 point; a family of a station losing more than 5
points and more than 20 instants (labelled instants no longer flagged, or wrongly flagged instants added)."""
out = [] out = []
for f in sorted(set(base) | set(new)): for f in sorted(set(base) | set(new)):
b, n = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0]) (l0, f0, b0), (l1, f1, b1) = base.get(f, [0, 0, 0]), new.get(f, [0, 0, 0])
for name, i in (("found", 0), ("correct", 1)): for name, pb, pn, lost in (("found", pct(b0, l0), pct(b1, l1), b0 - b1),
pb, pn = pct(b[2], b[i]), pct(n[2], n[i]) ("correct", pct(b0, f0), pct(b1, f1), (f1 - b1) - (f0 - b0))):
if pb is not None and pn is not None and pb - pn > REGRESSION: # nothing flagged: no correct to lose if pb is None or pn is None or round(pb) == round(pn): # nothing flagged: no correct to lose
out.append((f, name, pb, pn)) continue
if pb - pn > TOTAL_POINTS if whole else pb - pn > POINTS and lost > INSTANTS:
out.append((f, name, pb, pn, lost))
return out return out
@@ -90,10 +94,12 @@ def report_time_split(cache, data, proba, families, threshold=0.5):
yhat[:, FAMILIES.index(f)] = proba[s][:, j] >= threshold yhat[:, FAMILIES.index(f)] = proba[s][:, j] >= threshold
keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731 keep = lambda c: {f: v for f, v in c.items() if f in families} # noqa: E731
base[s], new[s] = keep(counts(d["y"][cut:], rules_yhat)), keep(counts(d["y"][cut:], yhat)) base[s], new[s] = keep(counts(d["y"][cut:], rules_yhat)), keep(counts(d["y"][cut:], yhat))
for f, name, pb, pn in regressions(base[s], new[s]): for f, name, pb, pn, lost in regressions(base[s], new[s]):
print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %" print(f" regression: {s}, {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants"
f" (rules {base[s][f]}, model {new[s].get(f)}: labelled, flagged, both)") f" (rules {base[s][f]}, model {new[s].get(f)}: labelled, flagged, both)")
show(f"Last {round(100 * (1 - CUT))} % of every station", total(base), total(new)) show(f"Last {round(100 * (1 - CUT))} % of every station", total(base), total(new))
for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True):
print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %")
return base, new return base, new
@@ -129,10 +135,12 @@ def score_folder(a):
base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"]) base[path.stem] = counts(y, np.load(Path(a.cache) / "rules" / path.name)["yhat"])
new[path.stem] = counts(y, np.load(p)["yhat"]) new[path.stem] = counts(y, np.load(p)["yhat"])
show(path.stem, base[path.stem], new[path.stem]) show(path.stem, base[path.stem], new[path.stem])
for f, name, pb, pn in regressions(base[path.stem], new[path.stem]): for f, name, pb, pn, lost in regressions(base[path.stem], new[path.stem]):
print(f" regression: {f}, {name} {pb:.0f} % -> {'—' if pn is None else f'{pn:.0f} %'}") print(f" regression: {f}, {name} {pb:.0f} % -> {pn:.0f} %, {lost} instants")
print() print()
show("Total", total(base), total(new)) show("Total", total(base), total(new))
for f, name, pb, pn, _ in regressions(total(base), total(new), whole=True):
print(f" regression: Total, {f}, {name} {pb:.0f} % -> {pn:.0f} %")
if __name__ == "__main__": if __name__ == "__main__":
+61 -31
View File
@@ -4,12 +4,15 @@ station's; per family, an attention over the sensors pools their scores into the
the sensors (step 6). the sensors (step 6).
Inputs: every feature through sign(x) log(1 + |x|) (jumps reach thousands of noises), NaN set to 0 with a presence mask. Inputs: every feature through sign(x) log(1 + |x|) (jumps reach thousands of noises), NaN set to 0 with a presence mask.
Training: binary cross-entropy per family on the reviewed labels (fine-tuning only: the pre-training is step 5), each Training: binary cross-entropy per family on the reviewed labels, each station drawn as often as the others.
station drawn as often as the others. --pretrain <folder of pretrain.py>: first the six families the rules flag, on every station and month, labelled or not;
then the same fine-tuning on the reviewed labels. Never on what is tested: time split, the instants from each labelled
station's cut on; station left out, the whole station.
default: the criterion, trained on the first 70 % of every station, scored on the last 30 % (evaluate.CUT); default: the criterion, trained on the first 70 % of every station, scored on the last 30 % (evaluate.CUT);
--leave-out: the indicator, each station predicted by the model trained on the others; one <STATION>.npz per station --leave-out: the indicator, each station predicted by the model trained on the others; one <STATION>.npz per station
in --out, with yhat [instant, family] (the rules outside the pilot families), proba and attention [instant, sensor, family]. in --out, with yhat [instant, family] (the rules outside the pilot families), proba and attention [instant, sensor, family].
Usage: python m2.py --cache <cache> [--leave-out --out <folder>] [--families Visibility,Obstruction] [--epochs 200 --seeds 5] Usage: python m2.py --cache <cache> [--pretrain <folder>] [--leave-out --out <folder>] [--families Visibility,Obstruction]
[--epochs 200 --seeds 5]
""" """
import argparse import argparse
from pathlib import Path from pathlib import Path
@@ -25,11 +28,12 @@ BATCHES, BATCH = 8, 128 # per station and per epoch: every station weighs the s
def prepare(d): def prepare(d):
"""A station of the cache -> tensors xs [T, S, 2 Fs], xc [T, 2 Fc]: the squashed features and their presence.""" """A station of the cache -> tensors xs [T, S, 2 Fs], xc [T, 2 Fc]: the squashed features and their presence, in
float16 to hold every month in memory (the squashed values stay below 20); cast per batch."""
def squash(x): def squash(x):
m = np.isfinite(x) m = np.isfinite(x)
v = np.where(m, x, 0) v = np.where(m, x, 0)
return np.concatenate([np.sign(v) * np.log1p(np.abs(v)), m], axis=-1).astype(np.float32) return np.concatenate([np.sign(v) * np.log1p(np.abs(v)), m], axis=-1).astype(np.float16)
return torch.from_numpy(squash(d["xs"])), torch.from_numpy(squash(d["xc"])) return torch.from_numpy(squash(d["xs"])), torch.from_numpy(squash(d["xc"]))
@@ -48,34 +52,54 @@ class MIL(nn.Module):
return (attention * self.score(h)).sum(dim=1), attention return (attention * self.score(h)).sum(dim=1), attention
def train(parts, k_index, epochs, seed=0, proportional=False): def train(parts, k_index, epochs, seed=0, weights=None, model=None):
"""parts: [(xs, xc, y [T, K])], one per station -> the trained model. Each station drawn as often as the others, or """parts: [(xs, xc, y [T, K])] -> the model trained on the families k_index, from model if given (fine-tuning).
in proportion to its instants.""" weights: the chance of each part (default: each part as often as the others); epochs: BATCHES batches per part."""
torch.manual_seed(seed) torch.manual_seed(seed)
rng = np.random.default_rng(seed) rng = np.random.default_rng(seed)
sizes = np.array([len(p[2]) for p in parts], float) p = np.full(len(parts), 1 / len(parts)) if weights is None else np.asarray(weights) / np.sum(weights)
model = MIL(parts[0][0].shape[-1], parts[0][1].shape[-1], len(k_index)) model = model or MIL(parts[0][0].shape[-1], parts[0][1].shape[-1], len(FAMILIES))
opt = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4) opt = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4)
loss = nn.BCEWithLogitsLoss() loss = nn.BCEWithLogitsLoss()
model.train() model.train()
for _ in range(epochs): for i in rng.choice(len(parts), epochs * BATCHES * len(parts), p=p):
for _ in range(BATCHES): xs, xc, y = parts[i]
order = (rng.choice(len(parts), len(parts), p=sizes / sizes.sum()) if proportional b = torch.from_numpy(rng.integers(0, len(y), BATCH))
else rng.permutation(len(parts))) logits, _ = model(xs[b].float(), xc[b].float())
for i in order: opt.zero_grad()
xs, xc, y = parts[i] loss(logits[:, k_index], y[b][:, k_index].float()).backward()
b = torch.from_numpy(rng.integers(0, len(y), BATCH)) opt.step()
logits, _ = model(xs[b], xc[b])
opt.zero_grad()
loss(logits, y[b][:, k_index]).backward()
opt.step()
return model.eval() return model.eval()
@torch.no_grad() @torch.no_grad()
def predict(model, xs, xc): def predict(model, xs, xc, k_index):
out = [model(xs[i:i + 1024], xc[i:i + 1024]) for i in range(0, len(xs), 1024)] out = [model(xs[i:i + 1024].float(), xc[i:i + 1024].float()) for i in range(0, len(xs), 1024)]
return torch.sigmoid(torch.cat([o[0] for o in out])).numpy(), torch.cat([o[1] for o in out]).numpy() return (torch.sigmoid(torch.cat([o[0] for o in out]))[:, k_index].numpy(),
torch.cat([o[1] for o in out])[:, :, k_index].numpy())
def pretraining_parts(folder, keep):
"""The chunks of pretrain.py -> [(station, parts)]; keep(station, instants) -> the boolean mask of usable instants."""
out = {}
for q in sorted(Path(folder).glob("*.npz")):
d = np.load(q)
station = q.stem.rsplit("_", 1)[0]
m = keep(station, d["instants"])
if m.any():
xs, xc = prepare(d)
out.setdefault(station, []).append((xs[m], xc[m], torch.from_numpy(d["yhat"][m])))
return out
def pretrain_then_train(pre, parts, k_index, epochs, seed):
"""pre: {station: [chunk parts]}, each station drawn as often as the others, its chunks by their size."""
model = None
if pre:
chunks = [c for cs in pre.values() for c in cs]
weights = [len(c[2]) / sum(len(x[2]) for x in cs) for cs in pre.values() for c in cs]
model = train(chunks, torch.arange(len(FAMILIES)), max(1, epochs * len(parts) // len(chunks)), seed, weights)
return train(parts, k_index, epochs, seed, model=model)
def main(): def main():
@@ -85,32 +109,38 @@ def main():
p.add_argument("--epochs", type=int, default=200) p.add_argument("--epochs", type=int, default=200)
p.add_argument("--seeds", type=int, default=5, help="models averaged: one run alone varies by 10 to 20 points") p.add_argument("--seeds", type=int, default=5, help="models averaged: one run alone varies by 10 to 20 points")
p.add_argument("--threshold", type=float, default=0.5) p.add_argument("--threshold", type=float, default=0.5)
p.add_argument("--proportional", action="store_true", help="stations drawn in proportion to their instants") p.add_argument("--pretrain", help="the folder of pretrain.py: pre-training on the rules' outputs first")
p.add_argument("--leave-out", action="store_true") p.add_argument("--leave-out", action="store_true")
p.add_argument("--out") p.add_argument("--out")
a = p.parse_args() a = p.parse_args()
families = a.families.split(",") families = a.families.split(",")
k_index = torch.tensor([FAMILIES.index(f) for f in families]) k_index = torch.tensor([FAMILIES.index(f) for f in families])
data = {q.stem: np.load(q) for q in sorted(Path(a.cache).glob("*.npz"))} data = {q.stem: np.load(q) for q in sorted(Path(a.cache).glob("*.npz"))}
tensors = {s: (*prepare(d), torch.from_numpy(d["y"].astype(np.float32))) for s, d in data.items()} tensors = {s: (*prepare(d), torch.from_numpy(d["y"])) for s, d in data.items()}
def ensemble(parts): def ensemble(parts, pre):
return [train(parts, k_index, a.epochs, seed, a.proportional) for seed in range(a.seeds)] return [pretrain_then_train(pre, parts, k_index, a.epochs, seed) for seed in range(a.seeds)]
def averaged(models, xs, xc): def averaged(models, xs, xc):
out = [predict(m, xs, xc) for m in models] out = [predict(m, xs, xc, k_index) for m in models]
return np.mean([o[0] for o in out], axis=0), np.mean([o[1] for o in out], axis=0) return np.mean([o[0] for o in out], axis=0), np.mean([o[1] for o in out], axis=0)
def pre(keep):
return pretraining_parts(a.pretrain, keep) if a.pretrain else {}
if not a.leave_out: if not a.leave_out:
cut = {s: int(CUT * len(d["y"])) for s, d in data.items()} cut = {s: int(CUT * len(d["y"])) for s, d in data.items()}
models = ensemble([tuple(t[:cut[s]] for t in tensors[s]) for s in data]) first_tested = {s: data[s]["instants"][cut[s]] for s in data}
models = ensemble([tuple(t[:cut[s]] for t in tensors[s]) for s in data],
pre(lambda s, inst: inst < first_tested[s] if s in first_tested else np.ones(len(inst), bool)))
proba = {s: averaged(models, tensors[s][0][cut[s]:], tensors[s][1][cut[s]:])[0] for s in data} proba = {s: averaged(models, tensors[s][0][cut[s]:], tensors[s][1][cut[s]:])[0] for s in data}
report_time_split(a.cache, data, proba, families, a.threshold) report_time_split(a.cache, data, proba, families, a.threshold)
return return
out = Path(a.out) out = Path(a.out)
out.mkdir(parents=True, exist_ok=True) out.mkdir(parents=True, exist_ok=True)
for test in data: for test in data:
proba, attention = averaged(ensemble([tensors[s] for s in data if s != test]), *tensors[test][:2]) models = ensemble([tensors[s] for s in data if s != test], pre(lambda s, inst: np.full(len(inst), s != test)))
proba, attention = averaged(models, *tensors[test][:2])
yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy() yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy()
for j, f in enumerate(families): for j, f in enumerate(families):
yhat[:, FAMILIES.index(f)] = proba[:, j] >= a.threshold yhat[:, FAMILIES.index(f)] = proba[:, j] >= a.threshold
+53
View File
@@ -0,0 +1,53 @@
"""The pre-training set (step 5): every station of the data folder, labelled or not, over its whole range, in chunks of
two weeks: the features of dataset.py and, as targets, the families the rules flag at each instant (bench/score.py
events(), each chunk classified with its 3 days of history). No label is read.
-> <out>/<STATION>_<YYYYMMDD>.npz with instants, kinds, xs, xc, yhat [instant, family]; an existing chunk is kept.
Usage: python pretrain.py --detectors <detectors> --data <data> --out <folder> [--stations A,B]
"""
import argparse
import sys
from datetime import timedelta
from pathlib import Path
import numpy as np
from dataset import HISTORY, build
from evaluate import events_to_yhat
CHUNK = timedelta(days=14)
def main():
p = argparse.ArgumentParser(usage=__doc__)
for k in ("detectors", "data", "out"):
p.add_argument("--" + k, required=True)
p.add_argument("--stations")
a = p.parse_args()
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
import score # noqa: E402 (puts the tools repository's reader on the path)
import data # noqa: E402
out = Path(a.out)
out.mkdir(parents=True, exist_ok=True)
for folder in sorted(Path(a.data).iterdir()):
if not (folder / "displacement").is_dir() or (a.stations and folder.name not in a.stations.split(",")):
continue
measurements = data.load(str(folder))
times = [m["t"] for v in measurements.values() for m in v]
t = min(times).replace(hour=0, minute=0, second=0, microsecond=0) + HISTORY # the first days: history only
while t <= max(times):
path = out / f"{folder.name}_{t:%Y%m%d}.npz"
start, end = data.iso(t), data.iso(t + CHUNK)
t += CHUNK
if path.exists():
continue
d = build(str(folder), [], start, end, data, measurements)
if not len(d["instants"]):
continue
yhat = events_to_yhat(score.events(measurements, start, end, str(folder)), list(d["instants"]))
np.savez_compressed(path, instants=d["instants"], kinds=d["kinds"], xs=d["xs"], xc=d["xc"], yhat=yhat)
print(f"{path.stem}: {len(d['instants'])} instants, {len(d['kinds'])} sensors, flagged "
f"{dict(zip(d['families'].tolist(), yhat.sum(0).tolist()))}", flush=True)
if __name__ == "__main__":
main()
+1
View File
@@ -6,6 +6,7 @@ from pathlib import Path
import numpy as np import numpy as np
sys.path.insert(0, str(Path(sys.argv[1]) / "scripts")) sys.path.insert(0, str(Path(sys.argv[1]) / "scripts"))
import categories # noqa: E402,F401 puts the tools repository's reader on the path
import data # noqa: E402 import data # noqa: E402
import dataset # noqa: E402 import dataset # noqa: E402
+5 -2
View File
@@ -13,7 +13,10 @@ y[1:, 1] = 1
y[2, 0] = 1 y[2, 0] = 1
c = evaluate.counts(y, yhat) c = evaluate.counts(y, yhat)
assert c == {"Visibility": [1, 1, 0], "Obstruction": [2, 2, 2]}, c assert c == {"Visibility": [1, 1, 0], "Obstruction": [2, 2, 2]}, c
assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}) == [("Obstruction", "found", 50, 40)] assert evaluate.regressions({"Obstruction": [100, 100, 50]}, {"Obstruction": [100, 30, 30]}) == [] # 20 instants: not more
assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 10, 5]}) == [] assert evaluate.regressions({"Obstruction": [100, 100, 50]}, {"Obstruction": [100, 29, 29]}) == [("Obstruction", "found", 50, 29, 21)]
assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}) == [] # 10 points, 1 instant: noise
assert evaluate.regressions({"Obstruction": [10, 10, 5]}, {"Obstruction": [10, 4, 4]}, whole=True) == [("Obstruction", "found", 50, 40, 1)]
assert evaluate.regressions({"Visibility": [100, 50, 50]}, {"Visibility": [100, 100, 71]}) == [("Visibility", "correct", 100, 71, 29)]
assert evaluate.regressions({"Obstruction": [0, 15, 0]}, {}) == [] # false alarms gone: not a regression assert evaluate.regressions({"Obstruction": [0, 15, 0]}, {}) == [] # false alarms gone: not a regression
print("ok") print("ok")