Steps 5 and 6: pre-trained M2 passes the criterion for Obstruction, not for Visibility; episodes and sensors (decode.py)

m2.py: one thread per process, --test to share the stations left out between processes, --out on the time split.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 11:59:49 +01:00
1 parent 5337dd0dac
commit 4836929bc7
4 files changed
+212 -42

No files matched your search

+26 -14
View File
@@ -8,10 +8,11 @@ Training: binary cross-entropy per family on the reviewed labels, each station d
--pretrain <folder of pretrain.py>: first the six families the rules flag, on every station and month, labelled or not;
then the same fine-tuning on the reviewed labels. Never on what is tested: time split, the instants from each labelled
station's cut on; station left out, the whole station.
default: the criterion, trained on the first 70 % of every station, scored on the last 30 % (evaluate.CUT);
default: the criterion, trained on the first 70 % of every station, scored on the last 30 % (evaluate.CUT); with
--out, the predictions of the last 30 % saved as below, with cut (the index of the first instant tested);
--leave-out: the indicator, each station predicted by the model trained on the others; one <STATION>.npz per station
in --out, with yhat [instant, family] (the rules outside the pilot families), proba and attention [instant, sensor, family].
Usage: python m2.py --cache <cache> [--pretrain <folder>] [--leave-out --out <folder>] [--families Visibility,Obstruction]
Usage: python m2.py --cache <cache> [--pretrain <folder>] [--leave-out [--test A,B] --out <folder>] [--families V,O]
[--epochs 200 --seeds 5]
"""
import argparse
@@ -24,6 +25,8 @@ from torch import nn
from dataset import FAMILIES
from evaluate import CUT, report_time_split
torch.set_num_threads(1) # a small model: one thread per process is as fast; share the work between processes instead
BATCHES, BATCH = 8, 128 # per station and per epoch: every station weighs the same
@@ -111,8 +114,11 @@ def main():
p.add_argument("--threshold", type=float, default=0.5)
p.add_argument("--pretrain", help="the folder of pretrain.py: pre-training on the rules' outputs first")
p.add_argument("--leave-out", action="store_true")
p.add_argument("--test", help="--leave-out: only these stations (several processes share the work)")
p.add_argument("--out")
a = p.parse_args()
if a.leave_out and not a.out:
p.error("--leave-out needs --out")
families = a.families.split(",")
k_index = torch.tensor([FAMILIES.index(f) for f in families])
data = {q.stem: np.load(q) for q in sorted(Path(a.cache).glob("*.npz"))}
@@ -128,25 +134,31 @@ def main():
def pre(keep):
return pretraining_parts(a.pretrain, keep) if a.pretrain else {}
out = Path(a.out or ".")
def save(station, proba, attention, cut=0):
out.mkdir(parents=True, exist_ok=True)
yhat = np.load(Path(a.cache) / "rules" / f"{station}.npz")["yhat"][cut:].copy()
for j, f in enumerate(families):
yhat[:, FAMILIES.index(f)] = proba[:, j] >= a.threshold
np.savez_compressed(out / f"{station}.npz", yhat=yhat, proba=proba, attention=attention.astype(np.float16),
families=np.array(families), cut=cut)
print(station, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True)
if not a.leave_out:
cut = {s: int(CUT * len(d["y"])) for s, d in data.items()}
first_tested = {s: data[s]["instants"][cut[s]] for s in data}
models = ensemble([tuple(t[:cut[s]] for t in tensors[s]) for s in data],
pre(lambda s, inst: inst < first_tested[s] if s in first_tested else np.ones(len(inst), bool)))
proba = {s: averaged(models, tensors[s][0][cut[s]:], tensors[s][1][cut[s]:])[0] for s in data}
report_time_split(a.cache, data, proba, families, a.threshold)
proba = {s: averaged(models, tensors[s][0][cut[s]:], tensors[s][1][cut[s]:]) for s in data}
report_time_split(a.cache, data, {s: p[0] for s, p in proba.items()}, families, a.threshold)
if a.out:
for s, (pr, att) in proba.items():
save(s, pr, att, cut[s])
return
out = Path(a.out)
out.mkdir(parents=True, exist_ok=True)
for test in data:
for test in a.test.split(",") if a.test else data:
models = ensemble([tensors[s] for s in data if s != test], pre(lambda s, inst: np.full(len(inst), s != test)))
proba, attention = averaged(models, *tensors[test][:2])
yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy()
for j, f in enumerate(families):
yhat[:, FAMILIES.index(f)] = proba[:, j] >= a.threshold
np.savez_compressed(out / f"{test}.npz", yhat=yhat, proba=proba, attention=attention.astype(np.float16),
families=np.array(families))
print(test, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True)
save(test, *averaged(models, *tensors[test][:2]))
if __name__ == "__main__":