2026-11-16 19:22:53 +01:00
|
|
|
"""M2, the model of the pilot: multiple-instance learning. A label names a station and a range, not its sensors, and
|
|
|
|
|
each station has its own number of sensors: one small MLP, shared by every sensor, reads a sensor's features with the
|
|
|
|
|
station's; per family, an attention over the sensors pools their scores into the station's. The attention weights give
|
|
|
|
|
the sensors (step 6).
|
|
|
|
|
|
|
|
|
|
Inputs: every feature through sign(x) log(1 + |x|) (jumps reach thousands of noises), NaN set to 0 with a presence mask.
|
2026-11-17 09:51:56 +01:00
|
|
|
Training: binary cross-entropy per family on the reviewed labels, each station drawn as often as the others.
|
|
|
|
|
--pretrain <folder of pretrain.py>: first the six families the rules flag, on every station and month, labelled or not;
|
|
|
|
|
then the same fine-tuning on the reviewed labels. Never on what is tested: time split, the instants from each labelled
|
|
|
|
|
station's cut on; station left out, the whole station.
|
2026-11-16 19:22:53 +01:00
|
|
|
default: the criterion, trained on the first 70 % of every station, scored on the last 30 % (evaluate.CUT);
|
|
|
|
|
--leave-out: the indicator, each station predicted by the model trained on the others; one <STATION>.npz per station
|
|
|
|
|
in --out, with yhat [instant, family] (the rules outside the pilot families), proba and attention [instant, sensor, family].
|
2026-11-17 09:51:56 +01:00
|
|
|
Usage: python m2.py --cache <cache> [--pretrain <folder>] [--leave-out --out <folder>] [--families Visibility,Obstruction]
|
|
|
|
|
[--epochs 200 --seeds 5]
|
2026-11-16 19:22:53 +01:00
|
|
|
"""
|
|
|
|
|
import argparse
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
|
|
import torch
|
|
|
|
|
from torch import nn
|
|
|
|
|
|
|
|
|
|
from dataset import FAMILIES
|
|
|
|
|
from evaluate import CUT, report_time_split
|
|
|
|
|
|
|
|
|
|
BATCHES, BATCH = 8, 128 # per station and per epoch: every station weighs the same
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def prepare(d):
|
2026-11-17 09:51:56 +01:00
|
|
|
"""A station of the cache -> tensors xs [T, S, 2 Fs], xc [T, 2 Fc]: the squashed features and their presence, in
|
|
|
|
|
float16 to hold every month in memory (the squashed values stay below 20); cast per batch."""
|
2026-11-16 19:22:53 +01:00
|
|
|
def squash(x):
|
|
|
|
|
m = np.isfinite(x)
|
|
|
|
|
v = np.where(m, x, 0)
|
2026-11-17 09:51:56 +01:00
|
|
|
return np.concatenate([np.sign(v) * np.log1p(np.abs(v)), m], axis=-1).astype(np.float16)
|
2026-11-16 19:22:53 +01:00
|
|
|
return torch.from_numpy(squash(d["xs"])), torch.from_numpy(squash(d["xc"]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MIL(nn.Module):
|
|
|
|
|
def __init__(self, fs, fc, k, hidden=64, dropout=0.1):
|
|
|
|
|
super().__init__()
|
|
|
|
|
self.encoder = nn.Sequential(nn.Linear(fs + fc, hidden), nn.ReLU(), nn.Dropout(dropout),
|
|
|
|
|
nn.Linear(hidden, hidden), nn.ReLU())
|
|
|
|
|
self.attention = nn.Linear(hidden, k) # per family: how much a sensor counts for the station
|
|
|
|
|
self.score = nn.Linear(hidden, k) # per family: the sensor's own score
|
|
|
|
|
|
|
|
|
|
def forward(self, xs, xc):
|
|
|
|
|
"""xs [B, S, Fs], xc [B, Fc] -> station logits [B, K], attention [B, S, K]."""
|
|
|
|
|
h = self.encoder(torch.cat([xs, xc[:, None, :].expand(-1, xs.shape[1], -1)], dim=-1))
|
|
|
|
|
attention = torch.softmax(self.attention(h), dim=1)
|
|
|
|
|
return (attention * self.score(h)).sum(dim=1), attention
|
|
|
|
|
|
|
|
|
|
|
2026-11-17 09:51:56 +01:00
|
|
|
def train(parts, k_index, epochs, seed=0, weights=None, model=None):
|
|
|
|
|
"""parts: [(xs, xc, y [T, K])] -> the model trained on the families k_index, from model if given (fine-tuning).
|
|
|
|
|
weights: the chance of each part (default: each part as often as the others); epochs: BATCHES batches per part."""
|
2026-11-16 19:22:53 +01:00
|
|
|
torch.manual_seed(seed)
|
|
|
|
|
rng = np.random.default_rng(seed)
|
2026-11-17 09:51:56 +01:00
|
|
|
p = np.full(len(parts), 1 / len(parts)) if weights is None else np.asarray(weights) / np.sum(weights)
|
|
|
|
|
model = model or MIL(parts[0][0].shape[-1], parts[0][1].shape[-1], len(FAMILIES))
|
2026-11-16 19:22:53 +01:00
|
|
|
opt = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4)
|
|
|
|
|
loss = nn.BCEWithLogitsLoss()
|
|
|
|
|
model.train()
|
2026-11-17 09:51:56 +01:00
|
|
|
for i in rng.choice(len(parts), epochs * BATCHES * len(parts), p=p):
|
|
|
|
|
xs, xc, y = parts[i]
|
|
|
|
|
b = torch.from_numpy(rng.integers(0, len(y), BATCH))
|
|
|
|
|
logits, _ = model(xs[b].float(), xc[b].float())
|
|
|
|
|
opt.zero_grad()
|
|
|
|
|
loss(logits[:, k_index], y[b][:, k_index].float()).backward()
|
|
|
|
|
opt.step()
|
2026-11-16 19:22:53 +01:00
|
|
|
return model.eval()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@torch.no_grad()
|
2026-11-17 09:51:56 +01:00
|
|
|
def predict(model, xs, xc, k_index):
|
|
|
|
|
out = [model(xs[i:i + 1024].float(), xc[i:i + 1024].float()) for i in range(0, len(xs), 1024)]
|
|
|
|
|
return (torch.sigmoid(torch.cat([o[0] for o in out]))[:, k_index].numpy(),
|
|
|
|
|
torch.cat([o[1] for o in out])[:, :, k_index].numpy())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def pretraining_parts(folder, keep):
|
|
|
|
|
"""The chunks of pretrain.py -> [(station, parts)]; keep(station, instants) -> the boolean mask of usable instants."""
|
|
|
|
|
out = {}
|
|
|
|
|
for q in sorted(Path(folder).glob("*.npz")):
|
|
|
|
|
d = np.load(q)
|
|
|
|
|
station = q.stem.rsplit("_", 1)[0]
|
|
|
|
|
m = keep(station, d["instants"])
|
|
|
|
|
if m.any():
|
|
|
|
|
xs, xc = prepare(d)
|
|
|
|
|
out.setdefault(station, []).append((xs[m], xc[m], torch.from_numpy(d["yhat"][m])))
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def pretrain_then_train(pre, parts, k_index, epochs, seed):
|
|
|
|
|
"""pre: {station: [chunk parts]}, each station drawn as often as the others, its chunks by their size."""
|
|
|
|
|
model = None
|
|
|
|
|
if pre:
|
|
|
|
|
chunks = [c for cs in pre.values() for c in cs]
|
|
|
|
|
weights = [len(c[2]) / sum(len(x[2]) for x in cs) for cs in pre.values() for c in cs]
|
|
|
|
|
model = train(chunks, torch.arange(len(FAMILIES)), max(1, epochs * len(parts) // len(chunks)), seed, weights)
|
|
|
|
|
return train(parts, k_index, epochs, seed, model=model)
|
2026-11-16 19:22:53 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
p = argparse.ArgumentParser(usage=__doc__)
|
|
|
|
|
p.add_argument("--cache", required=True)
|
|
|
|
|
p.add_argument("--families", default="Visibility,Obstruction")
|
|
|
|
|
p.add_argument("--epochs", type=int, default=200)
|
|
|
|
|
p.add_argument("--seeds", type=int, default=5, help="models averaged: one run alone varies by 10 to 20 points")
|
|
|
|
|
p.add_argument("--threshold", type=float, default=0.5)
|
2026-11-17 09:51:56 +01:00
|
|
|
p.add_argument("--pretrain", help="the folder of pretrain.py: pre-training on the rules' outputs first")
|
2026-11-16 19:22:53 +01:00
|
|
|
p.add_argument("--leave-out", action="store_true")
|
|
|
|
|
p.add_argument("--out")
|
|
|
|
|
a = p.parse_args()
|
|
|
|
|
families = a.families.split(",")
|
|
|
|
|
k_index = torch.tensor([FAMILIES.index(f) for f in families])
|
|
|
|
|
data = {q.stem: np.load(q) for q in sorted(Path(a.cache).glob("*.npz"))}
|
2026-11-17 09:51:56 +01:00
|
|
|
tensors = {s: (*prepare(d), torch.from_numpy(d["y"])) for s, d in data.items()}
|
2026-11-16 19:22:53 +01:00
|
|
|
|
2026-11-17 09:51:56 +01:00
|
|
|
def ensemble(parts, pre):
|
|
|
|
|
return [pretrain_then_train(pre, parts, k_index, a.epochs, seed) for seed in range(a.seeds)]
|
2026-11-16 19:22:53 +01:00
|
|
|
|
|
|
|
|
def averaged(models, xs, xc):
|
2026-11-17 09:51:56 +01:00
|
|
|
out = [predict(m, xs, xc, k_index) for m in models]
|
2026-11-16 19:22:53 +01:00
|
|
|
return np.mean([o[0] for o in out], axis=0), np.mean([o[1] for o in out], axis=0)
|
|
|
|
|
|
2026-11-17 09:51:56 +01:00
|
|
|
def pre(keep):
|
|
|
|
|
return pretraining_parts(a.pretrain, keep) if a.pretrain else {}
|
|
|
|
|
|
2026-11-16 19:22:53 +01:00
|
|
|
if not a.leave_out:
|
|
|
|
|
cut = {s: int(CUT * len(d["y"])) for s, d in data.items()}
|
2026-11-17 09:51:56 +01:00
|
|
|
first_tested = {s: data[s]["instants"][cut[s]] for s in data}
|
|
|
|
|
models = ensemble([tuple(t[:cut[s]] for t in tensors[s]) for s in data],
|
|
|
|
|
pre(lambda s, inst: inst < first_tested[s] if s in first_tested else np.ones(len(inst), bool)))
|
2026-11-16 19:22:53 +01:00
|
|
|
proba = {s: averaged(models, tensors[s][0][cut[s]:], tensors[s][1][cut[s]:])[0] for s in data}
|
|
|
|
|
report_time_split(a.cache, data, proba, families, a.threshold)
|
|
|
|
|
return
|
|
|
|
|
out = Path(a.out)
|
|
|
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
for test in data:
|
2026-11-17 09:51:56 +01:00
|
|
|
models = ensemble([tensors[s] for s in data if s != test], pre(lambda s, inst: np.full(len(inst), s != test)))
|
|
|
|
|
proba, attention = averaged(models, *tensors[test][:2])
|
2026-11-16 19:22:53 +01:00
|
|
|
yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy()
|
|
|
|
|
for j, f in enumerate(families):
|
|
|
|
|
yhat[:, FAMILIES.index(f)] = proba[:, j] >= a.threshold
|
|
|
|
|
np.savez_compressed(out / f"{test}.npz", yhat=yhat, proba=proba, attention=attention.astype(np.float16),
|
|
|
|
|
families=np.array(families))
|
|
|
|
|
print(test, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
main()
|