The detectors were simplified on 2026-11-16: the reader moved to aimsight-tools (scripts/data.py, put on the path by categories.py) and a regression is now a Total losing more than 1 point, or a family of a station losing more than 5 points and more than 20 instants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
54 lines
2.4 KiB
Python
54 lines
2.4 KiB
Python
"""The pre-training set (step 5): every station of the data folder, labelled or not, over its whole range, in chunks of
|
|
two weeks: the features of dataset.py and, as targets, the families the rules flag at each instant (bench/score.py
|
|
events(), each chunk classified with its 3 days of history). No label is read.
|
|
-> <out>/<STATION>_<YYYYMMDD>.npz with instants, kinds, xs, xc, yhat [instant, family]; an existing chunk is kept.
|
|
Usage: python pretrain.py --detectors <detectors> --data <data> --out <folder> [--stations A,B]
|
|
"""
|
|
import argparse
|
|
import sys
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
from dataset import HISTORY, build
|
|
from evaluate import events_to_yhat
|
|
|
|
CHUNK = timedelta(days=14)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(usage=__doc__)
|
|
for k in ("detectors", "data", "out"):
|
|
p.add_argument("--" + k, required=True)
|
|
p.add_argument("--stations")
|
|
a = p.parse_args()
|
|
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
|
|
import score # noqa: E402 (puts the tools repository's reader on the path)
|
|
import data # noqa: E402
|
|
out = Path(a.out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
for folder in sorted(Path(a.data).iterdir()):
|
|
if not (folder / "displacement").is_dir() or (a.stations and folder.name not in a.stations.split(",")):
|
|
continue
|
|
measurements = data.load(str(folder))
|
|
times = [m["t"] for v in measurements.values() for m in v]
|
|
t = min(times).replace(hour=0, minute=0, second=0, microsecond=0) + HISTORY # the first days: history only
|
|
while t <= max(times):
|
|
path = out / f"{folder.name}_{t:%Y%m%d}.npz"
|
|
start, end = data.iso(t), data.iso(t + CHUNK)
|
|
t += CHUNK
|
|
if path.exists():
|
|
continue
|
|
d = build(str(folder), [], start, end, data, measurements)
|
|
if not len(d["instants"]):
|
|
continue
|
|
yhat = events_to_yhat(score.events(measurements, start, end, str(folder)), list(d["instants"]))
|
|
np.savez_compressed(path, instants=d["instants"], kinds=d["kinds"], xs=d["xs"], xc=d["xc"], yhat=yhat)
|
|
print(f"{path.stem}: {len(d['instants'])} instants, {len(d['kinds'])} sensors, flagged "
|
|
f"{dict(zip(d['families'].tolist(), yhat.sum(0).tolist()))}", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|