"""The pre-training set (step 5): every station of the data folder, labelled or not, over its whole range, in chunks of two weeks: the features of dataset.py and, as targets, the families the rules flag at each instant (bench/score.py events(), each chunk classified with its 3 days of history). No label is read. -> /_.npz with instants, kinds, xs, xc, yhat [instant, family]; an existing chunk is kept. Usage: python pretrain.py --detectors --data --out [--stations A,B] """ import argparse import sys from datetime import timedelta from pathlib import Path import numpy as np from dataset import HISTORY, build from evaluate import events_to_yhat CHUNK = timedelta(days=14) def main(): p = argparse.ArgumentParser(usage=__doc__) for k in ("detectors", "data", "out"): p.add_argument("--" + k, required=True) p.add_argument("--stations") a = p.parse_args() sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")] import score # noqa: E402 (puts the tools repository's reader on the path) import data # noqa: E402 out = Path(a.out) out.mkdir(parents=True, exist_ok=True) for folder in sorted(Path(a.data).iterdir()): if not (folder / "displacement").is_dir() or (a.stations and folder.name not in a.stations.split(",")): continue measurements = data.load(str(folder)) times = [m["t"] for v in measurements.values() for m in v] t = min(times).replace(hour=0, minute=0, second=0, microsecond=0) + HISTORY # the first days: history only while t <= max(times): path = out / f"{folder.name}_{t:%Y%m%d}.npz" start, end = data.iso(t), data.iso(t + CHUNK) t += CHUNK if path.exists(): continue d = build(str(folder), [], start, end, data, measurements) if not len(d["instants"]): continue yhat = events_to_yhat(score.events(measurements, start, end, str(folder)), list(d["instants"])) np.savez_compressed(path, instants=d["instants"], kinds=d["kinds"], xs=d["xs"], xc=d["xc"], yhat=yhat) print(f"{path.stem}: {len(d['instants'])} instants, {len(d['kinds'])} sensors, flagged " f"{dict(zip(d['families'].tolist(), yhat.sum(0).tolist()))}", flush=True) if __name__ == "__main__": main()