Files

53 lines
2.4 KiB
Python
Raw Permalink Normal View History

"""The pre-training set (step 5): every station of the data folder, labelled or not, over its whole range, in chunks of
two weeks: the features of dataset.py and, as targets, the families the rules flag at each instant (bench/score.py
events(), each chunk classified with its 3 days of history). No label is read.
-> <out>/<STATION>_<YYYYMMDD>.npz with instants, kinds, xs, xc, yhat [instant, family]; an existing chunk is kept.
Usage: python pretrain.py --detectors <detectors> --data <data> --out <folder> [--stations A,B]
"""
import argparse
import sys
from datetime import timedelta
from pathlib import Path
import numpy as np
from dataset import HISTORY, build
from evaluate import events_to_yhat
CHUNK = timedelta(days=14)
def main():
p = argparse.ArgumentParser(usage=__doc__)
for k in ("detectors", "data", "out"):
p.add_argument("--" + k, required=True)
p.add_argument("--stations")
a = p.parse_args()
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
import score # noqa: E402 (puts the tools repository's reader on the path)
import data # noqa: E402
out = Path(a.out)
out.mkdir(parents=True, exist_ok=True)
for folder in sorted(Path(a.data).iterdir()):
if not (folder / "displacement").is_dir() or (a.stations and folder.name not in a.stations.split(",")):
continue
measurements = data.load(str(folder))
times = [m["t"] for v in measurements.values() for m in v]
t = min(times).replace(hour=0, minute=0, second=0, microsecond=0) + HISTORY # the first days: history only
while t <= max(times):
path = out / f"{folder.name}_{t:%Y%m%d}.npz"
start, end = data.iso(t), data.iso(t + CHUNK)
t += CHUNK
if path.exists():
continue
d = build(str(folder), [], start, end, data, measurements)
if not len(d["instants"]):
continue
yhat = events_to_yhat(score.events(measurements, start, end, str(folder)), list(d["instants"]))
np.savez_compressed(path, instants=d["instants"], kinds=d["kinds"], xs=d["xs"], xc=d["xc"], yhat=yhat)
print(f"{path.stem}: {len(d['instants'])} instants, {len(d['kinds'])} sensors, flagged "
f"{dict(zip(d['families'].tolist(), yhat.sum(0).tolist()))}", flush=True)
if __name__ == "__main__":
main()