Step 5: pre-training on the rules' outputs (pretrain.py, m2.py --pretrain); regressions as detectors/bench/compare.py; the reader from the tools repository
The detectors were simplified on 2026-11-16: the reader moved to aimsight-tools (scripts/data.py, put on the path by categories.py) and a regression is now a Total losing more than 1 point, or a family of a station losing more than 5 points and more than 20 instants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7312a1c3ca
commit
5337dd0dac
6 files changed
+147
-51
No files matched your search
+53
@@ -0,0 +1,53 @@
|
||||
"""The pre-training set (step 5): every station of the data folder, labelled or not, over its whole range, in chunks of
|
||||
two weeks: the features of dataset.py and, as targets, the families the rules flag at each instant (bench/score.py
|
||||
events(), each chunk classified with its 3 days of history). No label is read.
|
||||
-> <out>/<STATION>_<YYYYMMDD>.npz with instants, kinds, xs, xc, yhat [instant, family]; an existing chunk is kept.
|
||||
Usage: python pretrain.py --detectors <detectors> --data <data> --out <folder> [--stations A,B]
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from dataset import HISTORY, build
|
||||
from evaluate import events_to_yhat
|
||||
|
||||
CHUNK = timedelta(days=14)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(usage=__doc__)
|
||||
for k in ("detectors", "data", "out"):
|
||||
p.add_argument("--" + k, required=True)
|
||||
p.add_argument("--stations")
|
||||
a = p.parse_args()
|
||||
sys.path[:0] = [str(Path(a.detectors) / "scripts"), str(Path(a.detectors) / "bench")]
|
||||
import score # noqa: E402 (puts the tools repository's reader on the path)
|
||||
import data # noqa: E402
|
||||
out = Path(a.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
for folder in sorted(Path(a.data).iterdir()):
|
||||
if not (folder / "displacement").is_dir() or (a.stations and folder.name not in a.stations.split(",")):
|
||||
continue
|
||||
measurements = data.load(str(folder))
|
||||
times = [m["t"] for v in measurements.values() for m in v]
|
||||
t = min(times).replace(hour=0, minute=0, second=0, microsecond=0) + HISTORY # the first days: history only
|
||||
while t <= max(times):
|
||||
path = out / f"{folder.name}_{t:%Y%m%d}.npz"
|
||||
start, end = data.iso(t), data.iso(t + CHUNK)
|
||||
t += CHUNK
|
||||
if path.exists():
|
||||
continue
|
||||
d = build(str(folder), [], start, end, data, measurements)
|
||||
if not len(d["instants"]):
|
||||
continue
|
||||
yhat = events_to_yhat(score.events(measurements, start, end, str(folder)), list(d["instants"]))
|
||||
np.savez_compressed(path, instants=d["instants"], kinds=d["kinds"], xs=d["xs"], xc=d["xc"], yhat=yhat)
|
||||
print(f"{path.stem}: {len(d['instants'])} instants, {len(d['kinds'])} sensors, flagged "
|
||||
f"{dict(zip(d['families'].tolist(), yhat.sum(0).tolist()))}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user