"""Benchmark of the detectors against a reference truth: every labelled station. For each .jsonl of --labels that has its folder / in --data, the score per family and per instant of score.py, over the range of its labels: from the day of the first to the midnight after the last; with --from/--to, over that range for every station, a station without a label on it included (several stations labelled the same days). Then the total, family by family, which says whether the detectors do better overall. The block of a station is its score; the others, the non-regression: a family that goes back on a station is a regression, even if the total rises. Usage: python3 regression.py --labels .jsonl> --data /> [--stations A,B] [--from YYYY-MM-DD --to YYYY-MM-DD] """ import argparse from datetime import timedelta from pathlib import Path import score # next to it from data import iso, parse_time # on the path through score def label_range(path): """Labels JSONL -> (start, end) iso, in whole UTC days, end excluded.""" labels = score.read_labels(path) start = min(e["start"] for e in labels)[:10] # ponytail: the end of the labelled batch is not written; midnight closest to the last bound (a label bound # overflows by one measurement after midnight) end = iso(parse_time(max(e["end"] for e in labels)) + timedelta(hours=12))[:10] return iso(parse_time(start)), iso(parse_time(end)) if __name__ == "__main__": p = argparse.ArgumentParser(usage=__doc__) p.add_argument("--labels", required=True, help="folder of the labels, one .jsonl per station") p.add_argument("--data", required=True, help="folder of the station folders, /") p.add_argument("--stations", help="A,B: these stations only (a shorter run)") p.add_argument("--from", dest="start", help="YYYY-MM-DD: this range for every station, not the range of its labels") p.add_argument("--to", dest="end", help="YYYY-MM-DD, excluded (with --from)") a = p.parse_args() if bool(a.start) != bool(a.end): p.error("--from and --to go together") total = {} for path in sorted(Path(a.labels).glob("*.jsonl")): station = path.stem if a.stations and station not in a.stations.split(","): continue folder = Path(a.data) / station if not folder.is_dir(): print(f"{station}: no {folder}/, skipped\n") continue start, end = (iso(parse_time(a.start)), iso(parse_time(a.end))) if a.start else label_range(path) n, labels = score.family_scores(str(folder), str(path), start, end) score.show(f"{station}: {start} → {end}, {len(labels)} labels", n) print() for k, v in n.items(): total[k] = [x + y for x, y in zip(total.get(k, [0, 0, 0]), v)] score.show("Total", total)