Files

65 lines
3.4 KiB
Python

"""Benchmark of the detectors against a reference truth: every labelled station.
For each <STATION>.jsonl of --labels that has its folder <STATION>/ in --data, the score per family and per instant of
score.py, over the range of its labels: from the day of the first to the midnight after the last; with --from/--to, over
that range for every station, a station without a label on it included (several stations labelled the same days). Then the total,
family by family, which says whether the detectors do better overall, and the complexity of the detectors (code lines
and constants of scripts/). Two outputs are compared by compare.py (the regressions, CLAUDE.md).
Usage: python3 regression.py --labels <folder of <STATION>.jsonl> --data <folder of <STATION>/> [--stations A,B]
[--from YYYY-MM-DD --to YYYY-MM-DD]
"""
import argparse
import re
from datetime import timedelta
from pathlib import Path
import score # next to it
from data import iso, parse_time # on the path through score
def complexity(scripts=Path(__file__).resolve().parents[1] / "scripts"):
"""-> (code lines, constants) of the detectors: non-blank lines that are not a comment; NAME = lines at module level."""
lines = [x for f in sorted(scripts.glob("*.py")) for x in f.read_text().splitlines()]
code = [x for x in lines if x.strip() and not x.strip().startswith("#")]
return len(code), sum(bool(re.match(r"[A-Z][A-Z_0-9]* = ", x)) for x in lines)
def label_range(path):
"""Labels JSONL -> (start, end) iso, in whole UTC days, end excluded."""
labels = score.read_labels(path)
start = min(e["start"] for e in labels)[:10]
# ponytail: the end of the labelled batch is not written; midnight closest to the last bound (a label bound
# overflows by one measurement after midnight)
end = iso(parse_time(max(e["end"] for e in labels)) + timedelta(hours=12))[:10]
return iso(parse_time(start)), iso(parse_time(end))
if __name__ == "__main__":
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("--labels", required=True, help="folder of the labels, one <STATION>.jsonl per station")
p.add_argument("--data", required=True, help="folder of the station folders, <STATION>/")
p.add_argument("--stations", help="A,B: these stations only (a shorter run)")
p.add_argument("--from", dest="start", help="YYYY-MM-DD: this range for every station, not the range of its labels")
p.add_argument("--to", dest="end", help="YYYY-MM-DD, excluded (with --from)")
a = p.parse_args()
if bool(a.start) != bool(a.end):
p.error("--from and --to go together")
total = {}
for path in sorted(Path(a.labels).glob("*.jsonl")):
station = path.stem
if a.stations and station not in a.stations.split(","):
continue
folder = Path(a.data) / station
if not folder.is_dir():
print(f"{station}: no {folder}/, skipped\n")
continue
start, end = (iso(parse_time(a.start)), iso(parse_time(a.end))) if a.start else label_range(path)
n, labels = score.family_scores(str(folder), str(path), start, end)
score.show(f"{station}: {start} → {end}, {len(labels)} labels", n)
print()
for k, v in n.items():
total[k] = [x + y for x, y in zip(total.get(k, [0, 0, 0]), v)]
score.show("Total", total)
print("\nComplexity: %d code lines, %d constants" % complexity())