65 lines
3.4 KiB
Python
65 lines
3.4 KiB
Python
"""Benchmark of the detectors against a reference truth: every labelled station.
|
|
|
|
For each <STATION>.jsonl of --labels that has its folder <STATION>/ in --data, the score per family and per instant of
|
|
score.py, over the range of its labels: from the day of the first to the midnight after the last; with --from/--to, over
|
|
that range for every station, a station without a label on it included (several stations labelled the same days). Then the total,
|
|
family by family, which says whether the detectors do better overall, and the complexity of the detectors (code lines
|
|
and constants of scripts/). Two outputs are compared by compare.py (the regressions, CLAUDE.md).
|
|
Usage: python3 regression.py --labels <folder of <STATION>.jsonl> --data <folder of <STATION>/> [--stations A,B]
|
|
[--from YYYY-MM-DD --to YYYY-MM-DD]
|
|
"""
|
|
import argparse
|
|
import re
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
|
|
import score # next to it
|
|
|
|
from data import iso, parse_time # on the path through score
|
|
|
|
|
|
def complexity(scripts=Path(__file__).resolve().parents[1] / "scripts"):
|
|
"""-> (code lines, constants) of the detectors: non-blank lines that are not a comment; NAME = lines at module level."""
|
|
lines = [x for f in sorted(scripts.glob("*.py")) for x in f.read_text().splitlines()]
|
|
code = [x for x in lines if x.strip() and not x.strip().startswith("#")]
|
|
return len(code), sum(bool(re.match(r"[A-Z][A-Z_0-9]* = ", x)) for x in lines)
|
|
|
|
|
|
def label_range(path):
|
|
"""Labels JSONL -> (start, end) iso, in whole UTC days, end excluded."""
|
|
labels = score.read_labels(path)
|
|
start = min(e["start"] for e in labels)[:10]
|
|
# ponytail: the end of the labelled batch is not written; midnight closest to the last bound (a label bound
|
|
# overflows by one measurement after midnight)
|
|
end = iso(parse_time(max(e["end"] for e in labels)) + timedelta(hours=12))[:10]
|
|
return iso(parse_time(start)), iso(parse_time(end))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
p = argparse.ArgumentParser(usage=__doc__)
|
|
p.add_argument("--labels", required=True, help="folder of the labels, one <STATION>.jsonl per station")
|
|
p.add_argument("--data", required=True, help="folder of the station folders, <STATION>/")
|
|
p.add_argument("--stations", help="A,B: these stations only (a shorter run)")
|
|
p.add_argument("--from", dest="start", help="YYYY-MM-DD: this range for every station, not the range of its labels")
|
|
p.add_argument("--to", dest="end", help="YYYY-MM-DD, excluded (with --from)")
|
|
a = p.parse_args()
|
|
if bool(a.start) != bool(a.end):
|
|
p.error("--from and --to go together")
|
|
total = {}
|
|
for path in sorted(Path(a.labels).glob("*.jsonl")):
|
|
station = path.stem
|
|
if a.stations and station not in a.stations.split(","):
|
|
continue
|
|
folder = Path(a.data) / station
|
|
if not folder.is_dir():
|
|
print(f"{station}: no {folder}/, skipped\n")
|
|
continue
|
|
start, end = (iso(parse_time(a.start)), iso(parse_time(a.end))) if a.start else label_range(path)
|
|
n, labels = score.family_scores(str(folder), str(path), start, end)
|
|
score.show(f"{station}: {start} → {end}, {len(labels)} labels", n)
|
|
print()
|
|
for k, v in n.items():
|
|
total[k] = [x + y for x, y in zip(total.get(k, [0, 0, 0]), v)]
|
|
score.show("Total", total)
|
|
print("\nComplexity: %d code lines, %d constants" % complexity())
|