"""Benchmark of the learner (D-116): the detectors against all the labeller's labels (labeling/auto/). For each station of labeling/auto/ that has its folder in data/, the score per family and per instant of score.py (D-112), over the range of its labels: from the day of the first to the midnight after the last. Then the total, family by family, which says whether the detectors do better overall from one session to the next. The block of the session's station is its score; the other blocks, the non-regression: a family that goes back on a station is a regression, even if the total rises. Usage: python3 detectors/scripts/learner/regression.py [--stations A,B] """ import argparse import json from datetime import timedelta import score # next to it from data import iso, parse_time # on the path through score def label_range(path): """labeling/auto/.jsonl -> (start, end) iso, in whole UTC days, end excluded.""" with open(path, encoding="utf-8") as f: labels = [json.loads(l) for l in f if l.strip()] start = min(e["debut"] for e in labels)[:10] # ponytail: the end of the batch is not written; midnight closest to the last bound (a label bound overflows by one # measurement after midnight); the real end of the batch in labeling/auto/ if the gap matters end = iso(parse_time(max(e["fin"] for e in labels)) + timedelta(hours=12))[:10] return iso(parse_time(start)), iso(parse_time(end)) if __name__ == "__main__": p = argparse.ArgumentParser(usage=__doc__) p.add_argument("--stations", help="A,B: these stations only (a shorter run)") a = p.parse_args() total = {} for path in sorted((score.WORK / "labeling" / "auto").glob("*.jsonl")): station = path.stem if a.stations and station not in a.stations.split(","): continue folder = score.WORK / "data" / station if not folder.is_dir(): print(f"{station}: no data/{station}/, skipped\n") continue start, end = label_range(path) n, labels = score.family_scores(str(folder), start, end) score.show(f"{station}: {start} → {end}, {len(labels)} labels", n) print() for k, v in n.items(): total[k] = [x + y for x, y in zip(total.get(k, [0, 0, 0]), v)] score.show("Total", total)