"""Score of the detectors on a range: the detectors' events against the labels of labeling/auto/.jsonl that touch the range, per family (labeling/familles.md) and per instant (D-112). An instant: an hour of the range where at least one sensor has a measurement; it receives, on one side, the families of the labels covering it, bounds excluded (except for a hole, without a measurement inside), on the other, those of the detectors' events (categories.py, the range classified with its 3 days of history). The category of an event is its family. found: labelled instants that the detectors also flagged (share of the labelled ones); correct: flagged instants that a label also puts in the family (share of the flagged ones). The score is a fact, not a verdict: the labels can be wrong too. No label on the range: exits with an error. Label fields (written by Diagnostic-app): debut, fin, categorie (the family), description. Run from the working repository, where detectors/ is mounted. Usage: python3 detectors/skills/wf-improve/scripts/score.py --from YYYY-MM-DD --to YYYY-MM-DD """ import argparse import json import os import sys from datetime import timedelta from pathlib import Path REPO = Path(os.path.abspath(__file__)).parents[3] # the detectors repository WORK = REPO.parent # the working repository (labeling/, data/) sys.path.insert(0, str(REPO / "scripts")) import categories # noqa: E402 from data import iso, load, parse_time # noqa: E402 HISTORY = timedelta(days=3) # as the batch command (D-056) def events(measurements, start, end, folder=None): """Detectors' events touching the range, classified with its days of history (as the batch command).""" h, b = parse_time(start) - HISTORY, parse_time(end) chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])} return [v for v in (categories.events(chunk, folder=folder) if chunk else []) if v["end"] >= start] def count(pairs): """[(labelled keys, flagged keys)], one pair per instant -> {key: [labelled, flagged, both]}.""" n = {} for a, d in pairs: for k in a | d: x = n.setdefault(k, [0, 0, 0]) x[0], x[1], x[2] = x[0] + (k in a), x[1] + (k in d), x[2] + (k in a and k in d) return n def auto_labels(station, start, end): """labeling/auto/.jsonl -> the labeller's labels touching [start, end).""" path = WORK / "labeling" / "auto" / f"{station}.jsonl" if not path.exists(): return [] with open(path, encoding="utf-8") as f: return [e for e in map(json.loads, filter(str.strip, f)) if e["debut"] < end and e["fin"] >= start] def family_scores(folder, start, end): """Score of a range per family and per instant -> ({family: [labelled, flagged, both]}, labels).""" # ponytail: per instant and not per sensor, the labels only give their sensors in the text; a sensors field in the # label if this precision is missing measurements = load(folder) labels = auto_labels(os.path.basename(os.path.normpath(folder)), start, end) evts = events(measurements, start, end, folder) instants = sorted({iso(m["t"]) for ms in measurements.values() for m in ms if start <= iso(m["t"]) < end}) # bounds excluded: they are the healthy measurement before and after (D-111); a hole, without a measurement # inside, keeps them empty = {i for i, e in enumerate(labels) if not any(e["debut"] < t < e["fin"] for t in instants)} pairs = [] for t in instants: a = {e.get("categorie", "no family") for i, e in enumerate(labels) if e["debut"] < t < e["fin"] or (i in empty and e["debut"] <= t <= e["fin"])} - {"None"} d = {v["category"] for v in evts if v["start"] <= t <= v["end"]} pairs.append((a, d)) return count(pairs), labels def pct(a, b): return f"{a}/{b} ({round(100 * a / b)} %)" if b else "—" def show(title, n): print(title) print(f" {'':<22}{'found':>20}{'correct':>20}") for k in sorted(n): labelled, flagged, both = n[k] print(f" {k:<22}{pct(both, labelled):>20}{pct(both, flagged):>20}") if __name__ == "__main__": p = argparse.ArgumentParser(usage=__doc__) p.add_argument("folder") p.add_argument("--from", dest="start", required=True) p.add_argument("--to", dest="end", required=True) a = p.parse_args() station, start, end = os.path.basename(os.path.normpath(a.folder)), iso(parse_time(a.start)), iso(parse_time(a.end)) n, labels = family_scores(a.folder, start, end) if not labels: sys.exit(f"No label of {station} on {start} → {end} in labeling/auto/{station}.jsonl: analyse the range first.") show(f"{station}: {start} → {end}, {len(labels)} labels of labeling/auto/{station}.jsonl, per family and per instant", n)