"""Score of the detectors on a range: the detectors' events against the labels touching the range, per family and per instant. An instant: a time of the range where at least one sensor has a measurement; it receives, on one side, the families of the labels covering it, bounds excluded (except for a hole, without a measurement inside), on the other, those of the detectors' events (categories.py, the range classified with its 3 days of history). The category of an event is its family. found: labelled instants that the detectors also flagged (share of the labelled ones); correct: flagged instants that a label also puts in the family (share of the flagged ones). The score is a fact, not a verdict: the labels can be wrong too. No label on the range: exits with an error. Labels: a JSONL file, one label per line, {"id", "start", "end", "family", "description"} (UTC ISO times); the family); its bounds are the healthy measurement before and after the anomaly. Usage: python3 score.py --labels --from YYYY-MM-DD --to YYYY-MM-DD """ import argparse import json import os import sys from datetime import timedelta from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) # the detectors import categories # noqa: E402 from data import iso, load, parse_time # noqa: E402 HISTORY = timedelta(days=3) # as the batch command of categories.py def events(measurements, start, end, folder=None): """Detectors' events touching the range, classified with its days of history. Unlike the batch command, end is not passed: no_data does not see a hole touching the end of the range.""" h, b = parse_time(start) - HISTORY, parse_time(end) chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])} return [v for v in (categories.events(chunk, folder=folder) if chunk else []) if v["end"] >= start] def count(pairs): """[(labelled keys, flagged keys)], one pair per instant -> {key: [labelled, flagged, both]}.""" n = {} for a, d in pairs: for k in a | d: x = n.setdefault(k, [0, 0, 0]) x[0], x[1], x[2] = x[0] + (k in a), x[1] + (k in d), x[2] + (k in a and k in d) return n def read_labels(path, start=None, end=None): """Labels JSONL -> the labels touching [start, end) (all without bounds); [] if the file does not exist.""" if not os.path.exists(path): return [] with open(path, encoding="utf-8") as f: return [e for e in map(json.loads, filter(str.strip, f)) if (end is None or e["start"] < end) and (start is None or e["end"] >= start)] def family_scores(folder, labels_path, start, end): """Score of a range per family and per instant -> ({family: [labelled, flagged, both]}, labels).""" # ponytail: per instant and not per sensor, the labels only give their sensors in the text; a sensors field in the # label if this precision is missing measurements = load(folder) labels = read_labels(labels_path, start, end) evts = events(measurements, start, end, folder) instants = sorted({iso(m["t"]) for ms in measurements.values() for m in ms if start <= iso(m["t"]) < end}) # bounds excluded: they are the healthy measurement before and after; a hole, without a measurement inside, keeps them empty = {i for i, e in enumerate(labels) if not any(e["start"] < t < e["end"] for t in instants)} pairs = [] for t in instants: a = {e.get("family", "no family") for i, e in enumerate(labels) if e["start"] < t < e["end"] or (i in empty and e["start"] <= t <= e["end"])} - {"None"} d = {v["category"] for v in evts if v["start"] <= t <= v["end"]} pairs.append((a, d)) return count(pairs), labels def pct(a, b): return f"{a}/{b} ({round(100 * a / b)} %)" if b else "—" def show(title, n): print(title) print(f" {'':<22}{'found':>20}{'correct':>20}") for k in sorted(n): labelled, flagged, both = n[k] print(f" {k:<22}{pct(both, labelled):>20}{pct(both, flagged):>20}") if __name__ == "__main__": p = argparse.ArgumentParser(usage=__doc__) p.add_argument("folder") p.add_argument("--labels", required=True) p.add_argument("--from", dest="start", required=True) p.add_argument("--to", dest="end", required=True) a = p.parse_args() if not os.path.exists(a.labels): sys.exit(f"No labels file: {a.labels}") station, start, end = os.path.basename(os.path.normpath(a.folder)), iso(parse_time(a.start)), iso(parse_time(a.end)) n, labels = family_scores(a.folder, a.labels, start, end) if not labels: sys.exit(f"No label of {station} on {start} → {end} in {a.labels}.") show(f"{station}: {start} → {end}, {len(labels)} labels of {a.labels}, per family and per instant", n)