2026-11-13 12:38:29 +01:00
|
|
|
"""Score of the detectors on a range: the detectors' events against the labels touching the range, per family and per
|
|
|
|
|
instant. An instant: a time of the range where at least one sensor has a measurement; it receives, on one side, the
|
|
|
|
|
families of the labels covering it, bounds excluded (except for a hole, without a measurement inside), on the other,
|
|
|
|
|
those of the detectors' events (categories.py, the range classified with its 3 days of history). The category of an
|
|
|
|
|
event is its family.
|
2026-11-13 12:19:52 +01:00
|
|
|
found: labelled instants that the detectors also flagged (share of the labelled ones);
|
|
|
|
|
correct: flagged instants that a label also puts in the family (share of the flagged ones).
|
|
|
|
|
The score is a fact, not a verdict: the labels can be wrong too. No label on the range: exits with an error.
|
2026-11-16 12:40:03 +01:00
|
|
|
Labels: a JSONL file, one label per line, {"id", "start", "end", "family", "description"} (UTC ISO times);
|
2026-11-13 12:38:29 +01:00
|
|
|
the family); its bounds are the healthy measurement before and after the anomaly.
|
|
|
|
|
Usage: python3 score.py <station folder> --labels <labels.jsonl> --from YYYY-MM-DD --to YYYY-MM-DD
|
2026-11-13 12:19:52 +01:00
|
|
|
"""
|
|
|
|
|
import argparse
|
|
|
|
|
import json
|
|
|
|
|
import os
|
|
|
|
|
import sys
|
|
|
|
|
from datetime import timedelta
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
2026-11-16 11:00:27 +01:00
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) # the detectors
|
2026-11-13 12:19:52 +01:00
|
|
|
import categories # noqa: E402
|
|
|
|
|
from data import iso, load, parse_time # noqa: E402
|
|
|
|
|
|
2026-11-13 12:38:29 +01:00
|
|
|
HISTORY = timedelta(days=3) # as the batch command of categories.py
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def events(measurements, start, end, folder=None):
|
2026-11-13 12:45:14 +01:00
|
|
|
"""Detectors' events touching the range, classified with its days of history. Unlike the batch command, end is not
|
|
|
|
|
passed: no_data does not see a hole touching the end of the range."""
|
2026-11-13 12:19:52 +01:00
|
|
|
h, b = parse_time(start) - HISTORY, parse_time(end)
|
|
|
|
|
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
|
|
|
|
|
return [v for v in (categories.events(chunk, folder=folder) if chunk else []) if v["end"] >= start]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def count(pairs):
|
|
|
|
|
"""[(labelled keys, flagged keys)], one pair per instant -> {key: [labelled, flagged, both]}."""
|
|
|
|
|
n = {}
|
|
|
|
|
for a, d in pairs:
|
|
|
|
|
for k in a | d:
|
|
|
|
|
x = n.setdefault(k, [0, 0, 0])
|
|
|
|
|
x[0], x[1], x[2] = x[0] + (k in a), x[1] + (k in d), x[2] + (k in a and k in d)
|
|
|
|
|
return n
|
|
|
|
|
|
|
|
|
|
|
2026-11-13 12:38:29 +01:00
|
|
|
def read_labels(path, start=None, end=None):
|
|
|
|
|
"""Labels JSONL -> the labels touching [start, end) (all without bounds); [] if the file does not exist."""
|
|
|
|
|
if not os.path.exists(path):
|
2026-11-13 12:19:52 +01:00
|
|
|
return []
|
|
|
|
|
with open(path, encoding="utf-8") as f:
|
2026-11-13 12:38:29 +01:00
|
|
|
return [e for e in map(json.loads, filter(str.strip, f))
|
2026-11-16 12:40:03 +01:00
|
|
|
if (end is None or e["start"] < end) and (start is None or e["end"] >= start)]
|
2026-11-13 12:19:52 +01:00
|
|
|
|
|
|
|
|
|
2026-11-13 12:38:29 +01:00
|
|
|
def family_scores(folder, labels_path, start, end):
|
2026-11-13 12:19:52 +01:00
|
|
|
"""Score of a range per family and per instant -> ({family: [labelled, flagged, both]}, labels)."""
|
|
|
|
|
# ponytail: per instant and not per sensor, the labels only give their sensors in the text; a sensors field in the
|
|
|
|
|
# label if this precision is missing
|
|
|
|
|
measurements = load(folder)
|
2026-11-13 12:38:29 +01:00
|
|
|
labels = read_labels(labels_path, start, end)
|
2026-11-13 12:19:52 +01:00
|
|
|
evts = events(measurements, start, end, folder)
|
|
|
|
|
instants = sorted({iso(m["t"]) for ms in measurements.values() for m in ms if start <= iso(m["t"]) < end})
|
2026-11-13 12:38:29 +01:00
|
|
|
# bounds excluded: they are the healthy measurement before and after; a hole, without a measurement inside, keeps them
|
2026-11-16 12:40:03 +01:00
|
|
|
empty = {i for i, e in enumerate(labels) if not any(e["start"] < t < e["end"] for t in instants)}
|
2026-11-13 12:19:52 +01:00
|
|
|
pairs = []
|
|
|
|
|
for t in instants:
|
2026-11-16 12:40:03 +01:00
|
|
|
a = {e.get("family", "no family") for i, e in enumerate(labels)
|
|
|
|
|
if e["start"] < t < e["end"] or (i in empty and e["start"] <= t <= e["end"])} - {"None"}
|
2026-11-13 12:19:52 +01:00
|
|
|
d = {v["category"] for v in evts if v["start"] <= t <= v["end"]}
|
|
|
|
|
pairs.append((a, d))
|
|
|
|
|
return count(pairs), labels
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def pct(a, b):
|
|
|
|
|
return f"{a}/{b} ({round(100 * a / b)} %)" if b else "—"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def show(title, n):
|
|
|
|
|
print(title)
|
|
|
|
|
print(f" {'':<22}{'found':>20}{'correct':>20}")
|
|
|
|
|
for k in sorted(n):
|
|
|
|
|
labelled, flagged, both = n[k]
|
|
|
|
|
print(f" {k:<22}{pct(both, labelled):>20}{pct(both, flagged):>20}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
p = argparse.ArgumentParser(usage=__doc__)
|
|
|
|
|
p.add_argument("folder")
|
2026-11-13 12:38:29 +01:00
|
|
|
p.add_argument("--labels", required=True)
|
2026-11-13 12:19:52 +01:00
|
|
|
p.add_argument("--from", dest="start", required=True)
|
|
|
|
|
p.add_argument("--to", dest="end", required=True)
|
|
|
|
|
a = p.parse_args()
|
2026-11-13 12:45:14 +01:00
|
|
|
if not os.path.exists(a.labels):
|
|
|
|
|
sys.exit(f"No labels file: {a.labels}")
|
2026-11-13 12:19:52 +01:00
|
|
|
station, start, end = os.path.basename(os.path.normpath(a.folder)), iso(parse_time(a.start)), iso(parse_time(a.end))
|
2026-11-13 12:38:29 +01:00
|
|
|
n, labels = family_scores(a.folder, a.labels, start, end)
|
2026-11-13 12:19:52 +01:00
|
|
|
if not labels:
|
2026-11-13 12:38:29 +01:00
|
|
|
sys.exit(f"No label of {station} on {start} → {end} in {a.labels}.")
|
|
|
|
|
show(f"{station}: {start} → {end}, {len(labels)} labels of {a.labels}, per family and per instant", n)
|