Files
aimsight-detectors/scripts/learner/score.py
T

97 lines
4.7 KiB
Python
Raw Normal View History

"""Score of the detectors on a range: the detectors' events against the labels of labeling/auto/<STATION>.jsonl that
touch the range, per family (labeling/familles.md) and per instant (D-112). An instant: an hour of the range where at
least one sensor has a measurement; it receives, on one side, the families of the labels covering it, bounds excluded
(except for a hole, without a measurement inside), on the other, those of the detectors' events (categories.py, the
range classified with its 3 days of history). The category of an event is its family.
found: labelled instants that the detectors also flagged (share of the labelled ones);
correct: flagged instants that a label also puts in the family (share of the flagged ones).
The score is a fact, not a verdict: the labels can be wrong too. No label on the range: exits with an error.
Label fields (written by Diagnostic-app): debut, fin, categorie (the family), description.
Run from the working repository, where detectors/ is mounted.
Usage: python3 detectors/scripts/learner/score.py <station folder> --from YYYY-MM-DD --to YYYY-MM-DD
"""
import argparse
import json
import os
import sys
from datetime import timedelta
from pathlib import Path
REPO = Path(os.path.abspath(__file__)).parents[2] # the detectors repository
WORK = REPO.parent # the working repository (labeling/, data/)
sys.path.insert(0, str(REPO / "scripts"))
import categories # noqa: E402
from data import iso, load, parse_time # noqa: E402
HISTORY = timedelta(days=3) # as the batch command (D-056)
def events(measurements, start, end, folder=None):
"""Detectors' events touching the range, classified with its days of history (as the batch command)."""
h, b = parse_time(start) - HISTORY, parse_time(end)
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
return [v for v in (categories.events(chunk, folder=folder) if chunk else []) if v["end"] >= start]
def count(pairs):
"""[(labelled keys, flagged keys)], one pair per instant -> {key: [labelled, flagged, both]}."""
n = {}
for a, d in pairs:
for k in a | d:
x = n.setdefault(k, [0, 0, 0])
x[0], x[1], x[2] = x[0] + (k in a), x[1] + (k in d), x[2] + (k in a and k in d)
return n
def auto_labels(station, start, end):
"""labeling/auto/<STATION>.jsonl -> the labeller's labels touching [start, end)."""
path = WORK / "labeling" / "auto" / f"{station}.jsonl"
if not path.exists():
return []
with open(path, encoding="utf-8") as f:
return [e for e in map(json.loads, filter(str.strip, f)) if e["debut"] < end and e["fin"] >= start]
def family_scores(folder, start, end):
"""Score of a range per family and per instant -> ({family: [labelled, flagged, both]}, labels)."""
# ponytail: per instant and not per sensor, the labels only give their sensors in the text; a sensors field in the
# label if this precision is missing
measurements = load(folder)
labels = auto_labels(os.path.basename(os.path.normpath(folder)), start, end)
evts = events(measurements, start, end, folder)
instants = sorted({iso(m["t"]) for ms in measurements.values() for m in ms if start <= iso(m["t"]) < end})
# bounds excluded: they are the healthy measurement before and after (D-111); a hole, without a measurement
# inside, keeps them
empty = {i for i, e in enumerate(labels) if not any(e["debut"] < t < e["fin"] for t in instants)}
pairs = []
for t in instants:
a = {e.get("categorie", "no family") for i, e in enumerate(labels)
if e["debut"] < t < e["fin"] or (i in empty and e["debut"] <= t <= e["fin"])} - {"None"}
d = {v["category"] for v in evts if v["start"] <= t <= v["end"]}
pairs.append((a, d))
return count(pairs), labels
def pct(a, b):
return f"{a}/{b} ({round(100 * a / b)} %)" if b else "—"
def show(title, n):
print(title)
print(f" {'':<22}{'found':>20}{'correct':>20}")
for k in sorted(n):
labelled, flagged, both = n[k]
print(f" {k:<22}{pct(both, labelled):>20}{pct(both, flagged):>20}")
if __name__ == "__main__":
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("folder")
p.add_argument("--from", dest="start", required=True)
p.add_argument("--to", dest="end", required=True)
a = p.parse_args()
station, start, end = os.path.basename(os.path.normpath(a.folder)), iso(parse_time(a.start)), iso(parse_time(a.end))
n, labels = family_scores(a.folder, start, end)
if not labels:
sys.exit(f"No label of {station} on {start} → {end} in labeling/auto/{station}.jsonl: analyse the range first.")
show(f"{station}: {start} → {end}, {len(labels)} labels of labeling/auto/{station}.jsonl, per family and per instant", n)