Le benchmark dans bench/ ; plus de plugin : l'amélioration vit dans Optimizer, ses agents dans le dépôt agents

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 11:00:27 +01:00
1 parent 5438b0b469
commit e7c03e7304
9 files changed
+17 -316

No files matched your search

+55
View File
@@ -0,0 +1,55 @@
"""Benchmark of the detectors against a reference truth: every labelled station.
For each <STATION>.jsonl of --labels that has its folder <STATION>/ in --data, the score per family and per instant of
score.py, over the range of its labels: from the day of the first to the midnight after the last; with --from/--to, over
that range for every station, a station without a label on it included (several stations labelled the same days). Then the total,
family by family, which says whether the detectors do better overall. The block of a station is its score; the
others, the non-regression: a family that goes back on a station is a regression, even if the total rises.
Usage: python3 regression.py --labels <folder of <STATION>.jsonl> --data <folder of <STATION>/> [--stations A,B]
[--from YYYY-MM-DD --to YYYY-MM-DD]
"""
import argparse
from datetime import timedelta
from pathlib import Path
import score # next to it
from data import iso, parse_time # on the path through score
def label_range(path):
"""Labels JSONL -> (start, end) iso, in whole UTC days, end excluded."""
labels = score.read_labels(path)
start = min(e["debut"] for e in labels)[:10]
# ponytail: the end of the labelled batch is not written; midnight closest to the last bound (a label bound
# overflows by one measurement after midnight)
end = iso(parse_time(max(e["fin"] for e in labels)) + timedelta(hours=12))[:10]
return iso(parse_time(start)), iso(parse_time(end))
if __name__ == "__main__":
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("--labels", required=True, help="folder of the labels, one <STATION>.jsonl per station")
p.add_argument("--data", required=True, help="folder of the station folders, <STATION>/")
p.add_argument("--stations", help="A,B: these stations only (a shorter run)")
p.add_argument("--from", dest="start", help="YYYY-MM-DD: this range for every station, not the range of its labels")
p.add_argument("--to", dest="end", help="YYYY-MM-DD, excluded (with --from)")
a = p.parse_args()
if bool(a.start) != bool(a.end):
p.error("--from and --to go together")
total = {}
for path in sorted(Path(a.labels).glob("*.jsonl")):
station = path.stem
if a.stations and station not in a.stations.split(","):
continue
folder = Path(a.data) / station
if not folder.is_dir():
print(f"{station}: no {folder}/, skipped\n")
continue
start, end = (iso(parse_time(a.start)), iso(parse_time(a.end))) if a.start else label_range(path)
n, labels = score.family_scores(str(folder), str(path), start, end)
score.show(f"{station}: {start} → {end}, {len(labels)} labels", n)
print()
for k, v in n.items():
total[k] = [x + y for x, y in zip(total.get(k, [0, 0, 0]), v)]
score.show("Total", total)
+98
View File
@@ -0,0 +1,98 @@
"""Score of the detectors on a range: the detectors' events against the labels touching the range, per family and per
instant. An instant: a time of the range where at least one sensor has a measurement; it receives, on one side, the
families of the labels covering it, bounds excluded (except for a hole, without a measurement inside), on the other,
those of the detectors' events (categories.py, the range classified with its 3 days of history). The category of an
event is its family.
found: labelled instants that the detectors also flagged (share of the labelled ones);
correct: flagged instants that a label also puts in the family (share of the flagged ones).
The score is a fact, not a verdict: the labels can be wrong too. No label on the range: exits with an error.
Labels: a JSONL file, one label per line, {"id", "debut", "fin", "categorie", "description"} (UTC ISO times, categorie =
the family); its bounds are the healthy measurement before and after the anomaly.
Usage: python3 score.py <station folder> --labels <labels.jsonl> --from YYYY-MM-DD --to YYYY-MM-DD
"""
import argparse
import json
import os
import sys
from datetime import timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) # the detectors
import categories # noqa: E402
from data import iso, load, parse_time # noqa: E402
HISTORY = timedelta(days=3) # as the batch command of categories.py
def events(measurements, start, end, folder=None):
"""Detectors' events touching the range, classified with its days of history. Unlike the batch command, end is not
passed: no_data does not see a hole touching the end of the range."""
h, b = parse_time(start) - HISTORY, parse_time(end)
chunk = {s: k for s, ms in measurements.items() if (k := [m for m in ms if h <= m["t"] < b])}
return [v for v in (categories.events(chunk, folder=folder) if chunk else []) if v["end"] >= start]
def count(pairs):
"""[(labelled keys, flagged keys)], one pair per instant -> {key: [labelled, flagged, both]}."""
n = {}
for a, d in pairs:
for k in a | d:
x = n.setdefault(k, [0, 0, 0])
x[0], x[1], x[2] = x[0] + (k in a), x[1] + (k in d), x[2] + (k in a and k in d)
return n
def read_labels(path, start=None, end=None):
"""Labels JSONL -> the labels touching [start, end) (all without bounds); [] if the file does not exist."""
if not os.path.exists(path):
return []
with open(path, encoding="utf-8") as f:
return [e for e in map(json.loads, filter(str.strip, f))
if (end is None or e["debut"] < end) and (start is None or e["fin"] >= start)]
def family_scores(folder, labels_path, start, end):
"""Score of a range per family and per instant -> ({family: [labelled, flagged, both]}, labels)."""
# ponytail: per instant and not per sensor, the labels only give their sensors in the text; a sensors field in the
# label if this precision is missing
measurements = load(folder)
labels = read_labels(labels_path, start, end)
evts = events(measurements, start, end, folder)
instants = sorted({iso(m["t"]) for ms in measurements.values() for m in ms if start <= iso(m["t"]) < end})
# bounds excluded: they are the healthy measurement before and after; a hole, without a measurement inside, keeps them
empty = {i for i, e in enumerate(labels) if not any(e["debut"] < t < e["fin"] for t in instants)}
pairs = []
for t in instants:
a = {e.get("categorie", "no family") for i, e in enumerate(labels)
if e["debut"] < t < e["fin"] or (i in empty and e["debut"] <= t <= e["fin"])} - {"None"}
d = {v["category"] for v in evts if v["start"] <= t <= v["end"]}
pairs.append((a, d))
return count(pairs), labels
def pct(a, b):
return f"{a}/{b} ({round(100 * a / b)} %)" if b else "—"
def show(title, n):
print(title)
print(f" {'':<22}{'found':>20}{'correct':>20}")
for k in sorted(n):
labelled, flagged, both = n[k]
print(f" {k:<22}{pct(both, labelled):>20}{pct(both, flagged):>20}")
if __name__ == "__main__":
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("folder")
p.add_argument("--labels", required=True)
p.add_argument("--from", dest="start", required=True)
p.add_argument("--to", dest="end", required=True)
a = p.parse_args()
if not os.path.exists(a.labels):
sys.exit(f"No labels file: {a.labels}")
station, start, end = os.path.basename(os.path.normpath(a.folder)), iso(parse_time(a.start)), iso(parse_time(a.end))
n, labels = family_scores(a.folder, a.labels, start, end)
if not labels:
sys.exit(f"No label of {station} on {start} → {end} in {a.labels}.")
show(f"{station}: {start} → {end}, {len(labels)} labels of {a.labels}, per family and per instant", n)
+31
View File
@@ -0,0 +1,31 @@
"""For the upgrader: the cadence of the measurements of the range, the number of targets and references, then each event
of the range with its sensors (categories.py only gives their number). Same computation as categories.py without
--chunk (3 days of history).
Usage: python3 sensors.py <folder> --from YYYY-MM-DD --to YYYY-MM-DD
"""
import argparse
import statistics
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
import categories # noqa: E402
from data import iso, load # noqa: E402
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("folder")
p.add_argument("--from", dest="start", required=True)
p.add_argument("--to", dest="end", required=True)
a = p.parse_args()
day = lambda d: datetime.strptime(d[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc) # noqa: E731
start, end = day(a.start), day(a.end)
batch = {s: k for s, ms in load(a.folder).items() if (k := [m for m in ms if start - timedelta(days=3) <= m["t"] < end])}
ts = sorted({m["t"] for ms in batch.values() for m in ms if m["t"] >= start})
steps = [(y - x).total_seconds() / 60 for x, y in zip(ts, ts[1:])]
print(f"cadence: {statistics.median(steps):.0f} min (median of the steps between measurements)" if steps else "cadence: no measurement")
refs = sum(ms[0]["pattern_type"] == "global_ref" for ms in batch.values())
print(f"sensors: {len(batch) - refs} targets, {refs} references")
for v in (v for v in categories.events(batch, end, a.folder) if v["end"] >= iso(start)) if batch else []:
print(v["start"], "→", v["end"], f"{v['category']:<20}", "|", v["episodes"][0]["label"], "|",
", ".join(sorted(v["sensors"])) or "all sensors")