wf-improve scripts take their paths as arguments (--labels, --data)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
80e00a4898
commit
cff2b0ad1e
3 files changed
+40
-42
No files matched your search
@@ -1,15 +1,14 @@
|
||||
"""Benchmark of the wf-improve workflow (D-116): the detectors against all the labeller's labels (labeling/auto/).
|
||||
"""Benchmark of the detectors against a reference truth: every labelled station.
|
||||
|
||||
For each station of labeling/auto/ that has its folder in data/, the score per family and per instant of score.py
|
||||
(D-112), over the range of its labels: from the day of the first to the midnight after the last. Then the total, family
|
||||
by family, which says whether the detectors do better overall from one session to the next. The block of the session's
|
||||
station is its score; the other blocks, the non-regression: a family that goes back on a station is a regression, even
|
||||
if the total rises.
|
||||
Usage: python3 detectors/skills/wf-improve/scripts/regression.py [--stations A,B]
|
||||
For each <STATION>.jsonl of --labels that has its folder <STATION>/ in --data, the score per family and per instant of
|
||||
score.py, over the range of its labels: from the day of the first to the midnight after the last. Then the total,
|
||||
family by family, which says whether the detectors do better overall. The block of a station is its score; the
|
||||
others, the non-regression: a family that goes back on a station is a regression, even if the total rises.
|
||||
Usage: python3 regression.py --labels <folder of <STATION>.jsonl> --data <folder of <STATION>/> [--stations A,B]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import score # next to it
|
||||
|
||||
@@ -17,31 +16,32 @@ from data import iso, parse_time # on the path through score
|
||||
|
||||
|
||||
def label_range(path):
|
||||
"""labeling/auto/<STATION>.jsonl -> (start, end) iso, in whole UTC days, end excluded."""
|
||||
with open(path, encoding="utf-8") as f:
|
||||
labels = [json.loads(l) for l in f if l.strip()]
|
||||
"""Labels JSONL -> (start, end) iso, in whole UTC days, end excluded."""
|
||||
labels = score.read_labels(path)
|
||||
start = min(e["debut"] for e in labels)[:10]
|
||||
# ponytail: the end of the batch is not written; midnight closest to the last bound (a label bound overflows by one
|
||||
# measurement after midnight); the real end of the batch in labeling/auto/ if the gap matters
|
||||
# ponytail: the end of the labelled batch is not written; midnight closest to the last bound (a label bound
|
||||
# overflows by one measurement after midnight)
|
||||
end = iso(parse_time(max(e["fin"] for e in labels)) + timedelta(hours=12))[:10]
|
||||
return iso(parse_time(start)), iso(parse_time(end))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
p = argparse.ArgumentParser(usage=__doc__)
|
||||
p.add_argument("--labels", required=True, help="folder of the labels, one <STATION>.jsonl per station")
|
||||
p.add_argument("--data", required=True, help="folder of the station folders, <STATION>/")
|
||||
p.add_argument("--stations", help="A,B: these stations only (a shorter run)")
|
||||
a = p.parse_args()
|
||||
total = {}
|
||||
for path in sorted((score.WORK / "labeling" / "auto").glob("*.jsonl")):
|
||||
for path in sorted(Path(a.labels).glob("*.jsonl")):
|
||||
station = path.stem
|
||||
if a.stations and station not in a.stations.split(","):
|
||||
continue
|
||||
folder = score.WORK / "data" / station
|
||||
folder = Path(a.data) / station
|
||||
if not folder.is_dir():
|
||||
print(f"{station}: no data/{station}/, skipped\n")
|
||||
print(f"{station}: no {folder}/, skipped\n")
|
||||
continue
|
||||
start, end = label_range(path)
|
||||
n, labels = score.family_scores(str(folder), start, end)
|
||||
n, labels = score.family_scores(str(folder), str(path), start, end)
|
||||
score.show(f"{station}: {start} → {end}, {len(labels)} labels", n)
|
||||
print()
|
||||
for k, v in n.items():
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
"""Score of the detectors on a range: the detectors' events against the labels of labeling/auto/<STATION>.jsonl that
|
||||
touch the range, per family (labeling/familles.md) and per instant (D-112). An instant: an hour of the range where at
|
||||
least one sensor has a measurement; it receives, on one side, the families of the labels covering it, bounds excluded
|
||||
(except for a hole, without a measurement inside), on the other, those of the detectors' events (categories.py, the
|
||||
range classified with its 3 days of history). The category of an event is its family.
|
||||
"""Score of the detectors on a range: the detectors' events against the labels touching the range, per family and per
|
||||
instant. An instant: a time of the range where at least one sensor has a measurement; it receives, on one side, the
|
||||
families of the labels covering it, bounds excluded (except for a hole, without a measurement inside), on the other,
|
||||
those of the detectors' events (categories.py, the range classified with its 3 days of history). The category of an
|
||||
event is its family.
|
||||
found: labelled instants that the detectors also flagged (share of the labelled ones);
|
||||
correct: flagged instants that a label also puts in the family (share of the flagged ones).
|
||||
The score is a fact, not a verdict: the labels can be wrong too. No label on the range: exits with an error.
|
||||
Label fields (written by Diagnostic-app): debut, fin, categorie (the family), description.
|
||||
Run from the working repository, where detectors/ is mounted.
|
||||
Usage: python3 detectors/skills/wf-improve/scripts/score.py <station folder> --from YYYY-MM-DD --to YYYY-MM-DD
|
||||
Labels: a JSONL file, one label per line, {"id", "debut", "fin", "categorie", "description"} (UTC ISO times, categorie =
|
||||
the family); its bounds are the healthy measurement before and after the anomaly.
|
||||
Usage: python3 score.py <station folder> --labels <labels.jsonl> --from YYYY-MM-DD --to YYYY-MM-DD
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
@@ -17,13 +17,11 @@ import sys
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(os.path.abspath(__file__)).parents[3] # the detectors repository
|
||||
WORK = REPO.parent # the working repository (labeling/, data/)
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "scripts")) # the detectors
|
||||
import categories # noqa: E402
|
||||
from data import iso, load, parse_time # noqa: E402
|
||||
|
||||
HISTORY = timedelta(days=3) # as the batch command (D-056)
|
||||
HISTORY = timedelta(days=3) # as the batch command of categories.py
|
||||
|
||||
|
||||
def events(measurements, start, end, folder=None):
|
||||
@@ -43,25 +41,24 @@ def count(pairs):
|
||||
return n
|
||||
|
||||
|
||||
def auto_labels(station, start, end):
|
||||
"""labeling/auto/<STATION>.jsonl -> the labeller's labels touching [start, end)."""
|
||||
path = WORK / "labeling" / "auto" / f"{station}.jsonl"
|
||||
if not path.exists():
|
||||
def read_labels(path, start=None, end=None):
|
||||
"""Labels JSONL -> the labels touching [start, end) (all without bounds); [] if the file does not exist."""
|
||||
if not os.path.exists(path):
|
||||
return []
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return [e for e in map(json.loads, filter(str.strip, f)) if e["debut"] < end and e["fin"] >= start]
|
||||
return [e for e in map(json.loads, filter(str.strip, f))
|
||||
if (end is None or e["debut"] < end) and (start is None or e["fin"] >= start)]
|
||||
|
||||
|
||||
def family_scores(folder, start, end):
|
||||
def family_scores(folder, labels_path, start, end):
|
||||
"""Score of a range per family and per instant -> ({family: [labelled, flagged, both]}, labels)."""
|
||||
# ponytail: per instant and not per sensor, the labels only give their sensors in the text; a sensors field in the
|
||||
# label if this precision is missing
|
||||
measurements = load(folder)
|
||||
labels = auto_labels(os.path.basename(os.path.normpath(folder)), start, end)
|
||||
labels = read_labels(labels_path, start, end)
|
||||
evts = events(measurements, start, end, folder)
|
||||
instants = sorted({iso(m["t"]) for ms in measurements.values() for m in ms if start <= iso(m["t"]) < end})
|
||||
# bounds excluded: they are the healthy measurement before and after (D-111); a hole, without a measurement
|
||||
# inside, keeps them
|
||||
# bounds excluded: they are the healthy measurement before and after; a hole, without a measurement inside, keeps them
|
||||
empty = {i for i, e in enumerate(labels) if not any(e["debut"] < t < e["fin"] for t in instants)}
|
||||
pairs = []
|
||||
for t in instants:
|
||||
@@ -87,11 +84,12 @@ def show(title, n):
|
||||
if __name__ == "__main__":
|
||||
p = argparse.ArgumentParser(usage=__doc__)
|
||||
p.add_argument("folder")
|
||||
p.add_argument("--labels", required=True)
|
||||
p.add_argument("--from", dest="start", required=True)
|
||||
p.add_argument("--to", dest="end", required=True)
|
||||
a = p.parse_args()
|
||||
station, start, end = os.path.basename(os.path.normpath(a.folder)), iso(parse_time(a.start)), iso(parse_time(a.end))
|
||||
n, labels = family_scores(a.folder, start, end)
|
||||
n, labels = family_scores(a.folder, a.labels, start, end)
|
||||
if not labels:
|
||||
sys.exit(f"No label of {station} on {start} → {end} in labeling/auto/{station}.jsonl: analyse the range first.")
|
||||
show(f"{station}: {start} → {end}, {len(labels)} labels of labeling/auto/{station}.jsonl, per family and per instant", n)
|
||||
sys.exit(f"No label of {station} on {start} → {end} in {a.labels}.")
|
||||
show(f"{station}: {start} → {end}, {len(labels)} labels of {a.labels}, per family and per instant", n)
|
||||
@@ -1,7 +1,7 @@
|
||||
"""For the upgrader: the cadence of the measurements of the range, the number of targets and references, then each event
|
||||
of the range with its sensors (categories.py only gives their number). Same computation as categories.py without
|
||||
--chunk (3 days of history).
|
||||
Usage: python3 detectors/skills/wf-improve/scripts/sensors.py <folder> --from YYYY-MM-DD --to YYYY-MM-DD
|
||||
Usage: python3 sensors.py <folder> --from YYYY-MM-DD --to YYYY-MM-DD
|
||||
"""
|
||||
import argparse
|
||||
import statistics
|
||||
|
||||
Reference in new issue
Block a user