m2.py: one thread per process, --test to share the stations left out between processes, --out on the time split. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
133 lines
6.5 KiB
Python
133 lines
6.5 KiB
Python
"""Step 6: predictions per instant -> episodes, scored per episode beside the rules decoded the same way, and the
|
|
sensors the attention points at, against the sensors the labels name.
|
|
|
|
Episodes: the flagged instants of a family, consecutive on the station's grid, a gap of one instant bridged.
|
|
Per family: labels found (a label holding an instant of the range, overlapped by an episode of its family), episodes
|
|
correct (an episode overlapping a label of its family), and the bounds: for each label found, the instants shared by
|
|
the label and the episodes touching it, over the instants of either (1 = the same bounds).
|
|
Sensors: for the labels naming 1 to 3 of the station's sensors, is a named sensor the first (top 1) or among the first
|
|
three (top 3) by the attention of the family averaged over the label's instants; beside a plain baseline, the sensors
|
|
ranked by their error_value against their usual level (feature ev_z).
|
|
--out: the model's episodes, one <STATION>.jsonl, in the detectors' format.
|
|
Usage: python decode.py <predictions> --cache <cache> --labels <labels folder> [--out <folder>]
|
|
"""
|
|
import argparse
|
|
import json
|
|
import re
|
|
import warnings
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
from dataset import FAMILIES
|
|
|
|
GAP = 1 # instants without a flag bridged inside an episode
|
|
|
|
|
|
def episodes(flags):
|
|
"""flags [T] -> [[i, j]], the index ranges of the episodes, bounds included."""
|
|
out = []
|
|
for i in np.flatnonzero(flags):
|
|
if out and i - out[-1][1] <= GAP + 1:
|
|
out[-1][1] = i
|
|
else:
|
|
out.append([i, i])
|
|
return out
|
|
|
|
|
|
def named(description, sensors):
|
|
return [k for k, s in enumerate(sensors) if re.search(rf"(?<![A-Za-z0-9_]){re.escape(s)}(?![A-Za-z0-9_])", description)]
|
|
|
|
|
|
def score_station(labels, instants, yhat, k):
|
|
"""-> [labels, labels found, episodes, episodes correct, [bounds of each label found]] for family k."""
|
|
eps = episodes(yhat[:, k])
|
|
spans = [np.flatnonzero((instants >= e["start"]) & (instants <= e["end"])) for e in labels]
|
|
spans = [s for s in spans if len(s)]
|
|
flagged = np.zeros(len(instants), bool)
|
|
for i, j in eps:
|
|
flagged[i:j + 1] = True
|
|
found, bounds = 0, []
|
|
for s in spans:
|
|
touching = [(i, j) for i, j in eps if i <= s[-1] and j >= s[0]]
|
|
if touching:
|
|
found += 1
|
|
mine = np.zeros(len(instants), bool)
|
|
for i, j in touching:
|
|
mine[i:j + 1] = True
|
|
lab = np.zeros(len(instants), bool)
|
|
lab[s] = True
|
|
bounds.append((mine & lab).sum() / (mine | lab).sum())
|
|
correct = sum(any(i <= s[-1] and j >= s[0] for s in spans) for i, j in eps)
|
|
return [len(spans), found, len(eps), correct, bounds]
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(usage=__doc__)
|
|
p.add_argument("predictions")
|
|
p.add_argument("--cache", required=True)
|
|
p.add_argument("--labels", required=True)
|
|
p.add_argument("--out")
|
|
a = p.parse_args()
|
|
ep = {} # (family, "rules"|"model") -> summed score
|
|
sens = {} # family -> [labels naming sensors, top1 attention, top3 attention, top1 ev_z, top3 ev_z]
|
|
for q in sorted(Path(a.predictions).glob("*.npz")):
|
|
pr, d = np.load(q), np.load(Path(a.cache) / q.name)
|
|
cut = int(pr["cut"]) if "cut" in pr else 0
|
|
instants, sensors = d["instants"][cut:], list(d["sensors"])
|
|
ev_z = d["xs"][cut:, :, list(d["xs_names"]).index("ev_z")]
|
|
rules = np.load(Path(a.cache) / "rules" / q.name)["yhat"][cut:]
|
|
labels = [json.loads(x) for x in open(Path(a.labels) / f"{q.stem}.jsonl")]
|
|
labels = [e for e in labels if e["end"] >= instants[0] and e["start"] <= instants[-1]]
|
|
out = []
|
|
for j, f in enumerate(pr["families"].tolist()):
|
|
k, mine = FAMILIES.index(f), [e for e in labels if e["family"] == f]
|
|
for who, yhat in (("rules", rules), ("model", pr["yhat"])):
|
|
c = score_station(mine, instants, yhat, k)
|
|
t = ep.setdefault((f, who), [0, 0, 0, 0, []])
|
|
ep[(f, who)] = [x + y for x, y in zip(t, c)]
|
|
s = sens.setdefault(f, [0, 0, 0, 0, 0])
|
|
for e in mine:
|
|
n, m = named(e["description"], sensors), (instants >= e["start"]) & (instants <= e["end"])
|
|
if not 1 <= len(n) <= 3 or not m.any():
|
|
continue
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore", RuntimeWarning)
|
|
att = pr["attention"][m, :, j].astype(np.float32).mean(0)
|
|
z = np.nan_to_num(np.nanmean(ev_z[m], 0), nan=-np.inf)
|
|
ra, rz = np.argsort(-att), np.argsort(-z)
|
|
s[0] += 1
|
|
s[1] += ra[0] in n
|
|
s[2] += bool(set(ra[:3]) & set(n))
|
|
s[3] += rz[0] in n
|
|
s[4] += bool(set(rz[:3]) & set(n))
|
|
for i, i2 in episodes(pr["yhat"][:, k]):
|
|
att = pr["attention"][i:i2 + 1, :, j].astype(np.float32).mean(0)
|
|
top = [sensors[x] for x in np.argsort(-att)[:3] if att[x] >= 2 / len(sensors)]
|
|
out.append({"start": str(instants[i]), "end": str(instants[i2]), "kind": f, "label": f,
|
|
"text": f"{f}: model probability up to {pr['proba'][i:i2 + 1, j].max():.2f}",
|
|
"ignore": False, "sensors": top})
|
|
if a.out:
|
|
Path(a.out).mkdir(parents=True, exist_ok=True)
|
|
with open(Path(a.out) / f"{q.stem}.jsonl", "w") as w:
|
|
for v in sorted(out, key=lambda v: v["start"]):
|
|
w.write(json.dumps(v) + "\n")
|
|
|
|
def r(a_, b):
|
|
return f"{a_}/{b} ({round(100 * a_ / b)} %)" if b else "—"
|
|
print("Episodes")
|
|
print(f" {'':<14}{'labels found: rules':>22}{'model':>18}{'episodes correct: rules':>26}{'model':>18}"
|
|
f"{'bounds: rules':>15}{'model':>8}")
|
|
for f in sorted({f for f, _ in ep}, key=FAMILIES.index):
|
|
(n, fr, er, cr, br), (_, fm, em, cm, bm) = ep[(f, "rules")], ep[(f, "model")]
|
|
md = lambda x: f"{np.median(x):.2f}" if x else "—" # noqa: E731
|
|
print(f" {f:<14}{r(fr, n):>22}{r(fm, n):>18}{r(cr, er):>26}{r(cm, em):>18}{md(br):>15}{md(bm):>8}")
|
|
print("Sensors (labels naming 1 to 3 sensors: a named sensor first, or among the first three)")
|
|
print(f" {'':<14}{'labels':>8}{'top 1: attention':>20}{'ev_z':>14}{'top 3: attention':>20}{'ev_z':>14}")
|
|
for f, (n, a1, a3, z1, z3) in sens.items():
|
|
print(f" {f:<14}{n:>8}{r(a1, n):>20}{r(z1, n):>14}{r(a3, n):>20}{r(z3, n):>14}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|