Files
aimsight-models/decode.py
T

132 lines
6.5 KiB
Python
Raw Normal View History

"""Step 6: predictions per instant -> episodes, scored per episode beside the rules decoded the same way, and the
sensors the attention points at, against the sensors the labels name.
Episodes: the flagged instants of a family, consecutive on the station's grid, a gap of one instant bridged.
Per family: labels found (a label holding an instant of the range, overlapped by an episode of its family), episodes
correct (an episode overlapping a label of its family), and the bounds: for each label found, the instants shared by
the label and the episodes touching it, over the instants of either (1 = the same bounds).
Sensors: for the labels naming 1 to 3 of the station's sensors, is a named sensor the first (top 1) or among the first
three (top 3) by the attention of the family averaged over the label's instants; beside a plain baseline, the sensors
ranked by their error_value against their usual level (feature ev_z).
--out: the model's episodes, one <STATION>.jsonl, in the detectors' format.
Usage: python decode.py <predictions> --cache <cache> --labels <labels folder> [--out <folder>]
"""
import argparse
import json
import re
import warnings
from pathlib import Path
import numpy as np
from dataset import FAMILIES
GAP = 1 # instants without a flag bridged inside an episode
def episodes(flags):
"""flags [T] -> [[i, j]], the index ranges of the episodes, bounds included."""
out = []
for i in np.flatnonzero(flags):
if out and i - out[-1][1] <= GAP + 1:
out[-1][1] = i
else:
out.append([i, i])
return out
def named(description, sensors):
return [k for k, s in enumerate(sensors) if re.search(rf"(?<![A-Za-z0-9_]){re.escape(s)}(?![A-Za-z0-9_])", description)]
def score_station(labels, instants, yhat, k):
"""-> [labels, labels found, episodes, episodes correct, [bounds of each label found]] for family k."""
eps = episodes(yhat[:, k])
spans = [np.flatnonzero((instants >= e["start"]) & (instants <= e["end"])) for e in labels]
spans = [s for s in spans if len(s)]
flagged = np.zeros(len(instants), bool)
for i, j in eps:
flagged[i:j + 1] = True
found, bounds = 0, []
for s in spans:
touching = [(i, j) for i, j in eps if i <= s[-1] and j >= s[0]]
if touching:
found += 1
mine = np.zeros(len(instants), bool)
for i, j in touching:
mine[i:j + 1] = True
lab = np.zeros(len(instants), bool)
lab[s] = True
bounds.append((mine & lab).sum() / (mine | lab).sum())
correct = sum(any(i <= s[-1] and j >= s[0] for s in spans) for i, j in eps)
return [len(spans), found, len(eps), correct, bounds]
def main():
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("predictions")
p.add_argument("--cache", required=True)
p.add_argument("--labels", required=True)
p.add_argument("--out")
a = p.parse_args()
ep = {} # (family, "rules"|"model") -> summed score
sens = {} # family -> [labels naming sensors, top1 attention, top3 attention, top1 ev_z, top3 ev_z]
for q in sorted(Path(a.predictions).glob("*.npz")):
pr, d = np.load(q), np.load(Path(a.cache) / q.name)
cut = int(pr["cut"]) if "cut" in pr else 0
instants, sensors = d["instants"][cut:], list(d["sensors"])
ev_z = d["xs"][cut:, :, list(d["xs_names"]).index("ev_z")]
rules = np.load(Path(a.cache) / "rules" / q.name)["yhat"][cut:]
labels = [json.loads(x) for x in open(Path(a.labels) / f"{q.stem}.jsonl")]
labels = [e for e in labels if e["end"] >= instants[0] and e["start"] <= instants[-1]]
out = []
for j, f in enumerate(pr["families"].tolist()):
k, mine = FAMILIES.index(f), [e for e in labels if e["family"] == f]
for who, yhat in (("rules", rules), ("model", pr["yhat"])):
c = score_station(mine, instants, yhat, k)
t = ep.setdefault((f, who), [0, 0, 0, 0, []])
ep[(f, who)] = [x + y for x, y in zip(t, c)]
s = sens.setdefault(f, [0, 0, 0, 0, 0])
for e in mine:
n, m = named(e["description"], sensors), (instants >= e["start"]) & (instants <= e["end"])
if not 1 <= len(n) <= 3 or not m.any():
continue
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning)
att = pr["attention"][m, :, j].astype(np.float32).mean(0)
z = np.nan_to_num(np.nanmean(ev_z[m], 0), nan=-np.inf)
ra, rz = np.argsort(-att), np.argsort(-z)
s[0] += 1
s[1] += ra[0] in n
s[2] += bool(set(ra[:3]) & set(n))
s[3] += rz[0] in n
s[4] += bool(set(rz[:3]) & set(n))
for i, i2 in episodes(pr["yhat"][:, k]):
att = pr["attention"][i:i2 + 1, :, j].astype(np.float32).mean(0)
top = [sensors[x] for x in np.argsort(-att)[:3] if att[x] >= 2 / len(sensors)]
out.append({"start": str(instants[i]), "end": str(instants[i2]), "kind": f, "label": f,
"text": f"{f}: model probability up to {pr['proba'][i:i2 + 1, j].max():.2f}",
"ignore": False, "sensors": top})
if a.out:
Path(a.out).mkdir(parents=True, exist_ok=True)
with open(Path(a.out) / f"{q.stem}.jsonl", "w") as w:
for v in sorted(out, key=lambda v: v["start"]):
w.write(json.dumps(v) + "\n")
def r(a_, b):
return f"{a_}/{b} ({round(100 * a_ / b)} %)" if b else "—"
print("Episodes")
print(f" {'':<14}{'labels found: rules':>22}{'model':>18}{'episodes correct: rules':>26}{'model':>18}"
f"{'bounds: rules':>15}{'model':>8}")
for f in sorted({f for f, _ in ep}, key=FAMILIES.index):
(n, fr, er, cr, br), (_, fm, em, cm, bm) = ep[(f, "rules")], ep[(f, "model")]
md = lambda x: f"{np.median(x):.2f}" if x else "—" # noqa: E731
print(f" {f:<14}{r(fr, n):>22}{r(fm, n):>18}{r(cr, er):>26}{r(cm, em):>18}{md(br):>15}{md(bm):>8}")
print("Sensors (labels naming 1 to 3 sensors: a named sensor first, or among the first three)")
print(f" {'':<14}{'labels':>8}{'top 1: attention':>20}{'ev_z':>14}{'top 3: attention':>20}{'ev_z':>14}")
for f, (n, a1, a3, z1, z3) in sens.items():
print(f" {f:<14}{n:>8}{r(a1, n):>20}{r(z1, n):>14}{r(a3, n):>20}{r(z3, n):>14}")
if __name__ == "__main__":
main()