From 4836929bc7d0f88c600a1e75f5a7f3d7fa528539 Mon Sep 17 00:00:00 2001 From: maxime Date: Tue, 17 Nov 2026 11:59:49 +0100 Subject: [PATCH] Steps 5 and 6: pre-trained M2 passes the criterion for Obstruction, not for Visibility; episodes and sensors (decode.py) m2.py: one thread per process, --test to share the stations left out between processes, --out on the time split. Co-Authored-By: Claude Opus 5.5 --- README.md | 69 +++++++++++++++----------- decode.py | 132 +++++++++++++++++++++++++++++++++++++++++++++++++ m2.py | 40 +++++++++------ test_decode.py | 13 +++++ 4 files changed, 212 insertions(+), 42 deletions(-) create mode 100644 decode.py create mode 100644 test_decode.py diff --git a/README.md b/README.md index 7276535..e6bdf9a 100644 --- a/README.md +++ b/README.md @@ -6,61 +6,74 @@ Learned detectors for AIMsight: models trained on the reviewed labels to tell, a - **Families**: Visibility and Obstruction; the others stay rules for now. - **Model**: multiple-instance learning, since a label names a station and a range, not its sensors, and each station has its own number of sensors: one small MLP shared by every sensor, attention pooling to a score per family for the station; the attention weights give the sensors. -- **Inputs**: each sensor against its own usual level (median and MAD over the 3 days before), on windows in hours (the step goes from 5 min to 4 h); the station context (share of high and rejected sensors, `sigma0`, references, camera pose, housing temperature, common mode by PCA); the image quality metrics when present, with a mask. +- **Inputs**: each sensor against its own usual level (median and IQR over the 3 days before), on windows in hours (the step goes from 5 min to 4 h); the station context (share of high and rejected sensors, `sigma0`, references, camera pose, housing temperature, common mode as the median over the targets); the image quality metrics when present, with a mask. - **Training**: pre-training on the outputs of the current detectors over all the data, then fine-tuning on the reviewed labels of `labeling/auto/`. - **Evaluation**: the measure of `detectors/bench/score.py` (found, correct, per family and per instant), against the rules on the same instants. -- **The criterion** (decided 2026-11-16): on the stations seen, the way the model will run (every station followed keeps being labelled in the app): trained on the start of every station, tested on its end (a cut in time, never at random). The model beats the rules by at least 5 points of found and of correct, in total, for Visibility and for Obstruction, and loses nowhere more than 2 points on a family of a station (the regression of `detectors/CLAUDE.md`). A new station stays under the rules until it has a few weeks of reviewed labels; the score with a station left out is followed at every step, as an indicator, not a gate. +- **The criterion** (decided 2026-11-16): on the stations seen, the way the model will run (every station followed keeps being labelled in the app): trained on the start of every station, tested on its end (a cut in time, never at random). The model beats the rules by at least 5 points of found and of correct, in total, for Visibility and for Obstruction, with no regression as `detectors/bench/compare.py` counts them (a family of a station losing more than 5 points and more than 20 instants; since 2026-11-16, before: 2 points). A new station stays under the rules until it has a few weeks of reviewed labels; the score with a station left out is followed at every step, as an indicator, not a gate. - **Steps**: dataset, evaluator (reproducing the rules' figures first), gradient boosting baseline, the PyTorch model, pre-training compared with and without, decoding into episodes. ## Use ```bash uv venv .venv && uv pip install -p .venv -r requirements.txt -.venv/bin/python test_dataset.py ~/aimsight-detectors +.venv/bin/python test_dataset.py ~/aimsight-detectors && .venv/bin/python test_evaluate.py && .venv/bin/python test_decode.py .venv/bin/python dataset.py --detectors ~/aimsight-detectors --data ~/aimsight-data/data --labels ~/aimsight-labeling/auto --out cache -``` - -`dataset.py` (step 1, ~2 min): one `cache/.npz` per labelled station, over its labelled range as the benchmark reads it: `xs` [instant, sensor, feature], `xc` [instant, feature], `y` [instant, family] with the conventions of `bench/score.py` (its labelled instants per family are the benchmark's, exactly). NaN where a value does not exist (a 1 h window at a 4 h step, no temperature, no image quality outside LENZSPITZE). - -```bash -.venv/bin/python test_evaluate.py .venv/bin/python evaluate.py rules --detectors ~/aimsight-detectors --data ~/aimsight-data/data --labels ~/aimsight-labeling/auto --cache cache -.venv/bin/python evaluate.py score --cache cache +.venv/bin/python pretrain.py --detectors ~/aimsight-detectors --data ~/aimsight-data/data --out cache/pre ``` -`evaluate.py rules` (step 2, ~20 min, once per change of the detectors): the rules' predictions per instant in `cache/rules/`, and the check that their counts are those of `bench/score.py`, station by station (exit 1 otherwise). `evaluate.py score`: a folder of `.npz` with `yhat` [instant, family] against the labels, beside the rules, with the regressions of `detectors/CLAUDE.md` (more than 2 points of found or correct lost on a family of a station). +- `dataset.py` (step 1, ~1 min): one `cache/.npz` per labelled station, over its labelled range as the benchmark reads it: `xs` [instant, sensor, feature], `xc` [instant, feature], `y` [instant, family] with the conventions of `bench/score.py` (its labelled instants per family, exactly). NaN where a value does not exist (a 1 h window at a 4 h step, no temperature, no image quality outside LENZSPITZE). +- `evaluate.py rules` (step 2, ~1 min, after every change of the detectors or the labels): the rules' predictions per instant in `cache/rules/`, checked against the counts of `bench/score.py` station by station (exit 1 otherwise). `evaluate.py score `: predictions per station against the labels, beside the rules, with the regressions of `bench/compare.py`. +- `pretrain.py` (step 5, ~3 min): every station of the data folder, labelled or not, over its whole range, in chunks of two weeks: the features and the families the rules flag (`cache/pre/`). ```bash -.venv/bin/python m1.py --cache cache --out runs/m1 --balance && .venv/bin/python evaluate.py score runs/m1 --cache cache .venv/bin/python m1.py --cache cache --out - --split-time --balance +.venv/bin/python m2.py --cache cache --pretrain cache/pre --out runs/m2_split_pre +for g in BREITHORN,CLARIDEN,FLORIDA JUNGFRAU,LENZSPITZE,MICHIGAN NEBRASKA,NIEDERHORN,NORDEND SNCF01,WILDSTRUBEL; do + .venv/bin/python m2.py --cache cache --pretrain cache/pre --leave-out --seeds 3 --test $g --out runs/m2_pre & done; wait +.venv/bin/python evaluate.py score runs/m2_pre --cache cache +.venv/bin/python decode.py runs/m2_split_pre --cache cache --labels ~/aimsight-labeling/auto --out runs/episodes ``` -`m1.py` (step 3, a few minutes): gradient boosting on the station's features and the aggregates of its sensors', Visibility and Obstruction, leaving one station out; `--split-time`, the diagnosis on the last 30 % of every station, all stations seen in training. First results (2026-11-16, found / correct, total): +- `m1.py` (step 3, ~4 min): gradient boosting on the station's features and the aggregates of its sensors' (median, q90, min, max over the targets and over the references); `--split-time` is the criterion, without it a station left out. +- `m2.py` (steps 4 and 5, PyTorch, CPU, one thread per process): multiple-instance learning, one MLP shared by every sensor and an attention per family over the sensors. One model alone varies by 10 to 20 points with its seed: 5 averaged (3 with a station left out). `--pretrain`: first the six families the rules flag on every station and month (never the instants tested), then the fine-tuning on the reviewed labels. Time split ~40 min; a station left out, ~1 h in four processes (memory: ~0.6 GB each with the pre-training set). +- `decode.py` (step 6): the flagged instants into episodes (a gap of one instant bridged), scored per episode beside the rules decoded the same way; the sensors of the attention against the sensors the labels name; `--out`, the episodes in the detectors' format. + +## Results + +On 2026-11-17, after the simplification of the detectors (2026-11-16), which raised the rules a lot (on the last 30 %, Visibility found 49 → 58 %, Obstruction 15 → 33 %). Found / correct per instant, in total: | | Visibility | Obstruction | |---|---|---| -| rules, all stations | 47 / 77 % | 26 / 28 % | -| M1, station left out | 44 / 62 % | 9 / 7 % | -| M1, station left out, `--balance` | 46 / 70 % | 21 / 23 % | -| rules, last 30 % | 49 / 77 % | 15 / 41 % | -| M1, last 30 % (`--split-time`) | 58 / 87 % | 39 / 75 % | -| M2, last 30 % (5 seeds) | 61 / 81 % | 28 / 78 % | -| M2, station left out (3 seeds) | 58 / 56 % | 26 / 19 % | +| rules, last 30 % of every station | 58 / 78 % | 33 / 41 % | +| M1, last 30 % (`--balance`) | 58 / 85 % | 33 / 75 % | +| M2, last 30 % | 58 / 85 % | 28 / 79 % | +| **M2 pre-trained, last 30 %** | **58 / 84 %** | **56 / 74 %** | +| rules, all stations | 57 / 72 % | 40 / 34 % | +| M2, station left out | 57 / 56 % | 13 / 14 % | +| M2 pre-trained, station left out | 57 / 60 % | 9 / 17 % | -The information is in the features: with every station seen, M1 beats the rules; moving to a new station is what fails (Obstruction is labelled on three stations only). Against the criterion, M1 passes the totals but not the regressions: on the last 30 %, Obstruction found at BREITHORN (90 → 57 %, 21 instants), LENZSPITZE (15 → 9 %, 33) and SNCF01 (100 → 56 %, 9 instants, where the rules flag 272 for 9 right); Visibility at LENZSPITZE (found 64 → 58 %, correct 98 → 81 %), SNCF01 and WILDSTRUBEL (correct 96 → 89 % and 97 → 93 %, with found 10 → 68 % and 86 → 99 %). +**Against the criterion** (stations seen, the last 30 %), the pre-trained M2: +- Obstruction passes the totals by far: found +23 points, correct +33. The pre-training is what does it (28 → 56 % found). +- Visibility does not: found +0, correct +6. No model, no threshold passes both: lowering M1's threshold to 0.3 gives 63 / 80 % (+5, +2), to 0.2, 67 / 77 %. With these inputs, Visibility is at a ceiling. +- Six regressions: NORDEND Obstruction correct 99 → 81 % (245 instants, flagging twice as much: found 33 → 52 %); NORDEND Visibility found 51 → 43 % (94); JUNGFRAU Visibility found 81 → 69 % (26); LENZSPITZE Visibility correct 98 → 79 % (28); WILDSTRUBEL Visibility found 94 → 89 % (21) and correct 97 → 90 % (28). -```bash -.venv/bin/python m2.py --cache cache -.venv/bin/python m2.py --cache cache --leave-out --seeds 3 --out runs/m2 && .venv/bin/python evaluate.py score runs/m2 --cache cache -``` +**A new station** stays under the rules, with or without pre-training (Obstruction is labelled on three stations; leaving NORDEND out removes half of its labels): as decided, a new station keeps the rules until it has a few weeks of reviewed labels. -`m2.py` (step 4, PyTorch, CPU): multiple-instance learning, one MLP shared by every sensor and an attention per family over the sensors; trained on the reviewed labels only (no pre-training: step 5). One run alone varies by 10 to 20 points with the seed, hence the average of several (`--seeds`). Last 30 %: ~15 min; station left out: ~1.5 h. +**Episodes** (pre-trained M2, last 30 %, `decode.py`): the model finds more labels than the rules (Visibility 72/86 against 58/86, Obstruction 77/90 against 65/90), but its Visibility episodes are cut up: 64 % correct against 85 %, bounds 0.55 against 0.68 (shared instants over the instants of either). Obstruction: 79 % correct against 77 %, bounds 0.33 against 0.17. Bridging two instants instead of one gives 68 % correct for Visibility. -M2 is not better than M1. On the last 30 %, more Visibility found but Obstruction found falls back (39 → 28 %), and the same regressions as M1 (BREITHORN, LENZSPITZE, SNCF01 Obstruction; LENZSPITZE Visibility correct 98 → 60 %; SNCF01, WILDSTRUBEL). Leaving a station out, the attention does not carry over: Visibility correct falls to 56 % and Obstruction to 19 %, 18 regressions over 9 stations (Visibility correct 100 → 5 % at CLARIDEN, 88 → 17 % at NIEDERHORN; Obstruction found 91 → 5 % at SNCF01, 100 → 0 % at JUNGFRAU). With 11 stations, 3 of them labelled for Obstruction, the reviewed labels are too few for a new station; the station's own behaviour, seen in its first weeks, is what the models use. +**Sensors**: for the labels naming 1 to 3 sensors, a named sensor among the first three of the attention: Visibility 13/16, Obstruction 28/32; first: 9/16 and 11/32. A plain ranking by error_value against its usual level (`ev_z`) does as well for Obstruction (27/32 among three, 17/32 first) and worse for Visibility (10/16, 3/16). The attention does not add much to the plain ranking. + +**Tried and dropped**: the rules' flags as inputs of M1 (`--with-rules`): found falls (Visibility 56 %, Obstruction 30 %). The weather (MeteoSwiss, `~/aimsight-data/weather/`): 4 of the 11 labelled stations have a position in `extraction/stations.txt`, none of BREITHORN, CLARIDEN, FLORIDA, SNCF01, WILDSTRUBEL. + +**What comes next**, by expected gain: +1. Obstruction: the pre-trained M2 is a candidate beside the rules, on the stations already labelled; NORDEND's 245 instants to read with the analyst first (wrongly flagged, or Obstruction the labels missed). +2. Visibility needs new information rather than another model: the weather (the positions of the stations in `stations.txt`), the image quality metrics on the labelled stations. +3. Labels where the model hesitates or disagrees with the rules (active learning through the analyst and the review in the app), on more stations for Obstruction. ## Contracts -- Reads the measurements through `detectors/scripts/data.py`, the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments. +- Reads the measurements through `scripts/data.py` of `aimsight-tools` (on the path through `detectors/scripts/categories.py`), the reading of the benchmark; the labels from `labeling/auto/`; the data from `data/` (read-only). Paths are passed as arguments. - Never changes `aimsight-detectors` (neither `scripts/` nor `bench/`): the benchmark is the measure. - Episodes in the format of the detectors' contract (`start`, `end`, `kind`, `label`, `text`, `ignore`, `sensors`), so that a model can later sit behind `detect(measurements)`. diff --git a/decode.py b/decode.py new file mode 100644 index 0000000..ef8a8b5 --- /dev/null +++ b/decode.py @@ -0,0 +1,132 @@ +"""Step 6: predictions per instant -> episodes, scored per episode beside the rules decoded the same way, and the +sensors the attention points at, against the sensors the labels name. + +Episodes: the flagged instants of a family, consecutive on the station's grid, a gap of one instant bridged. +Per family: labels found (a label holding an instant of the range, overlapped by an episode of its family), episodes +correct (an episode overlapping a label of its family), and the bounds: for each label found, the instants shared by +the label and the episodes touching it, over the instants of either (1 = the same bounds). +Sensors: for the labels naming 1 to 3 of the station's sensors, is a named sensor the first (top 1) or among the first +three (top 3) by the attention of the family averaged over the label's instants; beside a plain baseline, the sensors +ranked by their error_value against their usual level (feature ev_z). +--out: the model's episodes, one .jsonl, in the detectors' format. +Usage: python decode.py --cache --labels [--out ] +""" +import argparse +import json +import re +import warnings +from pathlib import Path + +import numpy as np + +from dataset import FAMILIES + +GAP = 1 # instants without a flag bridged inside an episode + + +def episodes(flags): + """flags [T] -> [[i, j]], the index ranges of the episodes, bounds included.""" + out = [] + for i in np.flatnonzero(flags): + if out and i - out[-1][1] <= GAP + 1: + out[-1][1] = i + else: + out.append([i, i]) + return out + + +def named(description, sensors): + return [k for k, s in enumerate(sensors) if re.search(rf"(? [labels, labels found, episodes, episodes correct, [bounds of each label found]] for family k.""" + eps = episodes(yhat[:, k]) + spans = [np.flatnonzero((instants >= e["start"]) & (instants <= e["end"])) for e in labels] + spans = [s for s in spans if len(s)] + flagged = np.zeros(len(instants), bool) + for i, j in eps: + flagged[i:j + 1] = True + found, bounds = 0, [] + for s in spans: + touching = [(i, j) for i, j in eps if i <= s[-1] and j >= s[0]] + if touching: + found += 1 + mine = np.zeros(len(instants), bool) + for i, j in touching: + mine[i:j + 1] = True + lab = np.zeros(len(instants), bool) + lab[s] = True + bounds.append((mine & lab).sum() / (mine | lab).sum()) + correct = sum(any(i <= s[-1] and j >= s[0] for s in spans) for i, j in eps) + return [len(spans), found, len(eps), correct, bounds] + + +def main(): + p = argparse.ArgumentParser(usage=__doc__) + p.add_argument("predictions") + p.add_argument("--cache", required=True) + p.add_argument("--labels", required=True) + p.add_argument("--out") + a = p.parse_args() + ep = {} # (family, "rules"|"model") -> summed score + sens = {} # family -> [labels naming sensors, top1 attention, top3 attention, top1 ev_z, top3 ev_z] + for q in sorted(Path(a.predictions).glob("*.npz")): + pr, d = np.load(q), np.load(Path(a.cache) / q.name) + cut = int(pr["cut"]) if "cut" in pr else 0 + instants, sensors = d["instants"][cut:], list(d["sensors"]) + ev_z = d["xs"][cut:, :, list(d["xs_names"]).index("ev_z")] + rules = np.load(Path(a.cache) / "rules" / q.name)["yhat"][cut:] + labels = [json.loads(x) for x in open(Path(a.labels) / f"{q.stem}.jsonl")] + labels = [e for e in labels if e["end"] >= instants[0] and e["start"] <= instants[-1]] + out = [] + for j, f in enumerate(pr["families"].tolist()): + k, mine = FAMILIES.index(f), [e for e in labels if e["family"] == f] + for who, yhat in (("rules", rules), ("model", pr["yhat"])): + c = score_station(mine, instants, yhat, k) + t = ep.setdefault((f, who), [0, 0, 0, 0, []]) + ep[(f, who)] = [x + y for x, y in zip(t, c)] + s = sens.setdefault(f, [0, 0, 0, 0, 0]) + for e in mine: + n, m = named(e["description"], sensors), (instants >= e["start"]) & (instants <= e["end"]) + if not 1 <= len(n) <= 3 or not m.any(): + continue + with warnings.catch_warnings(): + warnings.simplefilter("ignore", RuntimeWarning) + att = pr["attention"][m, :, j].astype(np.float32).mean(0) + z = np.nan_to_num(np.nanmean(ev_z[m], 0), nan=-np.inf) + ra, rz = np.argsort(-att), np.argsort(-z) + s[0] += 1 + s[1] += ra[0] in n + s[2] += bool(set(ra[:3]) & set(n)) + s[3] += rz[0] in n + s[4] += bool(set(rz[:3]) & set(n)) + for i, i2 in episodes(pr["yhat"][:, k]): + att = pr["attention"][i:i2 + 1, :, j].astype(np.float32).mean(0) + top = [sensors[x] for x in np.argsort(-att)[:3] if att[x] >= 2 / len(sensors)] + out.append({"start": str(instants[i]), "end": str(instants[i2]), "kind": f, "label": f, + "text": f"{f}: model probability up to {pr['proba'][i:i2 + 1, j].max():.2f}", + "ignore": False, "sensors": top}) + if a.out: + Path(a.out).mkdir(parents=True, exist_ok=True) + with open(Path(a.out) / f"{q.stem}.jsonl", "w") as w: + for v in sorted(out, key=lambda v: v["start"]): + w.write(json.dumps(v) + "\n") + + def r(a_, b): + return f"{a_}/{b} ({round(100 * a_ / b)} %)" if b else "—" + print("Episodes") + print(f" {'':<14}{'labels found: rules':>22}{'model':>18}{'episodes correct: rules':>26}{'model':>18}" + f"{'bounds: rules':>15}{'model':>8}") + for f in sorted({f for f, _ in ep}, key=FAMILIES.index): + (n, fr, er, cr, br), (_, fm, em, cm, bm) = ep[(f, "rules")], ep[(f, "model")] + md = lambda x: f"{np.median(x):.2f}" if x else "—" # noqa: E731 + print(f" {f:<14}{r(fr, n):>22}{r(fm, n):>18}{r(cr, er):>26}{r(cm, em):>18}{md(br):>15}{md(bm):>8}") + print("Sensors (labels naming 1 to 3 sensors: a named sensor first, or among the first three)") + print(f" {'':<14}{'labels':>8}{'top 1: attention':>20}{'ev_z':>14}{'top 3: attention':>20}{'ev_z':>14}") + for f, (n, a1, a3, z1, z3) in sens.items(): + print(f" {f:<14}{n:>8}{r(a1, n):>20}{r(z1, n):>14}{r(a3, n):>20}{r(z3, n):>14}") + + +if __name__ == "__main__": + main() diff --git a/m2.py b/m2.py index e1caa35..92cd82a 100644 --- a/m2.py +++ b/m2.py @@ -8,10 +8,11 @@ Training: binary cross-entropy per family on the reviewed labels, each station d --pretrain : first the six families the rules flag, on every station and month, labelled or not; then the same fine-tuning on the reviewed labels. Never on what is tested: time split, the instants from each labelled station's cut on; station left out, the whole station. - default: the criterion, trained on the first 70 % of every station, scored on the last 30 % (evaluate.CUT); + default: the criterion, trained on the first 70 % of every station, scored on the last 30 % (evaluate.CUT); with + --out, the predictions of the last 30 % saved as below, with cut (the index of the first instant tested); --leave-out: the indicator, each station predicted by the model trained on the others; one .npz per station in --out, with yhat [instant, family] (the rules outside the pilot families), proba and attention [instant, sensor, family]. -Usage: python m2.py --cache [--pretrain ] [--leave-out --out ] [--families Visibility,Obstruction] +Usage: python m2.py --cache [--pretrain ] [--leave-out [--test A,B] --out ] [--families V,O] [--epochs 200 --seeds 5] """ import argparse @@ -24,6 +25,8 @@ from torch import nn from dataset import FAMILIES from evaluate import CUT, report_time_split +torch.set_num_threads(1) # a small model: one thread per process is as fast; share the work between processes instead + BATCHES, BATCH = 8, 128 # per station and per epoch: every station weighs the same @@ -111,8 +114,11 @@ def main(): p.add_argument("--threshold", type=float, default=0.5) p.add_argument("--pretrain", help="the folder of pretrain.py: pre-training on the rules' outputs first") p.add_argument("--leave-out", action="store_true") + p.add_argument("--test", help="--leave-out: only these stations (several processes share the work)") p.add_argument("--out") a = p.parse_args() + if a.leave_out and not a.out: + p.error("--leave-out needs --out") families = a.families.split(",") k_index = torch.tensor([FAMILIES.index(f) for f in families]) data = {q.stem: np.load(q) for q in sorted(Path(a.cache).glob("*.npz"))} @@ -128,25 +134,31 @@ def main(): def pre(keep): return pretraining_parts(a.pretrain, keep) if a.pretrain else {} + out = Path(a.out or ".") + + def save(station, proba, attention, cut=0): + out.mkdir(parents=True, exist_ok=True) + yhat = np.load(Path(a.cache) / "rules" / f"{station}.npz")["yhat"][cut:].copy() + for j, f in enumerate(families): + yhat[:, FAMILIES.index(f)] = proba[:, j] >= a.threshold + np.savez_compressed(out / f"{station}.npz", yhat=yhat, proba=proba, attention=attention.astype(np.float16), + families=np.array(families), cut=cut) + print(station, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True) + if not a.leave_out: cut = {s: int(CUT * len(d["y"])) for s, d in data.items()} first_tested = {s: data[s]["instants"][cut[s]] for s in data} models = ensemble([tuple(t[:cut[s]] for t in tensors[s]) for s in data], pre(lambda s, inst: inst < first_tested[s] if s in first_tested else np.ones(len(inst), bool))) - proba = {s: averaged(models, tensors[s][0][cut[s]:], tensors[s][1][cut[s]:])[0] for s in data} - report_time_split(a.cache, data, proba, families, a.threshold) + proba = {s: averaged(models, tensors[s][0][cut[s]:], tensors[s][1][cut[s]:]) for s in data} + report_time_split(a.cache, data, {s: p[0] for s, p in proba.items()}, families, a.threshold) + if a.out: + for s, (pr, att) in proba.items(): + save(s, pr, att, cut[s]) return - out = Path(a.out) - out.mkdir(parents=True, exist_ok=True) - for test in data: + for test in a.test.split(",") if a.test else data: models = ensemble([tensors[s] for s in data if s != test], pre(lambda s, inst: np.full(len(inst), s != test))) - proba, attention = averaged(models, *tensors[test][:2]) - yhat = np.load(Path(a.cache) / "rules" / f"{test}.npz")["yhat"].copy() - for j, f in enumerate(families): - yhat[:, FAMILIES.index(f)] = proba[:, j] >= a.threshold - np.savez_compressed(out / f"{test}.npz", yhat=yhat, proba=proba, attention=attention.astype(np.float16), - families=np.array(families)) - print(test, {f: int(yhat[:, FAMILIES.index(f)].sum()) for f in families}, flush=True) + save(test, *averaged(models, *tensors[test][:2])) if __name__ == "__main__": diff --git a/test_decode.py b/test_decode.py new file mode 100644 index 0000000..7ae1ac3 --- /dev/null +++ b/test_decode.py @@ -0,0 +1,13 @@ +"""Checks of decode.py: python test_decode.py.""" +import numpy as np + +import decode + +assert decode.episodes(np.array([0, 1, 1, 0, 1, 0, 0, 1])) == [[1, 4], [7, 7]] # one instant bridged, two not +assert decode.named("PB1_2 alone, unlike PB1 and PB2_2.", ["PB1", "PB1_2", "PB2", "PR12"]) == [0, 1] +instants = np.array(["t0", "t1", "t2", "t3", "t4", "t5"]) +yhat = np.zeros((6, 1), np.uint8) +yhat[1:3, 0] = 1 +c = decode.score_station([{"start": "t2", "end": "t3"}, {"start": "t5", "end": "t5"}], instants, yhat, 0) +assert c[:4] == [2, 1, 1, 1] and c[4] == [1 / 3], c # label t2..t3 and episode t1..t2 share t2 of t1..t3 +print("ok")