auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 15:13:44 +01:00
1 parent df979301b5
commit 2a5d87659b
279 files changed
+10758 -710

No files matched your search

@@ -0,0 +1,47 @@
"""Per instant: targets whose corr deviates from a local rolling median (+-W measurements, self excluded) by > K noises,
and rejected sensors; marks the existing non-thermal labels covering each instant.
usage: scan.py LABELS START END [--all] [--k 5] [--from T --to T] (print all instants in a sub-window)"""
import sys, json, statistics, argparse
sys.path.insert(0, "tools/scripts")
import data
ap = argparse.ArgumentParser(); ap.add_argument("labels"); ap.add_argument("start"); ap.add_argument("end")
ap.add_argument("--k", type=float, default=5); ap.add_argument("--w", type=int, default=4)
ap.add_argument("--frm"); ap.add_argument("--to"); ap.add_argument("--skip", default="PR04")
a = ap.parse_args()
S, E = data.parse_time(a.start), data.parse_time(a.end)
from datetime import timedelta
by = data.load("data/NORDEND")
for ms in by.values():
for m in ms: m["t"] = m["t"].replace(second=0, microsecond=0)
labs = [json.loads(l) for l in open(a.labels)]
for l in labs: l["s"], l["e"] = data.parse_time(l["start"]), data.parse_time(l["end"])
skip = set(a.skip.split(","))
win = {s: data.filter_window(m, S - timedelta(hours=3), E + timedelta(hours=3)) for s, m in by.items()}
times = sorted({m["t"] for ms in win.values() for m in ms})
flags = {t: [] for t in times}; rej = {t: [] for t in times}
for s, ms in win.items():
ref = data.is_reference(by[s])
for m in ms:
if data.rejected(m) and s not in skip: rej[m["t"]].append(s)
if s in skip: continue
for c in ("corr_x", "corr_y"):
pts = [(m["t"], m[c]) for m in ms if m[c] is not None]
v = [p[1] for p in pts]
if len(v) < 20: continue
noise = statistics.median(abs(v[i+1]-v[i]) for i in range(len(v)-1)) or 1e-9
for i, (t, x) in enumerate(pts):
nb = v[max(0, i-a.w):i] + v[i+1:i+1+a.w]
d = (x - statistics.median(nb)) / noise
if abs(d) > a.k: flags[t].append(f"{s}.{c[-1]}{d:+.0f}")
def cover(t):
return ",".join(l.get("id", "?").replace("NORDEND-", "") for l in labs if l["s"] <= t <= l["e"] and l["family"] != "Thermal displacement" and not (l["family"] == "Target issue"))
F = data.parse_time(a.frm) if a.frm else None; T = data.parse_time(a.to) if a.to else None
prev = None
for t in times:
if t < S or t > E:
if not (F and F <= t <= T): continue
if prev and t - prev > timedelta(minutes=15): print(f" -- gap {data.iso(prev)} -> {data.iso(t)}")
prev = t
show = (F and F <= t <= T) or len(flags[t]) >= 2 or rej[t] or any(abs(int(f.split(".")[1][1:])) >= 9 for f in flags[t])
if show:
print(data.iso(t), f"[{cover(t)}]", len(flags[t]), " ".join(flags[t]), "| rej:", ",".join(rej[t]))