auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 15:13:44 +01:00
1 parent df979301b5
commit 2a5d87659b
279 files changed
+10758 -710

No files matched your search

@@ -0,0 +1,47 @@
"""Per instant: targets whose corr deviates from a local rolling median (+-W measurements, self excluded) by > K noises,
and rejected sensors; marks the existing non-thermal labels covering each instant.
usage: scan.py LABELS START END [--all] [--k 5] [--from T --to T] (print all instants in a sub-window)"""
import sys, json, statistics, argparse
sys.path.insert(0, "tools/scripts")
import data
ap = argparse.ArgumentParser(); ap.add_argument("labels"); ap.add_argument("start"); ap.add_argument("end")
ap.add_argument("--k", type=float, default=5); ap.add_argument("--w", type=int, default=4)
ap.add_argument("--frm"); ap.add_argument("--to"); ap.add_argument("--skip", default="PR04")
a = ap.parse_args()
S, E = data.parse_time(a.start), data.parse_time(a.end)
from datetime import timedelta
by = data.load("data/NORDEND")
for ms in by.values():
for m in ms: m["t"] = m["t"].replace(second=0, microsecond=0)
labs = [json.loads(l) for l in open(a.labels)]
for l in labs: l["s"], l["e"] = data.parse_time(l["start"]), data.parse_time(l["end"])
skip = set(a.skip.split(","))
win = {s: data.filter_window(m, S - timedelta(hours=3), E + timedelta(hours=3)) for s, m in by.items()}
times = sorted({m["t"] for ms in win.values() for m in ms})
flags = {t: [] for t in times}; rej = {t: [] for t in times}
for s, ms in win.items():
ref = data.is_reference(by[s])
for m in ms:
if data.rejected(m) and s not in skip: rej[m["t"]].append(s)
if s in skip: continue
for c in ("corr_x", "corr_y"):
pts = [(m["t"], m[c]) for m in ms if m[c] is not None]
v = [p[1] for p in pts]
if len(v) < 20: continue
noise = statistics.median(abs(v[i+1]-v[i]) for i in range(len(v)-1)) or 1e-9
for i, (t, x) in enumerate(pts):
nb = v[max(0, i-a.w):i] + v[i+1:i+1+a.w]
d = (x - statistics.median(nb)) / noise
if abs(d) > a.k: flags[t].append(f"{s}.{c[-1]}{d:+.0f}")
def cover(t):
return ",".join(l.get("id", "?").replace("NORDEND-", "") for l in labs if l["s"] <= t <= l["e"] and l["family"] != "Thermal displacement" and not (l["family"] == "Target issue"))
F = data.parse_time(a.frm) if a.frm else None; T = data.parse_time(a.to) if a.to else None
prev = None
for t in times:
if t < S or t > E:
if not (F and F <= t <= T): continue
if prev and t - prev > timedelta(minutes=15): print(f" -- gap {data.iso(prev)} -> {data.iso(t)}")
prev = t
show = (F and F <= t <= T) or len(flags[t]) >= 2 or rej[t] or any(abs(int(f.split(".")[1][1:])) >= 9 for f in flags[t])
if show:
print(data.iso(t), f"[{cover(t)}]", len(flags[t]), " ".join(flags[t]), "| rej:", ",".join(rej[t]))
@@ -0,0 +1,33 @@
"""For each thermal label: per target, the hourly-median corr deviation from the night level (01-04 UTC of the label's
end day ... uses the night before the label start: the 3 h before start) at its extreme within the label; prints the
4 largest drops of corr_x and the largest moves of corr_y, and the pose excursions."""
import sys, json, statistics
sys.path.insert(0, "tools/scripts"); import data
from datetime import timedelta
labs = [json.loads(l) for l in open(sys.argv[1]) if '"Thermal displacement"' in l]
by = data.load("data/NORDEND"); cam = data.load_camera("data/NORDEND")
for ms in by.values():
for m in ms: m["t"] = m["t"].replace(second=0, microsecond=0)
tg = [s for s in by if not data.is_reference(by[s]) and s not in ("PR04", "CTR01")]
for l in labs:
S, E = data.parse_time(l["start"]), data.parse_time(l["end"])
out = {}
for c in ("corr_x", "corr_y"):
res = []
for s in tg:
night = [m[c] for m in by[s] if S - timedelta(hours=3) <= m["t"] <= S and m[c] is not None]
if not night: continue
lv = statistics.median(night)
hours = {}
for m in by[s]:
if S <= m["t"] <= E and m[c] is not None: hours.setdefault(m["t"].replace(minute=0), []).append(m[c])
hm = [(h, statistics.median(v) - lv) for h, v in hours.items() if len(v) >= 3]
if not hm: continue
h, d = max(hm, key=lambda x: abs(x[1]))
res.append((d, s, h))
res.sort(key=lambda x: -abs(x[0]))
out[c] = " ".join(f"{s}{d:+.3f}@{h:%d%H}" for d, s, h in res[:6])
pn = [p for p in cam if S <= p["t"] <= E and p["pan"] is not None]
rng = lambda k: max(p[k] for p in pn) - min(p[k] for p in pn)
print(l.get("id"), l["start"], l["end"], f"pan {rng('pan')*1000:.0f} tilt {rng('tilt')*1000:.0f} sc {rng('scale')*1e5:.0f}")
print(" x:", out["corr_x"]); print(" y:", out["corr_y"])