Files
aimsight-labeling/auto/analyses/NORDEND_2026-08-10T0115_2026-08-23T0515/scripts/requalify.py
T

94 lines
3.7 KiB
Python
Raw Normal View History

"""Requalification: per label, the largest deviation in noises of a target over the label's range.
Reads the ranges.py outputs (k 5, plain and rolling 1d) and, for the thermal labels,
computes each target's deviation from its end-of-night level (median of the 3 h before
the label start), hourly median, in its own noise (the noise of ranges.py over the batch).
Standard library only.
"""
import csv, glob, json, re, statistics
from collections import defaultdict
from datetime import datetime, timedelta
D = "labeling/auto/analyses/NORDEND_2026-08-10T0115_2026-08-23T0515"
REFS = {"CTR02", "CTR03", "CTR04"}
T = lambda s: datetime.strptime(s[:16].replace(" ", "T"), "%Y-%m-%dT%H:%M")
def parse(path):
noise, ranges, col, section = {}, [], None, None
for line in open(path):
m = re.match(r"== (corr_[xy]):", line)
if m:
col, section = m.group(1), "noise"; continue
m = re.match(r"(corr_[xy]): ranges", line)
if m:
col, section = m.group(1), "ranges"; continue
p = line.split()
if not p or not re.match(r"(PR|CTR)\d+$", p[0]):
continue
if section == "noise":
noise[(col, p[0])] = float(p[3])
elif section == "ranges" and len(p) >= 8:
ranges.append(dict(col=col, s=p[0], a=T(p[1]), b=T(p[2]), n=int(p[3]), dev=float(p[5]), at=p[6]))
return noise, ranges
noise, plain = parse(f"{D}/work/rq/ranges_k5.txt")
_, roll = parse(f"{D}/work/rq/ranges_k5_r1d.txt")
data = defaultdict(list) # (sensor, col) -> [(t, v)] accepted only
for f in sorted(glob.glob("data/NORDEND/displacement/NORDEND_advanced_202608*.csv")):
for r in csv.DictReader(open(f)):
try:
if float(r["error_value"]) > float(r["error_threshold"]) or r["corr_x"] == "":
continue
except ValueError:
continue
t = T(r["time_utc"])
for c in ("corr_x", "corr_y"):
data[(r["sensor_name"], c)].append((t, float(r[c])))
def top(rs, a, b, k=4):
best = {}
for r in rs:
if r["b"] >= a and r["a"] <= b:
key = (r["s"], r["col"])
if key not in best or abs(r["dev"]) > abs(best[key]["dev"]):
best[key] = r
tg = sorted((r for r in best.values() if r["s"] not in REFS), key=lambda r: -abs(r["dev"]))
txt = ", ".join(f"{r['s']} {r['col'][-1]} {r['dev']:+.1f} ({r['n']}m @{r['at'][5:16]})" for r in tg[:k]) or "-"
return txt, (abs(tg[0]["dev"]) if tg else 0.0)
def night_dev(a, b):
res = []
for (s, c), pts in data.items():
if s in REFS or (c, s) not in noise:
continue
pre = [v for t, v in pts if a - timedelta(hours=3) <= t < a]
if len(pre) < 3:
continue
lvl = statistics.median(pre)
hours = defaultdict(list)
for t, v in pts:
if a <= t <= b:
hours[t.replace(minute=0)].append(v - lvl)
if not hours:
continue
h, d = max(((h, statistics.median(v)) for h, v in hours.items()), key=lambda x: abs(x[1]))
res.append((abs(d) / noise[(c, s)], s, c[-1], d / noise[(c, s)], h))
res.sort(reverse=True)
return ", ".join(f"{s} {c} {d:+.1f} @{h:%d %H:%M}" for _, s, c, d, h in res[:4]), (res[0][0] if res else 0)
for line in open(f"{D}/work/requalify_input.jsonl"):
L = json.loads(line)
a, b = T(L["start"]), T(L["end"])
tp, mp = top(plain, a, b)
tr, mr = top(roll, a, b)
print(f"{L['id']:12s} {L['start'][5:16]} → {L['end'][5:16]} {L['family']:22s} plain max {mp:5.1f} | rolling max {mr:5.1f}")
print(f" plain: {tp}\n roll : {tr}")
if L["family"] == "Thermal displacement":
tn, mn = night_dev(a, b)
print(f" night-level hourly: max {mn:.1f}: {tn}")