"""Requalification: per label, the largest deviation in noises of a target over the label's range. Reads the ranges.py outputs (k 5, plain and rolling 1d) and, for the thermal labels, computes each target's deviation from its end-of-night level (median of the 3 h before the label start), hourly median, in its own noise (the noise of ranges.py over the batch). Standard library only. """ import csv, glob, json, re, statistics from collections import defaultdict from datetime import datetime, timedelta D = "labeling/auto/analyses/NORDEND_2026-08-10T0115_2026-08-23T0515" REFS = {"CTR02", "CTR03", "CTR04"} T = lambda s: datetime.strptime(s[:16].replace(" ", "T"), "%Y-%m-%dT%H:%M") def parse(path): noise, ranges, col, section = {}, [], None, None for line in open(path): m = re.match(r"== (corr_[xy]):", line) if m: col, section = m.group(1), "noise"; continue m = re.match(r"(corr_[xy]): ranges", line) if m: col, section = m.group(1), "ranges"; continue p = line.split() if not p or not re.match(r"(PR|CTR)\d+$", p[0]): continue if section == "noise": noise[(col, p[0])] = float(p[3]) elif section == "ranges" and len(p) >= 8: ranges.append(dict(col=col, s=p[0], a=T(p[1]), b=T(p[2]), n=int(p[3]), dev=float(p[5]), at=p[6])) return noise, ranges noise, plain = parse(f"{D}/work/rq/ranges_k5.txt") _, roll = parse(f"{D}/work/rq/ranges_k5_r1d.txt") data = defaultdict(list) # (sensor, col) -> [(t, v)] accepted only for f in sorted(glob.glob("data/NORDEND/displacement/NORDEND_advanced_202608*.csv")): for r in csv.DictReader(open(f)): try: if float(r["error_value"]) > float(r["error_threshold"]) or r["corr_x"] == "": continue except ValueError: continue t = T(r["time_utc"]) for c in ("corr_x", "corr_y"): data[(r["sensor_name"], c)].append((t, float(r[c]))) def top(rs, a, b, k=4): best = {} for r in rs: if r["b"] >= a and r["a"] <= b: key = (r["s"], r["col"]) if key not in best or abs(r["dev"]) > abs(best[key]["dev"]): best[key] = r tg = sorted((r for r in best.values() if r["s"] not in REFS), key=lambda r: -abs(r["dev"])) txt = ", ".join(f"{r['s']} {r['col'][-1]} {r['dev']:+.1f} ({r['n']}m @{r['at'][5:16]})" for r in tg[:k]) or "-" return txt, (abs(tg[0]["dev"]) if tg else 0.0) def night_dev(a, b): res = [] for (s, c), pts in data.items(): if s in REFS or (c, s) not in noise: continue pre = [v for t, v in pts if a - timedelta(hours=3) <= t < a] if len(pre) < 3: continue lvl = statistics.median(pre) hours = defaultdict(list) for t, v in pts: if a <= t <= b: hours[t.replace(minute=0)].append(v - lvl) if not hours: continue h, d = max(((h, statistics.median(v)) for h, v in hours.items()), key=lambda x: abs(x[1])) res.append((abs(d) / noise[(c, s)], s, c[-1], d / noise[(c, s)], h)) res.sort(reverse=True) return ", ".join(f"{s} {c} {d:+.1f} @{h:%d %H:%M}" for _, s, c, d, h in res[:4]), (res[0][0] if res else 0) for line in open(f"{D}/work/requalify_input.jsonl"): L = json.loads(line) a, b = T(L["start"]), T(L["end"]) tp, mp = top(plain, a, b) tr, mr = top(roll, a, b) print(f"{L['id']:12s} {L['start'][5:16]} → {L['end'][5:16]} {L['family']:22s} plain max {mp:5.1f} | rolling max {mr:5.1f}") print(f" plain: {tp}\n roll : {tr}") if L["family"] == "Thermal displacement": tn, mn = night_dev(a, b) print(f" night-level hourly: max {mn:.1f}: {tn}")