"""Requalification: for each label, the largest deviation of a target in its own noises. Short labels: the target ranges of `ranges.py --k 5 --rolling 8` (file argv[2]) overlapping the label (references CTR02, CTR03, CTR04 excluded: they are not targets). Thermal labels: per target, hourly median of corr minus the night level (median of the 3 h before the label start), in the target's own noise (the noise column of the same ranges.py output); the largest |deviation|. Usage: python3 requalify.py labels.jsonl ranges_roll8.txt """ import sys, json, re, statistics from datetime import timedelta sys.path.insert(0, "tools/scripts") import data labs = [json.loads(l) for l in open(sys.argv[1])] REFS = {"CTR02", "CTR03", "CTR04"} noise, ranges = {}, [] col = None for line in open(sys.argv[2]): m = re.match(r"== (corr_[xy]):", line) or re.match(r"(corr_[xy]): ranges", line) if m: col = m.group(1); mode = "noise" if line.startswith("==") else "ranges"; continue f = line.split() if not f or col is None or f[0] in ("sensor",): continue if mode == "noise" and len(f) >= 4: try: noise[(col, f[0])] = float(f[3]) except ValueError: pass if mode == "ranges" and len(f) >= 7 and f[1].startswith("20"): ranges.append((col, f[0], data.parse_time(f[1]), data.parse_time(f[2]), float(f[5]), f[6])) by = data.load("data/NORDEND") for ms in by.values(): for m in ms: m["t"] = m["t"].replace(second=0, microsecond=0) tg = [s for s in by if s not in REFS] # thermal check: instants inside a non-thermal label (bounds excluded) are left out, they carry another event OTHER = [(data.parse_time(l["start"]), data.parse_time(l["end"])) for l in labs if l["family"] not in ("Thermal displacement", "No data")] def excl(t): return any(a < t < b for a, b in OTHER) for l in labs: S, E = data.parse_time(l["start"]), data.parse_time(l["end"]) if l["family"] == "No data": print(l.get("id"), l["start"], l["end"], "No data"); continue if l["family"] == "Thermal displacement": res = [] for c in ("corr_x", "corr_y"): for s in tg: if s == "PR04": continue # Target issue NORDEND-24 night = [m[c] for m in by[s] if S - timedelta(hours=3) <= m["t"] <= S and m[c] is not None and not excl(m["t"])] if len(night) < 4: continue lv = statistics.median(night) hours = {} for m in by[s]: if S <= m["t"] <= E and m[c] is not None and not excl(m["t"]): hours.setdefault(m["t"].replace(minute=0), []).append(m[c]) for h, v in hours.items(): if len(v) >= 3: res.append(((statistics.median(v) - lv) / noise[(c, s)], s, c, h)) res.sort(key=lambda x: -abs(x[0])) top = " ".join(f"{s}.{c[-1]} {d:+.1f}@{h:%d %H}h" for d, s, c, h in res[:5]) print(l.get("id"), l["start"], l["end"], "THERMAL max", f"{abs(res[0][0]):.1f}" if res else "-", "|", top) continue hit = [r for r in ranges if r[1] not in REFS and r[2] <= E and r[3] >= S] hit.sort(key=lambda r: -abs(r[4])) top = " ".join(f"{r[1]}.{r[0][-1]} {r[4]:+.1f}@{r[5][11:16]}" for r in hit[:6]) mx = abs(hit[0][4]) if hit else 0 print(l.get("id"), l["start"], l["end"], l["family"], "max", f"{mx:.1f}", "|", top or "nothing beyond 5")