67 lines
3.3 KiB
Python
67 lines
3.3 KiB
Python
"""Requalification: for each label, the largest deviation of a target in its own noises.
|
|||
|
|
|
||
|
|
Short labels: the target ranges of `ranges.py --k 5 --rolling 8` (file argv[2]) overlapping the label
|
||
|
|
(references CTR02, CTR03, CTR04 excluded: they are not targets).
|
||
|
|
Thermal labels: per target, hourly median of corr minus the night level (median of the 3 h before the label start),
|
||
|
|
in the target's own noise (the noise column of the same ranges.py output); the largest |deviation|.
|
||
|
|
Usage: python3 requalify.py labels.jsonl ranges_roll8.txt
|
||
|
|
"""
|
||
|
|
import sys, json, re, statistics
|
||
|
|
from datetime import timedelta
|
||
|
|
sys.path.insert(0, "tools/scripts")
|
||
|
|
import data
|
||
|
|
|
||
|
|
labs = [json.loads(l) for l in open(sys.argv[1])]
|
||
|
|
REFS = {"CTR02", "CTR03", "CTR04"}
|
||
|
|
noise, ranges = {}, []
|
||
|
|
col = None
|
||
|
|
for line in open(sys.argv[2]):
|
||
|
|
m = re.match(r"== (corr_[xy]):", line) or re.match(r"(corr_[xy]): ranges", line)
|
||
|
|
if m:
|
||
|
|
col = m.group(1); mode = "noise" if line.startswith("==") else "ranges"; continue
|
||
|
|
f = line.split()
|
||
|
|
if not f or col is None or f[0] in ("sensor",):
|
||
|
|
continue
|
||
|
|
if mode == "noise" and len(f) >= 4:
|
||
|
|
try: noise[(col, f[0])] = float(f[3])
|
||
|
|
except ValueError: pass
|
||
|
|
if mode == "ranges" and len(f) >= 7 and f[1].startswith("20"):
|
||
|
|
ranges.append((col, f[0], data.parse_time(f[1]), data.parse_time(f[2]), float(f[5]), f[6]))
|
||
|
|
|
||
|
|
by = data.load("data/NORDEND")
|
||
|
|
for ms in by.values():
|
||
|
|
for m in ms: m["t"] = m["t"].replace(second=0, microsecond=0)
|
||
|
|
tg = [s for s in by if s not in REFS]
|
||
|
|
# thermal check: instants inside a non-thermal label (bounds excluded) are left out, they carry another event
|
||
|
|
OTHER = [(data.parse_time(l["start"]), data.parse_time(l["end"])) for l in labs if l["family"] not in ("Thermal displacement", "No data")]
|
||
|
|
def excl(t):
|
||
|
|
return any(a < t < b for a, b in OTHER)
|
||
|
|
|
||
|
|
for l in labs:
|
||
|
|
S, E = data.parse_time(l["start"]), data.parse_time(l["end"])
|
||
|
|
if l["family"] == "No data":
|
||
|
|
print(l.get("id"), l["start"], l["end"], "No data"); continue
|
||
|
|
if l["family"] == "Thermal displacement":
|
||
|
|
res = []
|
||
|
|
for c in ("corr_x", "corr_y"):
|
||
|
|
for s in tg:
|
||
|
|
if s == "PR04": continue # Target issue NORDEND-24
|
||
|
|
night = [m[c] for m in by[s] if S - timedelta(hours=3) <= m["t"] <= S and m[c] is not None and not excl(m["t"])]
|
||
|
|
if len(night) < 4: continue
|
||
|
|
lv = statistics.median(night)
|
||
|
|
hours = {}
|
||
|
|
for m in by[s]:
|
||
|
|
if S <= m["t"] <= E and m[c] is not None and not excl(m["t"]):
|
||
|
|
hours.setdefault(m["t"].replace(minute=0), []).append(m[c])
|
||
|
|
for h, v in hours.items():
|
||
|
|
if len(v) >= 3:
|
||
|
|
res.append(((statistics.median(v) - lv) / noise[(c, s)], s, c, h))
|
||
|
|
res.sort(key=lambda x: -abs(x[0]))
|
||
|
|
top = " ".join(f"{s}.{c[-1]} {d:+.1f}@{h:%d %H}h" for d, s, c, h in res[:5])
|
||
|
|
print(l.get("id"), l["start"], l["end"], "THERMAL max", f"{abs(res[0][0]):.1f}" if res else "-", "|", top)
|
||
|
|
continue
|
||
|
|
hit = [r for r in ranges if r[1] not in REFS and r[2] <= E and r[3] >= S]
|
||
|
|
hit.sort(key=lambda r: -abs(r[4]))
|
||
|
|
top = " ".join(f"{r[1]}.{r[0][-1]} {r[4]:+.1f}@{r[5][11:16]}" for r in hit[:6])
|
||
|
|
mx = abs(hit[0][4]) if hit else 0
|
||
|
|
print(l.get("id"), l["start"], l["end"], l["family"], "max", f"{mx:.1f}", "|", top or "nothing beyond 5")
|