- 32 analyses: each label kept, rewritten or removed by its analyst (section Requalification of the report; mm magnitudes where the targets have them) - SNCF01 x3, WILDSTRUBEL 2026-09-01 -> 2026-10-06 x3: verified from their existing labels under the new rules - long labels (no analysis): auto/analyses/long-labels/<STATION>/; NORDEND-169 and -195 run to 07/11, so NORDEND-283 and -284 (their continuation) are removed - reading-rules.md, stations.md: the citations of removed or rewritten labels point to their successor, or to [expert] Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
95 lines
3.7 KiB
Python
95 lines
3.7 KiB
Python
"""Requalification: per label, the largest deviation in noises of a target over the label's range.
|
|
|
|
Reads the ranges.py outputs (k 5, plain and rolling 1d) and, for the thermal labels,
|
|
computes each target's deviation from its end-of-night level (median of the 3 h before
|
|
the label start), hourly median, in its own noise (the noise of ranges.py over the batch).
|
|
Standard library only.
|
|
"""
|
|
import csv, glob, json, re, statistics
|
|
from collections import defaultdict
|
|
from datetime import datetime, timedelta
|
|
|
|
D = "labeling/auto/analyses/NORDEND_2026-08-10T0115_2026-08-23T0515"
|
|
REFS = {"CTR02", "CTR03", "CTR04"}
|
|
T = lambda s: datetime.strptime(s[:16].replace(" ", "T"), "%Y-%m-%dT%H:%M")
|
|
|
|
|
|
def parse(path):
|
|
noise, ranges, col, section = {}, [], None, None
|
|
for line in open(path):
|
|
m = re.match(r"== (corr_[xy]):", line)
|
|
if m:
|
|
col, section = m.group(1), "noise"; continue
|
|
m = re.match(r"(corr_[xy]): ranges", line)
|
|
if m:
|
|
col, section = m.group(1), "ranges"; continue
|
|
p = line.split()
|
|
if not p or not re.match(r"(PR|CTR)\d+$", p[0]):
|
|
continue
|
|
if section == "noise":
|
|
noise[(col, p[0])] = float(p[3])
|
|
elif section == "ranges" and len(p) >= 8:
|
|
ranges.append(dict(col=col, s=p[0], a=T(p[1]), b=T(p[2]), n=int(p[3]), dev=float(p[5]), at=p[6]))
|
|
return noise, ranges
|
|
|
|
|
|
noise, plain = parse(f"{D}/work/rq/ranges_k5.txt")
|
|
_, roll = parse(f"{D}/work/rq/ranges_k5_r1d.txt")
|
|
|
|
data = defaultdict(list) # (sensor, col) -> [(t, v)] accepted only
|
|
for f in sorted(glob.glob("data/NORDEND/displacement/NORDEND_advanced_202608*.csv")):
|
|
for r in csv.DictReader(open(f)):
|
|
try:
|
|
if float(r["error_value"]) > float(r["error_threshold"]) or r["corr_x"] == "":
|
|
continue
|
|
except ValueError:
|
|
continue
|
|
t = T(r["time_utc"])
|
|
for c in ("corr_x", "corr_y"):
|
|
data[(r["sensor_name"], c)].append((t, float(r[c])))
|
|
|
|
|
|
def top(rs, a, b, k=4):
|
|
best = {}
|
|
for r in rs:
|
|
if r["b"] >= a and r["a"] <= b:
|
|
key = (r["s"], r["col"])
|
|
if key not in best or abs(r["dev"]) > abs(best[key]["dev"]):
|
|
best[key] = r
|
|
tg = sorted((r for r in best.values() if r["s"] not in REFS), key=lambda r: -abs(r["dev"]))
|
|
txt = ", ".join(f"{r['s']} {r['col'][-1]} {r['dev']:+.1f} ({r['n']}m @{r['at'][5:16]})" for r in tg[:k]) or "-"
|
|
return txt, (abs(tg[0]["dev"]) if tg else 0.0)
|
|
|
|
|
|
def night_dev(a, b):
|
|
res = []
|
|
for (s, c), pts in data.items():
|
|
if s in REFS or (c, s) not in noise:
|
|
continue
|
|
pre = [v for t, v in pts if a - timedelta(hours=3) <= t < a]
|
|
if len(pre) < 3:
|
|
continue
|
|
lvl = statistics.median(pre)
|
|
hours = defaultdict(list)
|
|
for t, v in pts:
|
|
if a <= t <= b:
|
|
hours[t.replace(minute=0)].append(v - lvl)
|
|
if not hours:
|
|
continue
|
|
h, d = max(((h, statistics.median(v)) for h, v in hours.items()), key=lambda x: abs(x[1]))
|
|
res.append((abs(d) / noise[(c, s)], s, c[-1], d / noise[(c, s)], h))
|
|
res.sort(reverse=True)
|
|
return ", ".join(f"{s} {c} {d:+.1f} @{h:%d %H:%M}" for _, s, c, d, h in res[:4]), (res[0][0] if res else 0)
|
|
|
|
|
|
for line in open(f"{D}/work/requalify_input.jsonl"):
|
|
L = json.loads(line)
|
|
a, b = T(L["start"]), T(L["end"])
|
|
tp, mp = top(plain, a, b)
|
|
tr, mr = top(roll, a, b)
|
|
print(f"{L['id']:12s} {L['start'][5:16]} → {L['end'][5:16]} {L['family']:22s} plain max {mp:5.1f} | rolling max {mr:5.1f}")
|
|
print(f" plain: {tp}\n roll : {tr}")
|
|
if L["family"] == "Thermal displacement":
|
|
tn, mn = night_dev(a, b)
|
|
print(f" night-level hourly: max {mn:.1f}: {tn}")
|