Files
aimsight-labeling/auto/analyses/NORDEND_2026-08-10T0115_2026-08-23T0515/scripts/requalify.py
T
maximeandClaude Opus 5.5 ddb17c2189 auto: labels requalified under the 5-noise rules (only what stands out from the noise): 32 analyses refined, the 6 paused periods verified, the long labels requalified with the fixes the period analysts found; 1153 -> 961 labels
- 32 analyses: each label kept, rewritten or removed by its analyst (section Requalification of the report; mm magnitudes where the targets have them)
- SNCF01 x3, WILDSTRUBEL 2026-09-01 -> 2026-10-06 x3: verified from their existing labels under the new rules
- long labels (no analysis): auto/analyses/long-labels/<STATION>/; NORDEND-169 and -195 run to 07/11, so NORDEND-283 and -284 (their continuation) are removed
- reading-rules.md, stations.md: the citations of removed or rewritten labels point to their successor, or to [expert]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 23:58:00 +01:00

95 lines
3.7 KiB
Python

"""Requalification: per label, the largest deviation in noises of a target over the label's range.
Reads the ranges.py outputs (k 5, plain and rolling 1d) and, for the thermal labels,
computes each target's deviation from its end-of-night level (median of the 3 h before
the label start), hourly median, in its own noise (the noise of ranges.py over the batch).
Standard library only.
"""
import csv, glob, json, re, statistics
from collections import defaultdict
from datetime import datetime, timedelta
D = "labeling/auto/analyses/NORDEND_2026-08-10T0115_2026-08-23T0515"
REFS = {"CTR02", "CTR03", "CTR04"}
T = lambda s: datetime.strptime(s[:16].replace(" ", "T"), "%Y-%m-%dT%H:%M")
def parse(path):
noise, ranges, col, section = {}, [], None, None
for line in open(path):
m = re.match(r"== (corr_[xy]):", line)
if m:
col, section = m.group(1), "noise"; continue
m = re.match(r"(corr_[xy]): ranges", line)
if m:
col, section = m.group(1), "ranges"; continue
p = line.split()
if not p or not re.match(r"(PR|CTR)\d+$", p[0]):
continue
if section == "noise":
noise[(col, p[0])] = float(p[3])
elif section == "ranges" and len(p) >= 8:
ranges.append(dict(col=col, s=p[0], a=T(p[1]), b=T(p[2]), n=int(p[3]), dev=float(p[5]), at=p[6]))
return noise, ranges
noise, plain = parse(f"{D}/work/rq/ranges_k5.txt")
_, roll = parse(f"{D}/work/rq/ranges_k5_r1d.txt")
data = defaultdict(list) # (sensor, col) -> [(t, v)] accepted only
for f in sorted(glob.glob("data/NORDEND/displacement/NORDEND_advanced_202608*.csv")):
for r in csv.DictReader(open(f)):
try:
if float(r["error_value"]) > float(r["error_threshold"]) or r["corr_x"] == "":
continue
except ValueError:
continue
t = T(r["time_utc"])
for c in ("corr_x", "corr_y"):
data[(r["sensor_name"], c)].append((t, float(r[c])))
def top(rs, a, b, k=4):
best = {}
for r in rs:
if r["b"] >= a and r["a"] <= b:
key = (r["s"], r["col"])
if key not in best or abs(r["dev"]) > abs(best[key]["dev"]):
best[key] = r
tg = sorted((r for r in best.values() if r["s"] not in REFS), key=lambda r: -abs(r["dev"]))
txt = ", ".join(f"{r['s']} {r['col'][-1]} {r['dev']:+.1f} ({r['n']}m @{r['at'][5:16]})" for r in tg[:k]) or "-"
return txt, (abs(tg[0]["dev"]) if tg else 0.0)
def night_dev(a, b):
res = []
for (s, c), pts in data.items():
if s in REFS or (c, s) not in noise:
continue
pre = [v for t, v in pts if a - timedelta(hours=3) <= t < a]
if len(pre) < 3:
continue
lvl = statistics.median(pre)
hours = defaultdict(list)
for t, v in pts:
if a <= t <= b:
hours[t.replace(minute=0)].append(v - lvl)
if not hours:
continue
h, d = max(((h, statistics.median(v)) for h, v in hours.items()), key=lambda x: abs(x[1]))
res.append((abs(d) / noise[(c, s)], s, c[-1], d / noise[(c, s)], h))
res.sort(reverse=True)
return ", ".join(f"{s} {c} {d:+.1f} @{h:%d %H:%M}" for _, s, c, d, h in res[:4]), (res[0][0] if res else 0)
for line in open(f"{D}/work/requalify_input.jsonl"):
L = json.loads(line)
a, b = T(L["start"]), T(L["end"])
tp, mp = top(plain, a, b)
tr, mr = top(roll, a, b)
print(f"{L['id']:12s} {L['start'][5:16]} → {L['end'][5:16]} {L['family']:22s} plain max {mp:5.1f} | rolling max {mr:5.1f}")
print(f" plain: {tp}\n roll : {tr}")
if L["family"] == "Thermal displacement":
tn, mn = night_dev(a, b)
print(f" night-level hourly: max {mn:.1f}: {tn}")