Files
aimsight-labeling/auto/analyses/NORDEND_2026-08-23T0515_2026-09-04T0615/scripts/requalify.py
T
maximeandClaude Opus 5.5 ddb17c2189 auto: labels requalified under the 5-noise rules (only what stands out from the noise): 32 analyses refined, the 6 paused periods verified, the long labels requalified with the fixes the period analysts found; 1153 -> 961 labels
- 32 analyses: each label kept, rewritten or removed by its analyst (section Requalification of the report; mm magnitudes where the targets have them)
- SNCF01 x3, WILDSTRUBEL 2026-09-01 -> 2026-10-06 x3: verified from their existing labels under the new rules
- long labels (no analysis): auto/analyses/long-labels/<STATION>/; NORDEND-169 and -195 run to 07/11, so NORDEND-283 and -284 (their continuation) are removed
- reading-rules.md, stations.md: the citations of removed or rewritten labels point to their successor, or to [expert]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 23:58:00 +01:00

68 lines
3.3 KiB
Python

"""Requalification: for each label, the largest deviation of a target in its own noises.
Short labels: the target ranges of `ranges.py --k 5 --rolling 8` (file argv[2]) overlapping the label
(references CTR02, CTR03, CTR04 excluded: they are not targets).
Thermal labels: per target, hourly median of corr minus the night level (median of the 3 h before the label start),
in the target's own noise (the noise column of the same ranges.py output); the largest |deviation|.
Usage: python3 requalify.py labels.jsonl ranges_roll8.txt
"""
import sys, json, re, statistics
from datetime import timedelta
sys.path.insert(0, "tools/scripts")
import data
labs = [json.loads(l) for l in open(sys.argv[1])]
REFS = {"CTR02", "CTR03", "CTR04"}
noise, ranges = {}, []
col = None
for line in open(sys.argv[2]):
m = re.match(r"== (corr_[xy]):", line) or re.match(r"(corr_[xy]): ranges", line)
if m:
col = m.group(1); mode = "noise" if line.startswith("==") else "ranges"; continue
f = line.split()
if not f or col is None or f[0] in ("sensor",):
continue
if mode == "noise" and len(f) >= 4:
try: noise[(col, f[0])] = float(f[3])
except ValueError: pass
if mode == "ranges" and len(f) >= 7 and f[1].startswith("20"):
ranges.append((col, f[0], data.parse_time(f[1]), data.parse_time(f[2]), float(f[5]), f[6]))
by = data.load("data/NORDEND")
for ms in by.values():
for m in ms: m["t"] = m["t"].replace(second=0, microsecond=0)
tg = [s for s in by if s not in REFS]
# thermal check: instants inside a non-thermal label (bounds excluded) are left out, they carry another event
OTHER = [(data.parse_time(l["start"]), data.parse_time(l["end"])) for l in labs if l["family"] not in ("Thermal displacement", "No data")]
def excl(t):
return any(a < t < b for a, b in OTHER)
for l in labs:
S, E = data.parse_time(l["start"]), data.parse_time(l["end"])
if l["family"] == "No data":
print(l.get("id"), l["start"], l["end"], "No data"); continue
if l["family"] == "Thermal displacement":
res = []
for c in ("corr_x", "corr_y"):
for s in tg:
if s == "PR04": continue # Target issue NORDEND-24
night = [m[c] for m in by[s] if S - timedelta(hours=3) <= m["t"] <= S and m[c] is not None and not excl(m["t"])]
if len(night) < 4: continue
lv = statistics.median(night)
hours = {}
for m in by[s]:
if S <= m["t"] <= E and m[c] is not None and not excl(m["t"]):
hours.setdefault(m["t"].replace(minute=0), []).append(m[c])
for h, v in hours.items():
if len(v) >= 3:
res.append(((statistics.median(v) - lv) / noise[(c, s)], s, c, h))
res.sort(key=lambda x: -abs(x[0]))
top = " ".join(f"{s}.{c[-1]} {d:+.1f}@{h:%d %H}h" for d, s, c, h in res[:5])
print(l.get("id"), l["start"], l["end"], "THERMAL max", f"{abs(res[0][0]):.1f}" if res else "-", "|", top)
continue
hit = [r for r in ranges if r[1] not in REFS and r[2] <= E and r[3] >= S]
hit.sort(key=lambda r: -abs(r[4]))
top = " ".join(f"{r[1]}.{r[0][-1]} {r[4]:+.1f}@{r[5][11:16]}" for r in hit[:6])
mx = abs(hit[0][4]) if hit else 0
print(l.get("id"), l["start"], l["end"], l["family"], "max", f"{mx:.1f}", "|", top or "nothing beyond 5")