Files
aimsight-labeling/auto/analyses/NORDEND_2026-09-04T0615_2026-09-17T0545/scripts/requalify.py
T
maximeandClaude Opus 5.5 ddb17c2189 auto: labels requalified under the 5-noise rules (only what stands out from the noise): 32 analyses refined, the 6 paused periods verified, the long labels requalified with the fixes the period analysts found; 1153 -> 961 labels
- 32 analyses: each label kept, rewritten or removed by its analyst (section Requalification of the report; mm magnitudes where the targets have them)
- SNCF01 x3, WILDSTRUBEL 2026-09-01 -> 2026-10-06 x3: verified from their existing labels under the new rules
- long labels (no analysis): auto/analyses/long-labels/<STATION>/; NORDEND-169 and -195 run to 07/11, so NORDEND-283 and -284 (their continuation) are removed
- reading-rules.md, stations.md: the citations of removed or rewritten labels point to their successor, or to [expert]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 23:58:00 +01:00

58 lines
3.3 KiB
Python

"""Requalification: per label, the largest deviation of each sensor in its own noises.
Non-thermal: against a rolling local level (median of +-N accepted neighbours, without the point; ranges.py --rolling N).
Thermal: against the level of the preceding night (00:00 to the label start, or 21:00-04:00), on a 5-measurement running median
of the deviation (the cycle, not the spikes). Noise: median of |consecutive differences| over the whole range (ranges.py).
Usage: requalify.py labels.jsonl [N]"""
import sys, json, statistics
from datetime import timedelta
sys.path.insert(0, 'tools/scripts')
import data
lab = sys.argv[1]; EXCL = {'PR04'} # Target issue (NORDEND-24), rejected almost always
ALL = [json.loads(x) for x in open(lab)]
MULTI = {'NORDEND-91', 'NORDEND-107', 'NORDEND-262', 'NORDEND-257'} # multi-measurement episodes labelled apart
OTHER = [(data.parse_time(x['start']), data.parse_time(x['end'])) for x in ALL if x.get('id') in MULTI]
P17 = data.parse_time('2026-09-05T12:15Z') # PR17 jump (NORDEND-94): its thermal level is read after it
N = int(sys.argv[2]) if len(sys.argv) > 2 else 4
S, E = data.parse_time('2026-09-04T06:15Z'), data.parse_time('2026-09-17T05:45Z')
by = data.load('data/NORDEND')
series = {}
for s, ms in by.items():
ms = data.filter_window(ms, S - timedelta(hours=12), E)
for c in ('corr_x', 'corr_y'):
v = [(m['t'], m[c]) for m in ms if m[c] is not None]
w = [x for t, x in v if t >= S]
if len(w) < 20: continue
noise = statistics.median(abs(w[i+1]-w[i]) for i in range(len(w)-1))
if noise < 1e-9: continue
series[(s, c)] = (v, noise, data.is_reference(ms))
for line in open(lab):
l = json.loads(line); a, b = data.parse_time(l['start']), data.parse_time(l['end'])
res = []
for (s, c), (v, noise, ref) in series.items():
if s in EXCL: continue
if l['family'] == 'Thermal displacement':
n0 = a.replace(hour=0, minute=0, second=0) if a.hour >= 4 else a - timedelta(hours=4)
night = [x for t, x in v if n0 <= t < a - timedelta(minutes=10)] or [x for t, x in v if a - timedelta(hours=6) <= t < a]
if len(night) < 4: continue
lev = statistics.median(night)
if s == 'PR17' and a < P17 < b: continue
inn = [(t, x - lev) for t, x in v if a <= t <= b and 6 <= t.hour < 20 and not any(p < t < q for p, q in OTHER)]
sm = [(inn[i][0], statistics.median(d for _, d in inn[max(0, i-2):i+3])) for i in range(len(inn))]
if not sm: continue
t, d = max(sm, key=lambda z: abs(z[1]))
else:
best = (None, 0)
for i, (t, x) in enumerate(v):
if not (a < t < b): continue
nb = [w for _, w in v[max(0, i-N):i] + v[i+1:i+1+N]]
d = x - statistics.median(nb)
if abs(d) > abs(best[1]): best = (t, d)
t, d = best
if t is None: continue
res.append((abs(d)/noise, s, c, d, t, ref))
res.sort(reverse=True)
top = ' '.join(f"{s}{'(ref)' if r else ''}:{c[-1]}{d/abs(d)*k if d else 0:+.1f}@{data.iso(t)[11:16]}" for k, s, c, d, t, r in res[:6])
tgt = [z for z in res if not z[5]]
mx = tgt[0][0] if tgt else 0
print(f"{l.get('id','?'):12s} {l['start']}→{l['end'][11:]} {l['family'][:12]:12s} max_target={mx:5.1f} | {top}")