Files
aimsight-labeling/auto/analyses/NIEDERHORN_2026-10-21T1200_2026-11-01T0000/scripts/requal.py
T
maximeandClaude Opus 5.5 ddb17c2189 auto: labels requalified under the 5-noise rules (only what stands out from the noise): 32 analyses refined, the 6 paused periods verified, the long labels requalified with the fixes the period analysts found; 1153 -> 961 labels
- 32 analyses: each label kept, rewritten or removed by its analyst (section Requalification of the report; mm magnitudes where the targets have them)
- SNCF01 x3, WILDSTRUBEL 2026-09-01 -> 2026-10-06 x3: verified from their existing labels under the new rules
- long labels (no analysis): auto/analyses/long-labels/<STATION>/; NORDEND-169 and -195 run to 07/11, so NORDEND-283 and -284 (their continuation) are removed
- reading-rules.md, stations.md: the citations of removed or rewritten labels point to their successor, or to [expert]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 23:58:00 +01:00

97 lines
3.7 KiB
Python

"""Requalification: for each label, the largest deviation in own noises of a target on corr_x / corr_y
inside the label (start < t < end), read from ranges.py outputs (k 5, plain and rolling),
and for thermal labels the deviation from the preceding night level (01:00-11:00 UTC of the start day).
usage: python3 requal.py <analysis folder>"""
import json, re, sys, statistics
sys.path.insert(0, 'tools/scripts')
import data
D = sys.argv[1]
W = D + '/work/requal/'
LINE = re.compile(r'^(\S+?)(\d{4}-\d\d-\d\dT\d\d:\d\dZ)\s+(\S+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)')
def parse(path):
out, noise, col, mode = [], {}, None, None
for ln in open(path):
if ln.startswith('== '):
col, mode = ln.split()[1].rstrip(':'), 'noise'
continue
if 'ranges more than' in ln:
col, mode = ln.split(':')[0], 'ranges'
continue
if mode == 'noise':
p = ln.split()
if len(p) >= 4 and p[0] not in ('sensor',):
try:
noise[(p[0][:len(p[0])], col)] = float(p[3]) if len(p) > 4 else None
except ValueError:
pass
if mode == 'ranges':
m = LINE.match(ln.rstrip())
if m:
s, a, b, n, ext, kn, at, rej = m.groups()
out.append(dict(sensor=s, col=col, a=a, b=b, noises=float(kn), at=at))
return out, noise
def overlaps(r, lab):
return r['a'] < lab['end'] and r['b'] > lab['start']
plain, _ = parse(W + 'ranges_k5.txt')
roll, _ = parse(W + 'ranges_k5_roll3.txt')
refs = lambda s: 'REF' in s
# noises from ranges output (plain)
noise = {}
col = None
for ln in open(W + 'ranges_k5.txt'):
if ln.startswith('== '):
col = ln.split()[1].rstrip(':')
continue
if 'ranges more' in ln:
col = None
if col:
p = ln.split()
if len(p) >= 5 and p[0] != 'sensor' and p[1].isdigit():
try:
noise[(p[0], col)] = float(p[3])
except ValueError:
pass
M = data.load('data/NIEDERHORN')
labels = [json.loads(l) for l in open(D + '/work/requalify_input.jsonl')]
for lab in labels:
def best(rs):
rs = [r for r in rs if not refs(r['sensor']) and overlaps(r, lab)]
rs = [r for r in rs if lab['start'] < r['at'] < lab['end']]
if not rs:
return '-'
r = max(rs, key=lambda r: abs(r['noises']))
return f"{r['sensor']} {r['col']} {r['noises']:+.1f} at {r['at'][5:16]} ({len(rs)} ranges)"
line = f"{lab['id']:15s} {lab['start'][5:16]}→{lab['end'][5:16]} {lab['family'][:12]:12s} plain: {best(plain)} | roll3: {best(roll)}"
if lab['family'] == 'Thermal displacement':
day = lab['start'][:10]
n0, n1 = data.parse_time(day + 'T01:00Z'), data.parse_time(day + 'T11:00Z')
a, b = data.parse_time(lab['start']), data.parse_time(lab['end'])
top = (0, '')
for s, ms in M.items():
if data.is_reference(ms):
continue
for c in ('corr_x', 'corr_y'):
nz = noise.get((s, c))
if not nz:
continue
night = [m[c] for m in ms if n0 <= m['t'] <= n1 and m[c] is not None and not data.rejected(m)]
if len(night) < 4:
continue
lvl = statistics.median(night)
for m in ms:
if a < m['t'] < b and m[c] is not None and not data.rejected(m):
d = (m[c] - lvl) / nz
if abs(d) > abs(top[0]):
top = (d, f"{s} {c} at {data.iso(m['t'])[5:16]}")
line += f" | night: {top[1]} {top[0]:+.1f}"
print(line)