- 32 analyses: each label kept, rewritten or removed by its analyst (section Requalification of the report; mm magnitudes where the targets have them) - SNCF01 x3, WILDSTRUBEL 2026-09-01 -> 2026-10-06 x3: verified from their existing labels under the new rules - long labels (no analysis): auto/analyses/long-labels/<STATION>/; NORDEND-169 and -195 run to 07/11, so NORDEND-283 and -284 (their continuation) are removed - reading-rules.md, stations.md: the citations of removed or rewritten labels point to their successor, or to [expert] Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
97 lines
3.7 KiB
Python
97 lines
3.7 KiB
Python
"""Requalification: for each label, the largest deviation in own noises of a target on corr_x / corr_y
|
|
inside the label (start < t < end), read from ranges.py outputs (k 5, plain and rolling),
|
|
and for thermal labels the deviation from the preceding night level (01:00-11:00 UTC of the start day).
|
|
usage: python3 requal.py <analysis folder>"""
|
|
import json, re, sys, statistics
|
|
sys.path.insert(0, 'tools/scripts')
|
|
import data
|
|
|
|
D = sys.argv[1]
|
|
W = D + '/work/requal/'
|
|
LINE = re.compile(r'^(\S+?)(\d{4}-\d\d-\d\dT\d\d:\d\dZ)\s+(\S+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)')
|
|
|
|
|
|
def parse(path):
|
|
out, noise, col, mode = [], {}, None, None
|
|
for ln in open(path):
|
|
if ln.startswith('== '):
|
|
col, mode = ln.split()[1].rstrip(':'), 'noise'
|
|
continue
|
|
if 'ranges more than' in ln:
|
|
col, mode = ln.split(':')[0], 'ranges'
|
|
continue
|
|
if mode == 'noise':
|
|
p = ln.split()
|
|
if len(p) >= 4 and p[0] not in ('sensor',):
|
|
try:
|
|
noise[(p[0][:len(p[0])], col)] = float(p[3]) if len(p) > 4 else None
|
|
except ValueError:
|
|
pass
|
|
if mode == 'ranges':
|
|
m = LINE.match(ln.rstrip())
|
|
if m:
|
|
s, a, b, n, ext, kn, at, rej = m.groups()
|
|
out.append(dict(sensor=s, col=col, a=a, b=b, noises=float(kn), at=at))
|
|
return out, noise
|
|
|
|
|
|
def overlaps(r, lab):
|
|
return r['a'] < lab['end'] and r['b'] > lab['start']
|
|
|
|
|
|
plain, _ = parse(W + 'ranges_k5.txt')
|
|
roll, _ = parse(W + 'ranges_k5_roll3.txt')
|
|
refs = lambda s: 'REF' in s
|
|
|
|
# noises from ranges output (plain)
|
|
noise = {}
|
|
col = None
|
|
for ln in open(W + 'ranges_k5.txt'):
|
|
if ln.startswith('== '):
|
|
col = ln.split()[1].rstrip(':')
|
|
continue
|
|
if 'ranges more' in ln:
|
|
col = None
|
|
if col:
|
|
p = ln.split()
|
|
if len(p) >= 5 and p[0] != 'sensor' and p[1].isdigit():
|
|
try:
|
|
noise[(p[0], col)] = float(p[3])
|
|
except ValueError:
|
|
pass
|
|
|
|
M = data.load('data/NIEDERHORN')
|
|
labels = [json.loads(l) for l in open(D + '/work/requalify_input.jsonl')]
|
|
for lab in labels:
|
|
def best(rs):
|
|
rs = [r for r in rs if not refs(r['sensor']) and overlaps(r, lab)]
|
|
rs = [r for r in rs if lab['start'] < r['at'] < lab['end']]
|
|
if not rs:
|
|
return '-'
|
|
r = max(rs, key=lambda r: abs(r['noises']))
|
|
return f"{r['sensor']} {r['col']} {r['noises']:+.1f} at {r['at'][5:16]} ({len(rs)} ranges)"
|
|
line = f"{lab['id']:15s} {lab['start'][5:16]}→{lab['end'][5:16]} {lab['family'][:12]:12s} plain: {best(plain)} | roll3: {best(roll)}"
|
|
if lab['family'] == 'Thermal displacement':
|
|
day = lab['start'][:10]
|
|
n0, n1 = data.parse_time(day + 'T01:00Z'), data.parse_time(day + 'T11:00Z')
|
|
a, b = data.parse_time(lab['start']), data.parse_time(lab['end'])
|
|
top = (0, '')
|
|
for s, ms in M.items():
|
|
if data.is_reference(ms):
|
|
continue
|
|
for c in ('corr_x', 'corr_y'):
|
|
nz = noise.get((s, c))
|
|
if not nz:
|
|
continue
|
|
night = [m[c] for m in ms if n0 <= m['t'] <= n1 and m[c] is not None and not data.rejected(m)]
|
|
if len(night) < 4:
|
|
continue
|
|
lvl = statistics.median(night)
|
|
for m in ms:
|
|
if a < m['t'] < b and m[c] is not None and not data.rejected(m):
|
|
d = (m[c] - lvl) / nz
|
|
if abs(d) > abs(top[0]):
|
|
top = (d, f"{s} {c} at {data.iso(m['t'])[5:16]}")
|
|
line += f" | night: {top[1]} {top[0]:+.1f}"
|
|
print(line)
|