39 lines
2.3 KiB
Python
39 lines
2.3 KiB
Python
"""Per instant: rejected sensors and corr jumps (vs rolling local median, in own noises), and the labels covering it.
|
|||
|
|
usage: scan.py [labels_file] [K]"""
|
||
|
|
import json, sys, statistics as st
|
||
|
|
sys.path.insert(0, 'tools/scripts')
|
||
|
|
import data
|
||
|
|
from datetime import datetime, timezone, timedelta
|
||
|
|
D = 'labeling/auto/analyses/NIEDERHORN_2026-10-21T1200_2026-11-01T0000/'
|
||
|
|
ALWAYS = {'e03', 'e04', 'e05', 'e07', 'e08', 'e11', 'e12'}
|
||
|
|
lf = sys.argv[1] if len(sys.argv) > 1 else D + 'work/existing.jsonl'
|
||
|
|
K = float(sys.argv[2]) if len(sys.argv) > 2 else 5
|
||
|
|
labels = [json.loads(l) for f in (lf, D + 'work/spanning.jsonl') for l in open(f)]
|
||
|
|
for i, l in enumerate(labels):
|
||
|
|
l['s'] = data.parse_time(l['start'].replace('Z', '+00:00')); l['e'] = data.parse_time(l['end'].replace('Z', '+00:00'))
|
||
|
|
l.setdefault('id', f'new{i}')
|
||
|
|
lo = datetime(2026, 10, 19, tzinfo=timezone.utc); hi = datetime(2026, 11, 3, tzinfo=timezone.utc)
|
||
|
|
M = {s: [m for m in ms if lo <= m['t'] <= hi] for s, ms in data.load('data/NIEDERHORN').items()}
|
||
|
|
T = sorted({m['t'].replace(second=0, microsecond=0) for ms in M.values() for m in ms})
|
||
|
|
rej, dev = {}, {}
|
||
|
|
for s, ms in M.items():
|
||
|
|
for m in ms:
|
||
|
|
t = m['t'].replace(second=0, microsecond=0)
|
||
|
|
if data.rejected(m): rej.setdefault(t, []).append(s)
|
||
|
|
for c in ('corr_x', 'corr_y'):
|
||
|
|
v = [(m['t'].replace(second=0, microsecond=0), m[c]) for m in ms if m[c] is not None]
|
||
|
|
if len(v) < 20: continue
|
||
|
|
noise = st.median(abs(v[i + 1][1] - v[i][1]) for i in range(len(v) - 1)) or 1e-9
|
||
|
|
for i, (t, x) in enumerate(v):
|
||
|
|
nb = [y for _, y in v[max(0, i - 6):i] + v[i + 1:i + 7]]
|
||
|
|
d = (x - st.median(nb)) / noise
|
||
|
|
if abs(d) > K: dev.setdefault(t, {}).setdefault(s, []).append(f"{c[-1]}{d:+.0f}")
|
||
|
|
a, b = datetime(2026, 10, 21, 11, tzinfo=timezone.utc), datetime(2026, 11, 1, 1, tzinfo=timezone.utc)
|
||
|
|
sh = lambda x: x.replace('PWD_', '').replace('_C09', '')
|
||
|
|
for t in T:
|
||
|
|
if not a <= t <= b: continue
|
||
|
|
r = [sh(s) for s in rej.get(t, []) if s not in ALWAYS]
|
||
|
|
j = [f"{sh(s)}({'/'.join(v)})" for s, v in sorted(dev.get(t, {}).items())]
|
||
|
|
cov = [l['id'].replace('NIEDERHORN-', '') + l['family'][:3] for l in labels if l['s'] < t < l['e'] and l['family'] != 'Target issue']
|
||
|
|
if r or j:
|
||
|
|
print(t.strftime('%m-%dT%H'), 'REJ:', ','.join(sorted(r)), '| JMP:', ' '.join(j), '| cov:', ','.join(cov))
|