"""Per instant: rejected sensors and corr jumps (vs rolling local median, in own noises), and the labels covering it. usage: scan.py [labels_file] [K]""" import json, sys, statistics as st sys.path.insert(0, 'tools/scripts') import data from datetime import datetime, timezone, timedelta D = 'labeling/auto/analyses/NIEDERHORN_2026-10-21T1200_2026-11-01T0000/' ALWAYS = {'e03', 'e04', 'e05', 'e07', 'e08', 'e11', 'e12'} lf = sys.argv[1] if len(sys.argv) > 1 else D + 'work/existing.jsonl' K = float(sys.argv[2]) if len(sys.argv) > 2 else 5 labels = [json.loads(l) for f in (lf, D + 'work/spanning.jsonl') for l in open(f)] for i, l in enumerate(labels): l['s'] = data.parse_time(l['start'].replace('Z', '+00:00')); l['e'] = data.parse_time(l['end'].replace('Z', '+00:00')) l.setdefault('id', f'new{i}') lo = datetime(2026, 10, 19, tzinfo=timezone.utc); hi = datetime(2026, 11, 3, tzinfo=timezone.utc) M = {s: [m for m in ms if lo <= m['t'] <= hi] for s, ms in data.load('data/NIEDERHORN').items()} T = sorted({m['t'].replace(second=0, microsecond=0) for ms in M.values() for m in ms}) rej, dev = {}, {} for s, ms in M.items(): for m in ms: t = m['t'].replace(second=0, microsecond=0) if data.rejected(m): rej.setdefault(t, []).append(s) for c in ('corr_x', 'corr_y'): v = [(m['t'].replace(second=0, microsecond=0), m[c]) for m in ms if m[c] is not None] if len(v) < 20: continue noise = st.median(abs(v[i + 1][1] - v[i][1]) for i in range(len(v) - 1)) or 1e-9 for i, (t, x) in enumerate(v): nb = [y for _, y in v[max(0, i - 6):i] + v[i + 1:i + 7]] d = (x - st.median(nb)) / noise if abs(d) > K: dev.setdefault(t, {}).setdefault(s, []).append(f"{c[-1]}{d:+.0f}") a, b = datetime(2026, 10, 21, 11, tzinfo=timezone.utc), datetime(2026, 11, 1, 1, tzinfo=timezone.utc) sh = lambda x: x.replace('PWD_', '').replace('_C09', '') for t in T: if not a <= t <= b: continue r = [sh(s) for s in rej.get(t, []) if s not in ALWAYS] j = [f"{sh(s)}({'/'.join(v)})" for s, v in sorted(dev.get(t, {}).items())] cov = [l['id'].replace('NIEDERHORN-', '') + l['family'][:3] for l in labels if l['s'] < t < l['e'] and l['family'] != 'Target issue'] if r or j: print(t.strftime('%m-%dT%H'), 'REJ:', ','.join(sorted(r)), '| JMP:', ' '.join(j), '| cov:', ','.join(cov))