"""Requalification: for each label, the largest deviation in own noises of a target on corr_x / corr_y inside the label (start < t < end), read from ranges.py outputs (k 5, plain and rolling), and for thermal labels the deviation from the preceding night level (01:00-11:00 UTC of the start day). usage: python3 requal.py """ import json, re, sys, statistics sys.path.insert(0, 'tools/scripts') import data D = sys.argv[1] W = D + '/work/requal/' LINE = re.compile(r'^(\S+?)(\d{4}-\d\d-\d\dT\d\d:\d\dZ)\s+(\S+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)') def parse(path): out, noise, col, mode = [], {}, None, None for ln in open(path): if ln.startswith('== '): col, mode = ln.split()[1].rstrip(':'), 'noise' continue if 'ranges more than' in ln: col, mode = ln.split(':')[0], 'ranges' continue if mode == 'noise': p = ln.split() if len(p) >= 4 and p[0] not in ('sensor',): try: noise[(p[0][:len(p[0])], col)] = float(p[3]) if len(p) > 4 else None except ValueError: pass if mode == 'ranges': m = LINE.match(ln.rstrip()) if m: s, a, b, n, ext, kn, at, rej = m.groups() out.append(dict(sensor=s, col=col, a=a, b=b, noises=float(kn), at=at)) return out, noise def overlaps(r, lab): return r['a'] < lab['end'] and r['b'] > lab['start'] plain, _ = parse(W + 'ranges_k5.txt') roll, _ = parse(W + 'ranges_k5_roll3.txt') refs = lambda s: 'REF' in s # noises from ranges output (plain) noise = {} col = None for ln in open(W + 'ranges_k5.txt'): if ln.startswith('== '): col = ln.split()[1].rstrip(':') continue if 'ranges more' in ln: col = None if col: p = ln.split() if len(p) >= 5 and p[0] != 'sensor' and p[1].isdigit(): try: noise[(p[0], col)] = float(p[3]) except ValueError: pass M = data.load('data/NIEDERHORN') labels = [json.loads(l) for l in open(D + '/work/requalify_input.jsonl')] for lab in labels: def best(rs): rs = [r for r in rs if not refs(r['sensor']) and overlaps(r, lab)] rs = [r for r in rs if lab['start'] < r['at'] < lab['end']] if not rs: return '-' r = max(rs, key=lambda r: abs(r['noises'])) return f"{r['sensor']} {r['col']} {r['noises']:+.1f} at {r['at'][5:16]} ({len(rs)} ranges)" line = f"{lab['id']:15s} {lab['start'][5:16]}→{lab['end'][5:16]} {lab['family'][:12]:12s} plain: {best(plain)} | roll3: {best(roll)}" if lab['family'] == 'Thermal displacement': day = lab['start'][:10] n0, n1 = data.parse_time(day + 'T01:00Z'), data.parse_time(day + 'T11:00Z') a, b = data.parse_time(lab['start']), data.parse_time(lab['end']) top = (0, '') for s, ms in M.items(): if data.is_reference(ms): continue for c in ('corr_x', 'corr_y'): nz = noise.get((s, c)) if not nz: continue night = [m[c] for m in ms if n0 <= m['t'] <= n1 and m[c] is not None and not data.rejected(m)] if len(night) < 4: continue lvl = statistics.median(night) for m in ms: if a < m['t'] < b and m[c] is not None and not data.rejected(m): d = (m[c] - lvl) / nz if abs(d) > abs(top[0]): top = (d, f"{s} {c} at {data.iso(m['t'])[5:16]}") line += f" | night: {top[1]} {top[0]:+.1f}" print(line)