"""Per instant of a station over a range: sensors rejected, sensors 'high' (visibility rule 1), median error z (MAD) and median |corr| deviation. Usage: instants.py [--history 3]""" import statistics import sys from datetime import datetime, timedelta, timezone sys.path.insert(0, sys.argv[1] + "/scripts") import visibility as V # noqa: E402 from data import load # noqa: E402 folder, a, b = sys.argv[2], sys.argv[3], sys.argv[4] p = lambda s: datetime.fromisoformat(s).replace(tzinfo=timezone.utc) # noqa: E731 A, B = p(a), p(b) ms = load(folder) ms = {s: k for s, v in ms.items() if (k := [m for m in v if A - timedelta(days=3) <= m["t"] < B])} stats = {} for s, v in ms.items(): e = [m["error_value"] for m in v if m["error_value"] is not None] if len(e) < 10: continue med = statistics.median(e) mad = statistics.median(abs(x - med) for x in e) or 1e-9 cx = [m["corr_x"] for m in v if m["corr_x"] is not None] cy = [m["corr_y"] for m in v if m["corr_y"] is not None] stats[s] = (med, mad, statistics.median(cx) if cx else 0, statistics.median(cy) if cy else 0) by = {} for s, v in ms.items(): if s not in stats: continue med, mad, mx, my = stats[s] for m in v: if not A <= m["t"] < B: continue d = by.setdefault(m["t"], {"rej": 0, "high": 0, "z": [], "dev": [], "n": 0}) d["n"] += 1 x = m["error_value"] if x is None: continue rej = m["error_threshold"] and x > m["error_threshold"] d["rej"] += bool(rej) if x > med + V.K_MAD * mad and (x > V.FACTOR * med or rej and x > V.REJECT_FACTOR * med): d["high"] += 1 d["z"].append((x - med) / mad) if m["corr_x"] is not None: d["dev"].append(abs(m["corr_x"] - mx) + abs(m["corr_y"] - my)) for t in sorted(by): d = by[t] print(t.strftime("%m-%d %H:%M"), f"n={d['n']:3d} rej={d['rej']:3d} high={d['high']:3d}", f"zmed={statistics.median(d['z']) if d['z'] else 0:7.1f}", f"devmed={statistics.median(d['dev']) if d['dev'] else 0:7.2f}", f"devmax={max(d['dev']) if d['dev'] else 0:7.2f}")