auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
df979301b5
commit
2a5d87659b
279 files changed
+10758
-710
No files matched your search
@@ -0,0 +1,47 @@
|
||||
"""Per instant: targets whose corr deviates from a local rolling median (+-W measurements, self excluded) by > K noises,
|
||||
and rejected sensors; marks the existing non-thermal labels covering each instant.
|
||||
usage: scan.py LABELS START END [--all] [--k 5] [--from T --to T] (print all instants in a sub-window)"""
|
||||
import sys, json, statistics, argparse
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
ap = argparse.ArgumentParser(); ap.add_argument("labels"); ap.add_argument("start"); ap.add_argument("end")
|
||||
ap.add_argument("--k", type=float, default=5); ap.add_argument("--w", type=int, default=4)
|
||||
ap.add_argument("--frm"); ap.add_argument("--to"); ap.add_argument("--skip", default="PR04")
|
||||
a = ap.parse_args()
|
||||
S, E = data.parse_time(a.start), data.parse_time(a.end)
|
||||
from datetime import timedelta
|
||||
by = data.load("data/NORDEND")
|
||||
for ms in by.values():
|
||||
for m in ms: m["t"] = m["t"].replace(second=0, microsecond=0)
|
||||
labs = [json.loads(l) for l in open(a.labels)]
|
||||
for l in labs: l["s"], l["e"] = data.parse_time(l["start"]), data.parse_time(l["end"])
|
||||
skip = set(a.skip.split(","))
|
||||
win = {s: data.filter_window(m, S - timedelta(hours=3), E + timedelta(hours=3)) for s, m in by.items()}
|
||||
times = sorted({m["t"] for ms in win.values() for m in ms})
|
||||
flags = {t: [] for t in times}; rej = {t: [] for t in times}
|
||||
for s, ms in win.items():
|
||||
ref = data.is_reference(by[s])
|
||||
for m in ms:
|
||||
if data.rejected(m) and s not in skip: rej[m["t"]].append(s)
|
||||
if s in skip: continue
|
||||
for c in ("corr_x", "corr_y"):
|
||||
pts = [(m["t"], m[c]) for m in ms if m[c] is not None]
|
||||
v = [p[1] for p in pts]
|
||||
if len(v) < 20: continue
|
||||
noise = statistics.median(abs(v[i+1]-v[i]) for i in range(len(v)-1)) or 1e-9
|
||||
for i, (t, x) in enumerate(pts):
|
||||
nb = v[max(0, i-a.w):i] + v[i+1:i+1+a.w]
|
||||
d = (x - statistics.median(nb)) / noise
|
||||
if abs(d) > a.k: flags[t].append(f"{s}.{c[-1]}{d:+.0f}")
|
||||
def cover(t):
|
||||
return ",".join(l.get("id", "?").replace("NORDEND-", "") for l in labs if l["s"] <= t <= l["e"] and l["family"] != "Thermal displacement" and not (l["family"] == "Target issue"))
|
||||
F = data.parse_time(a.frm) if a.frm else None; T = data.parse_time(a.to) if a.to else None
|
||||
prev = None
|
||||
for t in times:
|
||||
if t < S or t > E:
|
||||
if not (F and F <= t <= T): continue
|
||||
if prev and t - prev > timedelta(minutes=15): print(f" -- gap {data.iso(prev)} -> {data.iso(t)}")
|
||||
prev = t
|
||||
show = (F and F <= t <= T) or len(flags[t]) >= 2 or rej[t] or any(abs(int(f.split(".")[1][1:])) >= 9 for f in flags[t])
|
||||
if show:
|
||||
print(data.iso(t), f"[{cover(t)}]", len(flags[t]), " ".join(flags[t]), "| rej:", ",".join(rej[t]))
|
||||
Reference in new issue
Block a user