auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
df979301b5
commit
2a5d87659b
279 files changed
+10758
-710
No files matched your search
@@ -0,0 +1,43 @@
|
||||
"""Instants not covered by a label (labels file given) where a sensor is rejected or its corr deviates by more than K noises
|
||||
from its rolling median (+-6 h, without it). Target issue labels cover only their target (named in the description).
|
||||
Usage: python3 uncovered.py LABELS.jsonl [K]"""
|
||||
import sys, json, statistics
|
||||
sys.path.insert(0, "tools/scripts")
|
||||
import data
|
||||
F = "data/FLORIDA"
|
||||
W0, W1 = data.parse_time("2026-10-01T00:00Z"), data.parse_time("2026-10-11T09:45Z")
|
||||
K = float(sys.argv[2]) if len(sys.argv) > 2 else 4
|
||||
labs = [json.loads(l) for l in open(sys.argv[1])]
|
||||
for l in labs:
|
||||
l["s"], l["e"] = data.parse_time(l["start"]), data.parse_time(l["end"])
|
||||
ms = data.load(F)
|
||||
def covered(t, sensor):
|
||||
for l in labs:
|
||||
if l["s"] <= t <= l["e"]:
|
||||
if l["family"] == "Target issue":
|
||||
if l["description"].startswith(sensor):
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
return False
|
||||
for s, mm in ms.items():
|
||||
w = [m for m in data.filter_window(mm, W0, W1)]
|
||||
for c in ("corr_x", "corr_y"):
|
||||
vals = [(m["t"].replace(second=0, microsecond=0), m[c]) for m in w if m[c] is not None]
|
||||
if len(vals) < 10:
|
||||
continue
|
||||
diffs = [abs(vals[i + 1][1] - vals[i][1]) for i in range(len(vals) - 1)]
|
||||
noise = statistics.median(diffs) or 1e-9
|
||||
if noise < 1e-6:
|
||||
continue
|
||||
for i, (t, v) in enumerate(vals):
|
||||
nb = [x for (tt, x) in vals if 0 < abs((tt - t).total_seconds()) <= 6 * 3600]
|
||||
if len(nb) < 5:
|
||||
continue
|
||||
d = (v - statistics.median(nb)) / noise
|
||||
if abs(d) > K and not covered(t, s):
|
||||
print(f"{data.iso(t)} {s:10} {c} dev {d:+.1f} noises ({v - statistics.median(nb):+.3f})")
|
||||
for m in w:
|
||||
t = m["t"].replace(second=0, microsecond=0)
|
||||
if data.rejected(m) and not covered(t, s):
|
||||
print(f"{data.iso(t)} {s:10} REJECTED ev {m['error_value']:.2f}")
|
||||
Reference in new issue
Block a user