auto: the analysts verified the labels of 32 periods (11 stations, up to 14 days each): 710 labels -> 835, 587 unchanged; one analysis folder per period with its report (section Verification)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 15:13:44 +01:00
1 parent df979301b5
commit 2a5d87659b
279 files changed
+10758 -710

No files matched your search

@@ -0,0 +1,43 @@
"""Instants not covered by a label (labels file given) where a sensor is rejected or its corr deviates by more than K noises
from its rolling median (+-6 h, without it). Target issue labels cover only their target (named in the description).
Usage: python3 uncovered.py LABELS.jsonl [K]"""
import sys, json, statistics
sys.path.insert(0, "tools/scripts")
import data
F = "data/FLORIDA"
W0, W1 = data.parse_time("2026-10-01T00:00Z"), data.parse_time("2026-10-11T09:45Z")
K = float(sys.argv[2]) if len(sys.argv) > 2 else 4
labs = [json.loads(l) for l in open(sys.argv[1])]
for l in labs:
l["s"], l["e"] = data.parse_time(l["start"]), data.parse_time(l["end"])
ms = data.load(F)
def covered(t, sensor):
for l in labs:
if l["s"] <= t <= l["e"]:
if l["family"] == "Target issue":
if l["description"].startswith(sensor):
return True
else:
return True
return False
for s, mm in ms.items():
w = [m for m in data.filter_window(mm, W0, W1)]
for c in ("corr_x", "corr_y"):
vals = [(m["t"].replace(second=0, microsecond=0), m[c]) for m in w if m[c] is not None]
if len(vals) < 10:
continue
diffs = [abs(vals[i + 1][1] - vals[i][1]) for i in range(len(vals) - 1)]
noise = statistics.median(diffs) or 1e-9
if noise < 1e-6:
continue
for i, (t, v) in enumerate(vals):
nb = [x for (tt, x) in vals if 0 < abs((tt - t).total_seconds()) <= 6 * 3600]
if len(nb) < 5:
continue
d = (v - statistics.median(nb)) / noise
if abs(d) > K and not covered(t, s):
print(f"{data.iso(t)} {s:10} {c} dev {d:+.1f} noises ({v - statistics.median(nb):+.3f})")
for m in w:
t = m["t"].replace(second=0, microsecond=0)
if data.rejected(m) and not covered(t, s):
print(f"{data.iso(t)} {s:10} REJECTED ev {m['error_value']:.2f}")