62 lines
3.0 KiB
Python
62 lines
3.0 KiB
Python
"""Requalification: for each label, the largest deviation of a target's corr_x / corr_y in its own noises.
|
|||
|
|
|
||
|
|
Usage: python3 <this> data/JUNGFRAU labels.jsonl START END [EXCLUDED,TARGETS]
|
||
|
|
|
||
|
|
Noise of a sensor: median of |difference| between two successive accepted measurements over START..END
|
||
|
|
(the definition of tools/scripts/ranges.py). Three deviations, targets only (references excluded), accepted
|
||
|
|
measurements strictly inside the label (its bounds are the measurement before / after):
|
||
|
|
roll : value minus the median of the 3 accepted measurements on each side (ranges.py --rolling 3);
|
||
|
|
pre : value minus the median of the accepted measurements of the hour ending at the label start;
|
||
|
|
post : value minus the median of the accepted measurements of the hour starting at the label end.
|
||
|
|
For a thermal phase, pre (warming) or post (cooling) is the night level.
|
||
|
|
"""
|
||
|
|
import json
|
||
|
|
import statistics
|
||
|
|
import sys
|
||
|
|
from datetime import timedelta
|
||
|
|
|
||
|
|
sys.path.insert(0, "tools/scripts")
|
||
|
|
import data # noqa: E402
|
||
|
|
|
||
|
|
folder, labels_path, start, end = sys.argv[1:5]
|
||
|
|
EXCLUDE = set(sys.argv[5].split(",")) if len(sys.argv) > 5 else set() # optional: targets left out
|
||
|
|
t0, t1 = data.parse_time(start), data.parse_time(end)
|
||
|
|
meas = data.load(folder)
|
||
|
|
series = {}
|
||
|
|
for name, ms in meas.items():
|
||
|
|
if data.is_reference(ms) or name in EXCLUDE:
|
||
|
|
continue
|
||
|
|
for col in ("corr_x", "corr_y"):
|
||
|
|
pts = [(m["t"], m[col]) for m in ms
|
||
|
|
if t0 - timedelta(hours=2) <= m["t"] <= t1 + timedelta(hours=2)
|
||
|
|
and not data.rejected(m) and m[col] is not None]
|
||
|
|
if len(pts) < 10:
|
||
|
|
continue
|
||
|
|
inside = [v for t, v in pts if t0 <= t <= t1]
|
||
|
|
noise = statistics.median(abs(b - a) for a, b in zip(inside, inside[1:]))
|
||
|
|
series[(name, col)] = (pts, noise)
|
||
|
|
|
||
|
|
for line in open(labels_path):
|
||
|
|
lab = json.loads(line)
|
||
|
|
a, b = data.parse_time(lab["start"]), data.parse_time(lab["end"])
|
||
|
|
best = {"roll": (0, ""), "pre": (0, ""), "post": (0, "")}
|
||
|
|
for (name, col), (pts, noise) in series.items():
|
||
|
|
if not noise:
|
||
|
|
continue
|
||
|
|
pre = [v for t, v in pts if a - timedelta(hours=1) <= t <= a]
|
||
|
|
post = [v for t, v in pts if b <= t <= b + timedelta(hours=1)]
|
||
|
|
lp = statistics.median(pre) if pre else None
|
||
|
|
lq = statistics.median(post) if post else None
|
||
|
|
for i, (t, v) in enumerate(pts):
|
||
|
|
if not (a < t.replace(second=0, microsecond=0) < b):
|
||
|
|
continue
|
||
|
|
nb = [w for _, w in pts[max(0, i - 3):i]] + [w for _, w in pts[i + 1:i + 4]]
|
||
|
|
cands = {"roll": statistics.median(nb) if nb else None, "pre": lp, "post": lq}
|
||
|
|
for k, lvl in cands.items():
|
||
|
|
if lvl is None:
|
||
|
|
continue
|
||
|
|
d = (v - lvl) / noise
|
||
|
|
if abs(d) > abs(best[k][0]):
|
||
|
|
best[k] = (d, f"{name} {col} {data.iso(t)}")
|
||
|
|
out = " ".join(f"{k} {d:+6.1f} ({w})" for k, (d, w) in best.items())
|
||
|
|
print(f"{lab.get('id', '?'):13s} {lab['start']} {lab['end']} {lab['family'][:10]:10s} {out}")
|