improve: S1

ranges.py --labels F: labels covering each instant of --instants; per label, the largest deviation strictly inside and the instants beyond --k

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-18 12:51:30 +01:00
1 parent 9900604002
commit f5ff3192ae
1 file changed
+37 -3
+37 -3
View File
@@ -2,7 +2,7 @@
Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y] Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y]
[--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N] [--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N]
[--sensors A,B] [--start ...] [--end ...] [--labels F] [--sensors A,B] [--start ...] [--end ...]
For each column, each sensor is compared with its own level over the --start/--end window (median of the For each column, each sensor is compared with its own level over the --start/--end window (median of the
non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a
@@ -27,10 +27,16 @@ measurements, outside --outside).
each, the sensors and the deviation in noises (cx corr_x, cy corr_y, ev error_value, otherwise the column name), each, the sensors and the deviation in noises (cx corr_x, cy corr_y, ev error_value, otherwise the column name),
the sensors rejected at that instant (among --sensors: removing the sensors that are always rejected shortens the the sensors rejected at that instant (among --sensors: removing the sensors that are always rejected shortens the
list), and each reference of the station, in --sensors or not: error_value ("r" if rejected) / inlier ("-" empty). list), and each reference of the station, in --sensors or not: error_value ("r" if rejected) / inlier ("-" empty).
4. With --labels F (one JSON per line, start, end, family, id if any; numbered in the order of the file): each line of
--instants ends with the numbers of the labels that cover the instant ("labels: none" otherwise); then, per label,
the measurements strictly inside it (start < instant < end, the instant to the minute: a measurement at a bound is
the point before or after), the largest deviation inside from the level (sensor, column, signed, in noises, instant)
and the number of instants inside where a sensor is more than --k noises from it.
Each column is read at the measurements, never averaged per step. Gives values, not a verdict. Each column is read at the measurements, never averaged per step. Gives values, not a verdict.
""" """
import argparse import argparse
import bisect import bisect
import json
import statistics import statistics
import sys import sys
@@ -63,10 +69,19 @@ def main():
a.add_argument("--rolling", type=lambda x: int(x) if x.isdigit() else duration(x), a.add_argument("--rolling", type=lambda x: int(x) if x.isdigit() else duration(x),
help="N (measurements on either side) or duration (6h, 1d): local level, median of the neighbours without the measurement") help="N (measurements on either side) or duration (6h, 1d): local level, median of the neighbours without the measurement")
a.add_argument("--instants", type=int, help="N: instants where at least N sensors are off level, with rejections and references") a.add_argument("--instants", type=int, help="N: instants where at least N sensors are off level, with rejections and references")
a.add_argument("--labels", help="label file (one JSON per line): labels covering each instant, deviation inside each label")
a.add_argument("--sensors", help="comma-separated list (default: all)") a.add_argument("--sensors", help="comma-separated list (default: all)")
a.add_argument("--start", type=parse_time) a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time) a.add_argument("--end", type=parse_time)
o = a.parse_args() o = a.parse_args()
labels = []
if o.labels:
for line in open(o.labels):
if line.strip():
e = json.loads(line)
labels.append((parse_time(e["start"]), parse_time(e["end"]), e))
minute = lambda t: t.replace(second=0, microsecond=0)
covering = lambda t: [str(n) for n, (t0, t1, _) in enumerate(labels, 1) if t0 < minute(t) < t1]
by, names = sensors(o.folder, o.sensors) by, names = sensors(o.folder, o.sensors)
columns = o.columns.split(",") columns = o.columns.split(",")
@@ -78,6 +93,7 @@ def main():
sys.exit("No measurement in the requested window.") sys.exit("No measurement in the requested window.")
roll = (f"{o.rolling} measurements" if isinstance(o.rolling, int) else f"{int(o.rolling.total_seconds() // 60)} min") if o.rolling is not None else "" roll = (f"{o.rolling} measurements" if isinstance(o.rolling, int) else f"{int(o.rolling.total_seconds() // 60)} min") if o.rolling is not None else ""
by_instant = {} # iso(t) -> ["sensor.col+deviation"] (--instants) by_instant = {} # iso(t) -> ["sensor.col+deviation"] (--instants)
deviations = [] # (t, sensor, col, deviation in noises) of every measurement (--labels)
for col in columns: for col in columns:
print(f"\n== {col}: level, noise, largest jump from one measurement to the next ==") print(f"\n== {col}: level, noise, largest jump from one measurement to the next ==")
@@ -106,6 +122,8 @@ def main():
lv = local_levels(ms, o.rolling) if o.rolling is not None else [lvl] * len(ms) lv = local_levels(ms, o.rolling) if o.rolling is not None else [lvl] * len(ms)
group = [] # (t, deviation from the level) group = [] # (t, deviation from the level)
for (t, v), n in zip(ms + [(None, 0)], lv + [None]): for (t, v), n in zip(ms + [(None, 0)], lv + [None]):
if labels and t is not None and n is not None:
deviations.append((t, s, col, (v - n) / noise))
if t is not None and n is not None and abs(v - n) > o.k * noise: if t is not None and n is not None and abs(v - n) > o.k * noise:
if group and (t - group[-1][0] > o.merge or (v > n) != (group[-1][1] > 0)): # same direction only if group and (t - group[-1][0] > o.merge or (v > n) != (group[-1][1] > 0)): # same direction only
ranges.append((s, group, noise, rej)) ranges.append((s, group, noise, rej))
@@ -139,11 +157,27 @@ def main():
rows = [(t, c) for t, c in sorted(by_instant.items()) if len(c) >= o.instants] rows = [(t, c) for t, c in sorted(by_instant.items()) if len(c) >= o.instants]
print(f"\nInstants where at least {o.instants} sensors are more than {o.k:g} noises from the level{' rolling ±' + roll if roll else ''} " print(f"\nInstants where at least {o.instants} sensors are more than {o.k:g} noises from the level{' rolling ±' + roll if roll else ''} "
f"({', '.join(columns)}): {len(rows)} of {len(by_instant)} instants with at least one sensor off level") f"({', '.join(columns)}): {len(rows)} of {len(by_instant)} instants with at least one sensor off level")
print("instant n sensors off level (deviation in noises) | rejected | references: error_value (r rejected) / inlier") print("instant n sensors off level (deviation in noises) | rejected | references: error_value (r rejected) / inlier"
+ (" | labels covering" if labels else ""))
for t, c in rows: for t, c in rows:
r = " ".join(f"{s} {f(ref_t[s][t]['error_value'], '.3g')}{'r' if rejected(ref_t[s][t]) else ''}/{f(ref_t[s][t]['inlier'], 'g')}" r = " ".join(f"{s} {f(ref_t[s][t]['error_value'], '.3g')}{'r' if rejected(ref_t[s][t]) else ''}/{f(ref_t[s][t]['inlier'], 'g')}"
for s in refs if t in ref_t[s]) for s in refs if t in ref_t[s])
print(f"{t} {len(c):3} {' '.join(s + '.' + ','.join(v) for s, v in c.items())} | {','.join(rej_t.get(t, [])) or '-'} | {r or '-'}") cov = f" | labels: {','.join(covering(parse_time(t))) or 'none'}" if labels else ""
print(f"{t} {len(c):3} {' '.join(s + '.' + ','.join(v) for s, v in c.items())} | {','.join(rej_t.get(t, [])) or '-'} | {r or '-'}{cov}")
if labels:
print(f"\nPer label ({o.labels}): measurements strictly inside, largest deviation from the level"
f"{' rolling ±' + roll if roll else ''} ({', '.join(columns)}), instants with a sensor more than {o.k:g} noises")
print(f"{'n':>4} {'id':<16}{'start':<18}{'end':<18}{'family':<22}{'inside':>7} largest deviation (noises, at) instants > k")
for n, (t0, t1, e) in enumerate(labels, 1):
ins = [d for d in deviations if t0 < minute(d[0]) < t1]
head = f"{n:4} {str(e.get('id', '-')):<16}{iso(t0):<18}{iso(t1):<18}{e.get('family', '-')[:21]:<22}"
if not ins:
print(head + f"{0:7} (no measurement inside)")
continue
t, s, col, d = max(ins, key=lambda x: abs(x[3]))
over = len({minute(x[0]) for x in ins if abs(x[3]) > o.k})
print(head + f"{len({minute(x[0]) for x in ins}):7} {s + '.' + ABBREV.get(col, col):<14}{d:+8.1f} {iso(t):<17}{over:>5}")
if __name__ == "__main__": if __name__ == "__main__":