From f5ff3192aec54f6924486695de92c5c4c7d185ee Mon Sep 17 00:00:00 2001 From: maxime Date: Wed, 18 Nov 2026 12:51:30 +0100 Subject: [PATCH] improve: S1 ranges.py --labels F: labels covering each instant of --instants; per label, the largest deviation strictly inside and the instants beyond --k Co-Authored-By: Claude Opus 5.5 --- scripts/ranges.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/scripts/ranges.py b/scripts/ranges.py index 7f38fd5..c8999a0 100644 --- a/scripts/ranges.py +++ b/scripts/ranges.py @@ -2,7 +2,7 @@ Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y] [--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N] - [--sensors A,B] [--start ...] [--end ...] + [--labels F] [--sensors A,B] [--start ...] [--end ...] For each column, each sensor is compared with its own level over the --start/--end window (median of the non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a @@ -27,10 +27,16 @@ measurements, outside --outside). each, the sensors and the deviation in noises (cx corr_x, cy corr_y, ev error_value, otherwise the column name), the sensors rejected at that instant (among --sensors: removing the sensors that are always rejected shortens the list), and each reference of the station, in --sensors or not: error_value ("r" if rejected) / inlier ("-" empty). +4. With --labels F (one JSON per line, start, end, family, id if any; numbered in the order of the file): each line of + --instants ends with the numbers of the labels that cover the instant ("labels: none" otherwise); then, per label, + the measurements strictly inside it (start < instant < end, the instant to the minute: a measurement at a bound is + the point before or after), the largest deviation inside from the level (sensor, column, signed, in noises, instant) + and the number of instants inside where a sensor is more than --k noises from it. Each column is read at the measurements, never averaged per step. Gives values, not a verdict. """ import argparse import bisect +import json import statistics import sys @@ -63,10 +69,19 @@ def main(): a.add_argument("--rolling", type=lambda x: int(x) if x.isdigit() else duration(x), help="N (measurements on either side) or duration (6h, 1d): local level, median of the neighbours without the measurement") a.add_argument("--instants", type=int, help="N: instants where at least N sensors are off level, with rejections and references") + a.add_argument("--labels", help="label file (one JSON per line): labels covering each instant, deviation inside each label") a.add_argument("--sensors", help="comma-separated list (default: all)") a.add_argument("--start", type=parse_time) a.add_argument("--end", type=parse_time) o = a.parse_args() + labels = [] + if o.labels: + for line in open(o.labels): + if line.strip(): + e = json.loads(line) + labels.append((parse_time(e["start"]), parse_time(e["end"]), e)) + minute = lambda t: t.replace(second=0, microsecond=0) + covering = lambda t: [str(n) for n, (t0, t1, _) in enumerate(labels, 1) if t0 < minute(t) < t1] by, names = sensors(o.folder, o.sensors) columns = o.columns.split(",") @@ -78,6 +93,7 @@ def main(): sys.exit("No measurement in the requested window.") roll = (f"{o.rolling} measurements" if isinstance(o.rolling, int) else f"{int(o.rolling.total_seconds() // 60)} min") if o.rolling is not None else "" by_instant = {} # iso(t) -> ["sensor.col+deviation"] (--instants) + deviations = [] # (t, sensor, col, deviation in noises) of every measurement (--labels) for col in columns: print(f"\n== {col}: level, noise, largest jump from one measurement to the next ==") @@ -106,6 +122,8 @@ def main(): lv = local_levels(ms, o.rolling) if o.rolling is not None else [lvl] * len(ms) group = [] # (t, deviation from the level) for (t, v), n in zip(ms + [(None, 0)], lv + [None]): + if labels and t is not None and n is not None: + deviations.append((t, s, col, (v - n) / noise)) if t is not None and n is not None and abs(v - n) > o.k * noise: if group and (t - group[-1][0] > o.merge or (v > n) != (group[-1][1] > 0)): # same direction only ranges.append((s, group, noise, rej)) @@ -139,11 +157,27 @@ def main(): rows = [(t, c) for t, c in sorted(by_instant.items()) if len(c) >= o.instants] print(f"\nInstants where at least {o.instants} sensors are more than {o.k:g} noises from the level{' rolling ±' + roll if roll else ''} " f"({', '.join(columns)}): {len(rows)} of {len(by_instant)} instants with at least one sensor off level") - print("instant n sensors off level (deviation in noises) | rejected | references: error_value (r rejected) / inlier") + print("instant n sensors off level (deviation in noises) | rejected | references: error_value (r rejected) / inlier" + + (" | labels covering" if labels else "")) for t, c in rows: r = " ".join(f"{s} {f(ref_t[s][t]['error_value'], '.3g')}{'r' if rejected(ref_t[s][t]) else ''}/{f(ref_t[s][t]['inlier'], 'g')}" for s in refs if t in ref_t[s]) - print(f"{t} {len(c):3} {' '.join(s + '.' + ','.join(v) for s, v in c.items())} | {','.join(rej_t.get(t, [])) or '-'} | {r or '-'}") + cov = f" | labels: {','.join(covering(parse_time(t))) or 'none'}" if labels else "" + print(f"{t} {len(c):3} {' '.join(s + '.' + ','.join(v) for s, v in c.items())} | {','.join(rej_t.get(t, [])) or '-'} | {r or '-'}{cov}") + + if labels: + print(f"\nPer label ({o.labels}): measurements strictly inside, largest deviation from the level" + f"{' rolling ±' + roll if roll else ''} ({', '.join(columns)}), instants with a sensor more than {o.k:g} noises") + print(f"{'n':>4} {'id':<16}{'start':<18}{'end':<18}{'family':<22}{'inside':>7} largest deviation (noises, at) instants > k") + for n, (t0, t1, e) in enumerate(labels, 1): + ins = [d for d in deviations if t0 < minute(d[0]) < t1] + head = f"{n:4} {str(e.get('id', '-')):<16}{iso(t0):<18}{iso(t1):<18}{e.get('family', '-')[:21]:<22}" + if not ins: + print(head + f"{0:7} (no measurement inside)") + continue + t, s, col, d = max(ins, key=lambda x: abs(x[3])) + over = len({minute(x[0]) for x in ins if abs(x[3]) > o.k}) + print(head + f"{len({minute(x[0]) for x in ins}):7} {s + '.' + ABBREV.get(col, col):<14}{d:+8.1f} {iso(t):<17}{over:>5}") if __name__ == "__main__":