From a9948d3404e94946b7cd1f03d29baa8a20c3a919 Mon Sep 17 00:00:00 2001 From: maxime Date: Wed, 18 Nov 2026 16:58:59 +0100 Subject: [PATCH] improve: S1 Co-Authored-By: Claude Opus 5.5 --- scripts/ranges.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/ranges.py b/scripts/ranges.py index 80aed19..7b58e7a 100644 --- a/scripts/ranges.py +++ b/scripts/ranges.py @@ -2,7 +2,7 @@ Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y] [--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N] - [--labels F [--margin 6h] [--top 5]] [--sensors A,B] [--start ...] [--end ...] + [--labels F [--margin 6h] [--top 5] [--smoothing N] [--shorter-out]] [--sensors A,B] [--start ...] [--end ...] For each column, each sensor is compared with its own level over the --start/--end window (median of the non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a @@ -39,7 +39,11 @@ measurements, outside --outside). the --margin before its start, default 6h; signed, in units and in noises, instant) and the net change (median of the --margin after its end minus that level), with the number of measurements before and after, and how many of the measurements inside are more than --k noises from the level before (one: a single point; many: sustained); a label without - a measurement inside (an outage) gives the net change alone, largest first. Labels near the + a measurement inside (an outage) gives the net change alone, largest first. --smoothing N: in this size, each + measurement inside is the median of the N centred on it (a long label is not sized by a one-point jump); + --shorter-out: this size leaves out the measurements inside the shorter labels of the file that overlap the label, + before, inside and after (a long label is not sized by a short event inside it or by one under its level before; + a label that only touches it, like the previous day, is kept). Labels near the edge of --start/--end: widen the window by --margin, or before/after lack measurements. Each column is read at the measurements, never averaged per step. Gives values, not a verdict. """ @@ -81,6 +85,8 @@ def main(): a.add_argument("--labels", help="label file (one JSON per line): labels covering each instant, deviation inside each label") a.add_argument("--margin", default="6h", type=duration, help="with --labels: the level before (and after) each label is the median of this duration before its start (after its end)") a.add_argument("--top", type=int, default=5, help="with --labels: sensor-columns listed per label, largest deviation from the level before first") + a.add_argument("--smoothing", type=int, default=1, metavar="N", help="with --labels: in the size of a label, each measurement inside is the median of the N centred on it (default 1: none)") + a.add_argument("--shorter-out", action="store_true", help="with --labels: the size of a label leaves out the measurements inside the shorter labels of the file that overlap it (before, inside, after)") a.add_argument("--sensors", help="comma-separated list (default: all)") a.add_argument("--start", type=parse_time) a.add_argument("--end", type=parse_time) @@ -187,7 +193,9 @@ def main(): m = int(o.margin.total_seconds() // 60) print(f"{'':6}{'sensor.column':<22}{'before':>7}{'extreme':>11}{'(n)':>8} {'at':<17}{'net':>11}{'(n)':>8}{'after':>6}{'>k':>9}" f" (then, up to {o.top} per label: extreme inside from the median of the {m} min before the start;" - f" net: median of the {m} min after the end minus it, alone when no measurement is inside; before, after: their measurements; >k: measurements inside more than {o.k:g} noises from the level before, of those inside; n: noises)") + f" net: median of the {m} min after the end minus it, alone when no measurement is inside; before, after: their measurements; >k: measurements inside more than {o.k:g} noises from the level before, of those inside; n: noises" + + (f"; inside smoothed by the median of {o.smoothing} measurements" if o.smoothing > 1 else "") + + ("; the measurements inside shorter labels left out" if o.shorter_out else "") + ")") for n, (t0, t1, e) in enumerate(labels, 1): ins = [d for d in deviations if t0 < minute(d[0]) < t1] head = f"{n:4} {str(e.get('id', '-')):<16}{iso(t0):<18}{iso(t1):<18}{e.get('family', '-')[:21]:<22}" @@ -201,9 +209,14 @@ def main(): print(head + f"{len({minute(x[0]) for x in ins}):7} {s + '.' + ABBREV.get(col, col):<14}{d:+8.1f} {iso(t):<17}{over:>5}" f"{f'{sum(v >= abs(d) for v in near)}/{len(near)}':>12}") rows = [] # size against the level before the label, and the level after it (net alone without a measurement inside) + shorter = [(a_, b) for a_, b, _ in labels if b - a_ < t1 - t0 and a_ < t1 and b > t0] if o.shorter_out else [] for (s, col), (ms, noise) in values.items(): + ms = [(t, v) for t, v in ms if t0 - o.margin <= minute(t) <= t1 + o.margin + and not any(a_ < minute(t) < b for a_, b in shorter)] pre = [v for t, v in ms if t0 - o.margin <= minute(t) <= t0] - inside = [(t, v) for t, v in ms if t0 < minute(t) < t1] + h = o.smoothing // 2 # inside: each measurement is the median of the o.smoothing centred on it + inside = [(t, statistics.median(v for _, v in ms[max(0, i - h):i + h + 1]) if h else v) + for i, (t, v) in enumerate(ms) if t0 < minute(t) < t1] post = [v for t, v in ms if t1 <= minute(t) <= t1 + o.margin] if not pre or not (inside if ins else post): continue