improve: S1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-18 16:58:59 +01:00
1 parent fc305697f7
commit a9948d3404
1 file changed
+17 -4
+17 -4
View File
@@ -2,7 +2,7 @@
Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y] Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y]
[--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N] [--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N]
[--labels F [--margin 6h] [--top 5]] [--sensors A,B] [--start ...] [--end ...] [--labels F [--margin 6h] [--top 5] [--smoothing N] [--shorter-out]] [--sensors A,B] [--start ...] [--end ...]
For each column, each sensor is compared with its own level over the --start/--end window (median of the For each column, each sensor is compared with its own level over the --start/--end window (median of the
non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a
@@ -39,7 +39,11 @@ measurements, outside --outside).
the --margin before its start, default 6h; signed, in units and in noises, instant) and the net change (median of the --margin before its start, default 6h; signed, in units and in noises, instant) and the net change (median of
the --margin after its end minus that level), with the number of measurements before and after, and how many of the --margin after its end minus that level), with the number of measurements before and after, and how many of
the measurements inside are more than --k noises from the level before (one: a single point; many: sustained); a label without the measurements inside are more than --k noises from the level before (one: a single point; many: sustained); a label without
a measurement inside (an outage) gives the net change alone, largest first. Labels near the a measurement inside (an outage) gives the net change alone, largest first. --smoothing N: in this size, each
measurement inside is the median of the N centred on it (a long label is not sized by a one-point jump);
--shorter-out: this size leaves out the measurements inside the shorter labels of the file that overlap the label,
before, inside and after (a long label is not sized by a short event inside it or by one under its level before;
a label that only touches it, like the previous day, is kept). Labels near the
edge of --start/--end: widen the window by --margin, or before/after lack measurements. edge of --start/--end: widen the window by --margin, or before/after lack measurements.
Each column is read at the measurements, never averaged per step. Gives values, not a verdict. Each column is read at the measurements, never averaged per step. Gives values, not a verdict.
""" """
@@ -81,6 +85,8 @@ def main():
a.add_argument("--labels", help="label file (one JSON per line): labels covering each instant, deviation inside each label") a.add_argument("--labels", help="label file (one JSON per line): labels covering each instant, deviation inside each label")
a.add_argument("--margin", default="6h", type=duration, help="with --labels: the level before (and after) each label is the median of this duration before its start (after its end)") a.add_argument("--margin", default="6h", type=duration, help="with --labels: the level before (and after) each label is the median of this duration before its start (after its end)")
a.add_argument("--top", type=int, default=5, help="with --labels: sensor-columns listed per label, largest deviation from the level before first") a.add_argument("--top", type=int, default=5, help="with --labels: sensor-columns listed per label, largest deviation from the level before first")
a.add_argument("--smoothing", type=int, default=1, metavar="N", help="with --labels: in the size of a label, each measurement inside is the median of the N centred on it (default 1: none)")
a.add_argument("--shorter-out", action="store_true", help="with --labels: the size of a label leaves out the measurements inside the shorter labels of the file that overlap it (before, inside, after)")
a.add_argument("--sensors", help="comma-separated list (default: all)") a.add_argument("--sensors", help="comma-separated list (default: all)")
a.add_argument("--start", type=parse_time) a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time) a.add_argument("--end", type=parse_time)
@@ -187,7 +193,9 @@ def main():
m = int(o.margin.total_seconds() // 60) m = int(o.margin.total_seconds() // 60)
print(f"{'':6}{'sensor.column':<22}{'before':>7}{'extreme':>11}{'(n)':>8} {'at':<17}{'net':>11}{'(n)':>8}{'after':>6}{'>k':>9}" print(f"{'':6}{'sensor.column':<22}{'before':>7}{'extreme':>11}{'(n)':>8} {'at':<17}{'net':>11}{'(n)':>8}{'after':>6}{'>k':>9}"
f" (then, up to {o.top} per label: extreme inside from the median of the {m} min before the start;" f" (then, up to {o.top} per label: extreme inside from the median of the {m} min before the start;"
f" net: median of the {m} min after the end minus it, alone when no measurement is inside; before, after: their measurements; >k: measurements inside more than {o.k:g} noises from the level before, of those inside; n: noises)") f" net: median of the {m} min after the end minus it, alone when no measurement is inside; before, after: their measurements; >k: measurements inside more than {o.k:g} noises from the level before, of those inside; n: noises"
+ (f"; inside smoothed by the median of {o.smoothing} measurements" if o.smoothing > 1 else "")
+ ("; the measurements inside shorter labels left out" if o.shorter_out else "") + ")")
for n, (t0, t1, e) in enumerate(labels, 1): for n, (t0, t1, e) in enumerate(labels, 1):
ins = [d for d in deviations if t0 < minute(d[0]) < t1] ins = [d for d in deviations if t0 < minute(d[0]) < t1]
head = f"{n:4} {str(e.get('id', '-')):<16}{iso(t0):<18}{iso(t1):<18}{e.get('family', '-')[:21]:<22}" head = f"{n:4} {str(e.get('id', '-')):<16}{iso(t0):<18}{iso(t1):<18}{e.get('family', '-')[:21]:<22}"
@@ -201,9 +209,14 @@ def main():
print(head + f"{len({minute(x[0]) for x in ins}):7} {s + '.' + ABBREV.get(col, col):<14}{d:+8.1f} {iso(t):<17}{over:>5}" print(head + f"{len({minute(x[0]) for x in ins}):7} {s + '.' + ABBREV.get(col, col):<14}{d:+8.1f} {iso(t):<17}{over:>5}"
f"{f'{sum(v >= abs(d) for v in near)}/{len(near)}':>12}") f"{f'{sum(v >= abs(d) for v in near)}/{len(near)}':>12}")
rows = [] # size against the level before the label, and the level after it (net alone without a measurement inside) rows = [] # size against the level before the label, and the level after it (net alone without a measurement inside)
shorter = [(a_, b) for a_, b, _ in labels if b - a_ < t1 - t0 and a_ < t1 and b > t0] if o.shorter_out else []
for (s, col), (ms, noise) in values.items(): for (s, col), (ms, noise) in values.items():
ms = [(t, v) for t, v in ms if t0 - o.margin <= minute(t) <= t1 + o.margin
and not any(a_ < minute(t) < b for a_, b in shorter)]
pre = [v for t, v in ms if t0 - o.margin <= minute(t) <= t0] pre = [v for t, v in ms if t0 - o.margin <= minute(t) <= t0]
inside = [(t, v) for t, v in ms if t0 < minute(t) < t1] h = o.smoothing // 2 # inside: each measurement is the median of the o.smoothing centred on it
inside = [(t, statistics.median(v for _, v in ms[max(0, i - h):i + h + 1]) if h else v)
for i, (t, v) in enumerate(ms) if t0 < minute(t) < t1]
post = [v for t, v in ms if t1 <= minute(t) <= t1 + o.margin] post = [v for t, v in ms if t1 <= minute(t) <= t1 + o.margin]
if not pre or not (inside if ins else post): if not pre or not (inside if ins else post):
continue continue