improve: S2
ranges.py --labels: per label, the size of the label against the level before it (--margin, default 6h): extreme inside and net change after the end, in units and noises, for the --top sensor-columns (default 5); net alone across a label without measurement inside Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
72ff92126c
commit
af6cf23f94
2 files changed
+36
-9
No files matched your search
+34
-7
@@ -2,7 +2,7 @@
|
||||
|
||||
Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y]
|
||||
[--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N]
|
||||
[--labels F] [--sensors A,B] [--start ...] [--end ...]
|
||||
[--labels F [--margin 6h] [--top 5]] [--sensors A,B] [--start ...] [--end ...]
|
||||
|
||||
For each column, each sensor is compared with its own level over the --start/--end window (median of the
|
||||
non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a
|
||||
@@ -31,7 +31,12 @@ measurements, outside --outside).
|
||||
--instants ends with the numbers of the labels that cover the instant ("labels: none" otherwise); then, per label,
|
||||
the measurements strictly inside it (start < instant < end, the instant to the minute: a measurement at a bound is
|
||||
the point before or after), the largest deviation inside from the level (sensor, column, signed, in noises, instant)
|
||||
and the number of instants inside where a sensor is more than --k noises from it.
|
||||
and the number of instants inside where a sensor is more than --k noises from it; then the size of the label, up
|
||||
to --top sensor-columns (default 5), largest first: the extreme inside from the level before the label (median of
|
||||
the --margin before its start, default 6h; signed, in units and in noises, instant) and the net change (median of
|
||||
the --margin after its end minus that level), with the number of measurements before and after; a label without
|
||||
a measurement inside (an outage) gives the net change alone, largest first. Labels near the
|
||||
edge of --start/--end: widen the window by --margin, or before/after lack measurements.
|
||||
Each column is read at the measurements, never averaged per step. Gives values, not a verdict.
|
||||
"""
|
||||
import argparse
|
||||
@@ -70,6 +75,8 @@ def main():
|
||||
help="N (measurements on either side) or duration (6h, 1d): local level, median of the neighbours without the measurement")
|
||||
a.add_argument("--instants", type=int, help="N: instants where at least N sensors are off level, with rejections and references")
|
||||
a.add_argument("--labels", help="label file (one JSON per line): labels covering each instant, deviation inside each label")
|
||||
a.add_argument("--margin", default="6h", type=duration, help="with --labels: the level before (and after) each label is the median of this duration before its start (after its end)")
|
||||
a.add_argument("--top", type=int, default=5, help="with --labels: sensor-columns listed per label, largest deviation from the level before first")
|
||||
a.add_argument("--sensors", help="comma-separated list (default: all)")
|
||||
a.add_argument("--start", type=parse_time)
|
||||
a.add_argument("--end", type=parse_time)
|
||||
@@ -94,6 +101,7 @@ def main():
|
||||
roll = (f"{o.rolling} measurements" if isinstance(o.rolling, int) else f"{int(o.rolling.total_seconds() // 60)} min") if o.rolling is not None else ""
|
||||
by_instant = {} # iso(t) -> ["sensor.col+deviation"] (--instants)
|
||||
deviations = [] # (t, sensor, col, deviation in noises) of every measurement (--labels)
|
||||
values = {} # (sensor, col) -> (measurements (t, v), noise) (--labels: size against the level before each label)
|
||||
|
||||
for col in columns:
|
||||
print(f"\n== {col}: level, noise, largest jump from one measurement to the next ==")
|
||||
@@ -119,6 +127,7 @@ def main():
|
||||
+ (f" (rejected: {', '.join(flagged)})" if flagged else ""))
|
||||
if not noise:
|
||||
continue
|
||||
values[s, col] = (ms, noise)
|
||||
lv = local_levels(ms, o.rolling) if o.rolling is not None else [lvl] * len(ms)
|
||||
group = [] # (t, deviation from the level)
|
||||
for (t, v), n in zip(ms + [(None, 0)], lv + [None]):
|
||||
@@ -169,16 +178,34 @@ def main():
|
||||
print(f"\nPer label ({o.labels}): measurements strictly inside, largest deviation from the level"
|
||||
f"{' rolling ±' + roll if roll else ''} ({', '.join(columns)}), instants with a sensor more than {o.k:g} noises")
|
||||
print(f"{'n':>4} {'id':<16}{'start':<18}{'end':<18}{'family':<22}{'inside':>7} largest deviation (noises, at) instants > k")
|
||||
m = int(o.margin.total_seconds() // 60)
|
||||
print(f"{'':6}{'sensor.column':<22}{'before':>7}{'extreme':>11}{'(n)':>8} {'at':<17}{'net':>11}{'(n)':>8}{'after':>6}"
|
||||
f" (then, up to {o.top} per label: extreme inside from the median of the {m} min before the start;"
|
||||
f" net: median of the {m} min after the end minus it, alone when no measurement is inside; before, after: their measurements; n: noises)")
|
||||
for n, (t0, t1, e) in enumerate(labels, 1):
|
||||
ins = [d for d in deviations if t0 < minute(d[0]) < t1]
|
||||
head = f"{n:4} {str(e.get('id', '-')):<16}{iso(t0):<18}{iso(t1):<18}{e.get('family', '-')[:21]:<22}"
|
||||
if not ins:
|
||||
print(head + f"{0:7} (no measurement inside)")
|
||||
continue
|
||||
t, s, col, d = max(ins, key=lambda x: abs(x[3]))
|
||||
over = len({minute(x[0]) for x in ins if abs(x[3]) > o.k})
|
||||
print(head + f"{len({minute(x[0]) for x in ins}):7} {s + '.' + ABBREV.get(col, col):<14}{d:+8.1f} {iso(t):<17}{over:>5}")
|
||||
|
||||
else:
|
||||
t, s, col, d = max(ins, key=lambda x: abs(x[3]))
|
||||
over = len({minute(x[0]) for x in ins if abs(x[3]) > o.k})
|
||||
print(head + f"{len({minute(x[0]) for x in ins}):7} {s + '.' + ABBREV.get(col, col):<14}{d:+8.1f} {iso(t):<17}{over:>5}")
|
||||
rows = [] # size against the level before the label, and the level after it (net alone without a measurement inside)
|
||||
for (s, col), (ms, noise) in values.items():
|
||||
pre = [v for t, v in ms if t0 - o.margin <= minute(t) <= t0]
|
||||
inside = [(t, v) for t, v in ms if t0 < minute(t) < t1]
|
||||
post = [v for t, v in ms if t1 <= minute(t) <= t1 + o.margin]
|
||||
if not pre or not (inside if ins else post):
|
||||
continue
|
||||
before = statistics.median(pre)
|
||||
t, d = max(((t, v - before) for t, v in inside), key=lambda x: abs(x[1])) if inside else (None, None)
|
||||
net = statistics.median(post) - before if post else None
|
||||
rows.append((abs(d if ins else net) / noise, s, col, d, noise, t, len(pre), net, len(post)))
|
||||
for _, s, col, d, noise, t, n_pre, net, n_post in sorted(rows, key=lambda r: -r[0])[:o.top]:
|
||||
ext = f"{d:+11.4g}{d / noise:+8.1f} {iso(t):<17}" if d is not None else f"{'-':>11}{'-':>8} {'-':<17}"
|
||||
after = f"{net:+11.4g}{net / noise:+8.1f}{n_post:6}" if net is not None else f"{'-':>11}{'-':>8}{0:6}"
|
||||
print(f"{'':6}{s + '.' + col:<22}{n_pre:7}{ext}{after}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user