2026-11-16 12:40:03 +01:00
"""Dated ranges per sensor and per column, at the native step of the measurements, for the diagnostic agent.
Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y]
[--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N]
2026-11-18 13:57:31 +01:00
[--labels F [--margin 6h] [--top 5]] [--sensors A,B] [--start ...] [--end ...]
2026-11-16 12:40:03 +01:00
For each column, each sensor is compared with its own level over the --start/--end window (median of the
non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a
long episode skews the median) and with its own noise (median of the differences between two consecutive non-empty
measurements, outside --outside).
1. One table per column: level, noise, and the largest jump from one measurement to the next (value, in noises,
the two instants).
2. The ranges where the column deviates from its level by more than --k noises: start and end (first and last
off-level measurement), number of off-level measurements, extreme value (signed deviation from the level, in units
and in noises) and its instant. Two ranges in the same direction less than --merge apart (default 1h) make
only one (a deviation downwards then upwards makes two ranges); rejected measurements (empty column) do not cut
a range. Column "rejected": how many of the measurements of the range are rejected by the station (error_value
above the threshold), and "rejected extreme" if the extreme is; a jump with a rejected end says so.
corr_x, corr_y of a rejected measurement are empty, references included (data.load). A range that touches the edge
of the window is dated at the edge (the level is the window's: a lasting plateau can make the other half the "range").
With --min-measurements N, only the ranges of at least N off-level measurements are printed (their total number too).
With --rolling N (an integer: N non-empty measurements on either side) or --rolling DURATION (6h, 1d: the
measurements less than DURATION before and after), the level of each measurement is the median of its neighbours,
without it (local level): a long drift or a plateau follows the level, only the deviations shorter than the window
remain. The "level" column of table 1 stays the median of the window; the noise does not change.
3. With --instants N: the instants where at least N sensors are off level (on one of the columns), with, for
each, the sensors and the deviation in noises (cx corr_x, cy corr_y, ev error_value, otherwise the column name),
the sensors rejected at that instant (among --sensors: removing the sensors that are always rejected shortens the
list), and each reference of the station, in --sensors or not: error_value ("r" if rejected) / inlier ("-" empty).
2026-11-18 12:51:30 +01:00
4. With --labels F (one JSON per line, start, end, family, id if any; numbered in the order of the file): each line of
--instants ends with the numbers of the labels that cover the instant ("labels: none" otherwise); then, per label,
the measurements strictly inside it (start < instant < end, the instant to the minute: a measurement at a bound is
the point before or after), the largest deviation inside from the level (sensor, column, signed, in noises, instant)
2026-11-18 13:57:31 +01:00
and the number of instants inside where a sensor is more than --k noises from it; then the size of the label, up
to --top sensor-columns (default 5), largest first: the extreme inside from the level before the label (median of
the --margin before its start, default 6h; signed, in units and in noises, instant) and the net change (median of
2026-11-18 14:20:15 +01:00
the --margin after its end minus that level), with the number of measurements before and after, and how many of
the measurements inside are more than --k noises from the level before (one: a single point; many: sustained); a label without
2026-11-18 13:57:31 +01:00
a measurement inside (an outage) gives the net change alone, largest first. Labels near the
edge of --start/--end: widen the window by --margin, or before/after lack measurements.
2026-11-16 12:40:03 +01:00
Each column is read at the measurements, never averaged per step. Gives values, not a verdict.
"""
import argparse
import bisect
2026-11-18 12:51:30 +01:00
import json
2026-11-16 12:40:03 +01:00
import statistics
import sys
2026-11-16 15:48:42 +01:00
from common import COLUMNS , check , duration , outside , own_noise , sensors
from data import filter_window , is_reference , iso , parse_time , rejected
2026-11-16 12:40:03 +01:00
ABBREV = { "corr_x" : "cx" , "corr_y" : "cy" , "error_value" : "ev" }
def local_levels ( ms , g ):
"""Local level of each measurement (t, v): median of the non-empty neighbours, without it; g an integer
(measurements on either side) or a timedelta (measurements less than g away); None without a neighbour."""
ts , vs = [ t for t , _ in ms ], [ v for _ , v in ms ]
out = []
for i , t in enumerate ( ts ):
lo , hi = ( max ( 0 , i - g ), i + g + 1 ) if isinstance ( g , int ) else ( bisect . bisect_left ( ts , t - g ), bisect . bisect_right ( ts , t + g ))
w = vs [ lo : i ] + vs [ i + 1 : hi ]
out . append ( statistics . median ( w ) if w else None )
return out
def main ():
a = argparse . ArgumentParser ( description = __doc__ , formatter_class = argparse . RawDescriptionHelpFormatter )
a . add_argument ( "folder" )
a . add_argument ( "--columns" , default = "error_value,corr_x,corr_y" )
a . add_argument ( "--k" , type = float , default = 5 , help = "deviation from the level that counts, in the sensor's own noises" )
a . add_argument ( "--merge" , default = "1h" , type = duration , help = "two ranges closer than this make only one" )
a . add_argument ( "--min-measurements" , type = int , default = 1 , help = "prints only the ranges of at least N off-level measurements" )
a . add_argument ( "--outside" , help = "T1/T2: level and noise are computed outside this moment (ranges are searched everywhere)" )
a . add_argument ( "--rolling" , type = lambda x : int ( x ) if x . isdigit () else duration ( x ),
help = "N (measurements on either side) or duration (6h, 1d): local level, median of the neighbours without the measurement" )
a . add_argument ( "--instants" , type = int , help = "N: instants where at least N sensors are off level, with rejections and references" )
2026-11-18 12:51:30 +01:00
a . add_argument ( "--labels" , help = "label file (one JSON per line): labels covering each instant, deviation inside each label" )
2026-11-18 13:57:31 +01:00
a . add_argument ( "--margin" , default = "6h" , type = duration , help = "with --labels: the level before (and after) each label is the median of this duration before its start (after its end)" )
a . add_argument ( "--top" , type = int , default = 5 , help = "with --labels: sensor-columns listed per label, largest deviation from the level before first" )
2026-11-16 12:40:03 +01:00
a . add_argument ( "--sensors" , help = "comma-separated list (default: all)" )
a . add_argument ( "--start" , type = parse_time )
a . add_argument ( "--end" , type = parse_time )
o = a . parse_args ()
2026-11-18 12:51:30 +01:00
labels = []
if o . labels :
for line in open ( o . labels ):
if line . strip ():
e = json . loads ( line )
labels . append (( parse_time ( e [ "start" ]), parse_time ( e [ "end" ]), e ))
minute = lambda t : t . replace ( second = 0 , microsecond = 0 )
covering = lambda t : [ str ( n ) for n , ( t0 , t1 , _ ) in enumerate ( labels , 1 ) if t0 < minute ( t ) < t1 ]
2026-11-16 12:40:03 +01:00
2026-11-16 15:48:42 +01:00
by , names = sensors ( o . folder , o . sensors )
2026-11-16 12:40:03 +01:00
columns = o . columns . split ( "," )
2026-11-16 15:48:42 +01:00
kept = outside ( o . outside )
check ( "sensors" , names , by )
check ( "columns" , columns , COLUMNS )
2026-11-16 12:40:03 +01:00
series = { s : filter_window ( by [ s ], o . start , o . end ) for s in names }
2026-11-18 14:19:36 +01:00
w = max ( 9 , max ( len ( s ) for s in names ) + 1 ) # sensor column: a long name never runs into the next column
2026-11-16 12:40:03 +01:00
if not any ( series . values ()):
sys . exit ( "No measurement in the requested window." )
roll = ( f " { o . rolling } measurements" if isinstance ( o . rolling , int ) else f " { int ( o . rolling . total_seconds () // 60 ) } min" ) if o . rolling is not None else ""
by_instant = {} # iso(t) -> ["sensor.col+deviation"] (--instants)
2026-11-18 12:51:30 +01:00
deviations = [] # (t, sensor, col, deviation in noises) of every measurement (--labels)
2026-11-18 13:57:31 +01:00
values = {} # (sensor, col) -> (measurements (t, v), noise) (--labels: size against the level before each label)
2026-11-16 12:40:03 +01:00
for col in columns :
print ( f " \n == { col } : level, noise, largest jump from one measurement to the next ==" )
2026-11-18 14:19:36 +01:00
print ( f " { 'sensor' :{ w }}{ 'n' : >5 }{ 'level' : >11 }{ 'noise' : >11 }{ 'jump' : >11 }{ '(noises)' : >9 } between" )
2026-11-16 12:40:03 +01:00
ranges = []
for s in names :
ms = [( m [ "t" ], m [ col ]) for m in series [ s ] if m [ col ] is not None ]
rej = { m [ "t" ] for m in series [ s ] if rejected ( m )}
if len ( ms ) < 2 :
2026-11-18 14:19:36 +01:00
print ( f " { s :{ w }}{ len ( ms ) : 5 } (fewer than 2 values)" )
2026-11-16 12:40:03 +01:00
continue
2026-11-16 15:48:42 +01:00
ref = [ x for x in ms if kept ( x [ 0 ])]
2026-11-16 12:40:03 +01:00
if len ( ref ) < 2 :
2026-11-18 14:19:36 +01:00
print ( f " { s :{ w }}{ len ( ms ) : 5 } (fewer than 2 values outside --outside)" )
2026-11-16 12:40:03 +01:00
continue
lvl = statistics . median ( v for _ , v in ref )
jumps = [( b [ 1 ] - a_ [ 1 ], a_ [ 0 ], b [ 0 ]) for a_ , b in zip ( ms , ms [ 1 :])]
noise = own_noise ([ v for _ , v in ref ])
d , t1 , t2 = max ( jumps , key = lambda x : abs ( x [ 0 ]))
in_noises = f " { d / noise : 9.1f } " if noise else f " { '-' : >9 } "
flagged = [ iso ( x ) for x in ( t1 , t2 ) if x in rej ]
2026-11-18 14:19:36 +01:00
print ( f " { s :{ w }}{ len ( ms ) : 5 }{ lvl : 11.4g }{ noise : 11.3g }{ d : +11.4g }{ in_noises } { iso ( t1 ) } and { iso ( t2 ) } "
2026-11-16 12:40:03 +01:00
+ ( f " (rejected: { ', ' . join ( flagged ) } )" if flagged else "" ))
if not noise :
continue
2026-11-18 13:57:31 +01:00
values [ s , col ] = ( ms , noise )
2026-11-16 12:40:03 +01:00
lv = local_levels ( ms , o . rolling ) if o . rolling is not None else [ lvl ] * len ( ms )
group = [] # (t, deviation from the level)
for ( t , v ), n in zip ( ms + [( None , 0 )], lv + [ None ]):
2026-11-18 12:51:30 +01:00
if labels and t is not None and n is not None :
deviations . append (( t , s , col , ( v - n ) / noise ))
2026-11-16 12:40:03 +01:00
if t is not None and n is not None and abs ( v - n ) > o . k * noise :
if group and ( t - group [ - 1 ][ 0 ] > o . merge or ( v > n ) != ( group [ - 1 ][ 1 ] > 0 )): # same direction only
ranges . append (( s , group , noise , rej ))
group = []
group . append (( t , v - n ))
by_instant . setdefault ( iso ( t ), {}) . setdefault ( s , []) . append ( f " { ABBREV . get ( col , col ) }{ ( v - n ) / noise : +.0f } " )
elif t is None and group :
ranges . append (( s , group , noise , rej ))
total = len ( ranges )
ranges = [ p for p in ranges if len ( p [ 1 ]) >= o . min_measurements ]
2026-11-16 15:48:42 +01:00
print ( f " \n { col } : ranges more than { o . k : g } noises from the level { ' rolling ±' + roll + ' (median of the neighbours)' if roll else '' }{ ' (level outside ' + o . outside + ')' if o . outside and not roll else '' }{ ' (noise outside ' + o . outside + ')' if o . outside and roll else '' } (merge { int ( o . merge . total_seconds () // 60 ) } min)"
2026-11-16 12:40:03 +01:00
+ ( f "; { len ( ranges ) } of at least { o . min_measurements } measurements out of { total } " if o . min_measurements > 1 else "" ))
2026-11-18 14:19:36 +01:00
print ( f " { 'sensor' :{ w }}{ 'start' : <18 }{ 'end' : <18 }{ 'measurements' : >13 }{ 'extreme' : >11 }{ '(noises)' : >9 } { 'at' : <17 }{ 'rejected' : >9 } " )
2026-11-16 12:40:03 +01:00
for s , g , noise , rej in ranges :
t , d = max ( g , key = lambda x : abs ( x [ 1 ]))
2026-11-18 14:19:36 +01:00
print ( f " { s :{ w }}{ iso ( g [ 0 ][ 0 ]) : <18 }{ iso ( g [ - 1 ][ 0 ]) : <18 }{ len ( g ) : 13 }{ d : +11.4g }{ d / noise : 9.1f } { iso ( t ) : <17 } "
2026-11-16 12:40:03 +01:00
f " { sum ( x [ 0 ] in rej for x in g ) : 9 }{ ' rejected extreme' if t in rej else '' } " )
if not ranges :
print ( "(none)" )
if o . instants is not None :
refs = sorted ( s for s in by if is_reference ( by [ s ]))
rej_t , ref_t = {}, { s : {} for s in refs }
for s in set ( names ) | set ( refs ):
for m in filter_window ( by [ s ], o . start , o . end ):
if rejected ( m ) and s in names :
rej_t . setdefault ( iso ( m [ "t" ]), []) . append ( s )
if s in ref_t :
ref_t [ s ][ iso ( m [ "t" ])] = m
f = lambda v , fmt : "-" if v is None else format ( v , fmt )
rows = [( t , c ) for t , c in sorted ( by_instant . items ()) if len ( c ) >= o . instants ]
print ( f " \n Instants where at least { o . instants } sensors are more than { o . k : g } noises from the level { ' rolling ±' + roll if roll else '' } "
f "( { ', ' . join ( columns ) } ): { len ( rows ) } of { len ( by_instant ) } instants with at least one sensor off level" )
2026-11-18 12:51:30 +01:00
print ( "instant n sensors off level (deviation in noises) | rejected | references: error_value (r rejected) / inlier"
+ ( " | labels covering" if labels else "" ))
2026-11-16 12:40:03 +01:00
for t , c in rows :
r = " " . join ( f " { s } { f ( ref_t [ s ][ t ][ 'error_value' ], '.3g' ) }{ 'r' if rejected ( ref_t [ s ][ t ]) else '' } / { f ( ref_t [ s ][ t ][ 'inlier' ], 'g' ) } "
for s in refs if t in ref_t [ s ])
2026-11-18 12:51:30 +01:00
cov = f " | labels: { ',' . join ( covering ( parse_time ( t ))) or 'none' } " if labels else ""
print ( f " { t } { len ( c ) : 3 } { ' ' . join ( s + '.' + ',' . join ( v ) for s , v in c . items ()) } | { ',' . join ( rej_t . get ( t , [])) or '-' } | { r or '-' }{ cov } " )
if labels :
print ( f " \n Per label ( { o . labels } ): measurements strictly inside, largest deviation from the level"
f " { ' rolling ±' + roll if roll else '' } ( { ', ' . join ( columns ) } ), instants with a sensor more than { o . k : g } noises" )
print ( f " { 'n' : >4 } { 'id' : <16 }{ 'start' : <18 }{ 'end' : <18 }{ 'family' : <22 }{ 'inside' : >7 } largest deviation (noises, at) instants > k" )
2026-11-18 13:57:31 +01:00
m = int ( o . margin . total_seconds () // 60 )
2026-11-18 14:20:15 +01:00
print ( f " { '' : 6 }{ 'sensor.column' : <22 }{ 'before' : >7 }{ 'extreme' : >11 }{ '(n)' : >8 } { 'at' : <17 }{ 'net' : >11 }{ '(n)' : >8 }{ 'after' : >6 }{ '>k' : >9 } "
2026-11-18 13:57:31 +01:00
f " (then, up to { o . top } per label: extreme inside from the median of the { m } min before the start;"
2026-11-18 14:20:15 +01:00
f " net: median of the { m } min after the end minus it, alone when no measurement is inside; before, after: their measurements; >k: measurements inside more than { o . k : g } noises from the level before, of those inside; n: noises)" )
2026-11-18 12:51:30 +01:00
for n , ( t0 , t1 , e ) in enumerate ( labels , 1 ):
ins = [ d for d in deviations if t0 < minute ( d [ 0 ]) < t1 ]
head = f " { n : 4 } { str ( e . get ( 'id' , '-' )) : <16 }{ iso ( t0 ) : <18 }{ iso ( t1 ) : <18 }{ e . get ( 'family' , '-' )[: 21 ] : <22 } "
if not ins :
print ( head + f " { 0 : 7 } (no measurement inside)" )
2026-11-18 13:57:31 +01:00
else :
t , s , col , d = max ( ins , key = lambda x : abs ( x [ 3 ]))
over = len ({ minute ( x [ 0 ]) for x in ins if abs ( x [ 3 ]) > o . k })
print ( head + f " { len ({ minute ( x [ 0 ]) for x in ins }) : 7 } { s + '.' + ABBREV . get ( col , col ) : <14 }{ d : +8.1f } { iso ( t ) : <17 }{ over : >5 } " )
rows = [] # size against the level before the label, and the level after it (net alone without a measurement inside)
for ( s , col ), ( ms , noise ) in values . items ():
pre = [ v for t , v in ms if t0 - o . margin <= minute ( t ) <= t0 ]
inside = [( t , v ) for t , v in ms if t0 < minute ( t ) < t1 ]
post = [ v for t , v in ms if t1 <= minute ( t ) <= t1 + o . margin ]
if not pre or not ( inside if ins else post ):
continue
before = statistics . median ( pre )
t , d = max ((( t , v - before ) for t , v in inside ), key = lambda x : abs ( x [ 1 ])) if inside else ( None , None )
net = statistics . median ( post ) - before if post else None
2026-11-18 14:20:15 +01:00
beyond = f " { sum ( abs ( v - before ) > o . k * noise for _ , v in inside ) } / { len ( inside ) } " if inside else "-"
rows . append (( abs ( d if ins else net ) / noise , s , col , d , noise , t , len ( pre ), net , len ( post ), beyond ))
for _ , s , col , d , noise , t , n_pre , net , n_post , beyond in sorted ( rows , key = lambda r : - r [ 0 ])[: o . top ]:
2026-11-18 13:57:31 +01:00
ext = f " { d : +11.4g }{ d / noise : +8.1f } { iso ( t ) : <17 } " if d is not None else f " { '-' : >11 }{ '-' : >8 } { '-' : <17 } "
after = f " { net : +11.4g }{ net / noise : +8.1f }{ n_post : 6 } " if net is not None else f " { '-' : >11 }{ '-' : >8 }{ 0 : 6 } "
2026-11-18 14:20:15 +01:00
print ( f " { '' : 6 }{ s + '.' + col : <22 }{ n_pre : 7 }{ ext }{ after }{ beyond : >9 } " )
2026-11-16 12:40:03 +01:00
if __name__ == "__main__" :
main ()