2026-11-16 12:40:03 +01:00
"""Several stations on one timeline: enclosure temperature, camera pose, corr of the targets, rejections, gaps.
Usage: .venv/bin/python tools/scripts/stations.py data/BREITHORN data/MICHIGAN data/NEBRASKA
[--columns temperature,cam_tilt,cam_rotation,cam_scale,tgt_corr_x,tgt_corr_y] [--step 1h] [--stat median|min|max]
[--table T1/T2|no] [--start ...] [--end ...] [--out FOLDER]
For each station (one station folder each), over the window --start/--end:
- possible columns: temperature (median of the sensor_*_deg_c components, as in metrics.py), cam_pan, cam_tilt,
cam_rotation, cam_scale, cam_sigma0 (raw camera pose), tgt_corr_x, tgt_corr_y (median over the targets of
corr minus each target's own level, accepted measurements, as in metrics.py), rej_targets, rej_refs (number of
rejected targets, of rejected references: the largest over the measurement times of the step);
- prints, per station, the period of the measurements, the cadence (median step) and the number of times; then the **gaps**:
the runs of steps of --step without a measurement or without temperature, with the stations where they are missing (a gap shared by
several stations is on a single line); then the table, one line per step (start of the step, UTC; --stat of the
values of the step; "-": no value), one column per station and per column, tab-separated
(--table T1/T2: only between T1 and T2, "no": not at all; the rest covers the whole window); then, for
each column and each pair of stations, the correlation r of the values and that of the steps (difference from one value to
the next, over the steps where both stations have a value: a common movement shows on the steps, a common
trend on the values), with the number of steps;
- one PNG "stations": one panel per column, one line per station (raw temperature; the other columns minus
their median over the window, so that they overlap), broken over a gap; measurement gaps shaded below the axis, one band
per station.
Never an identifier from metrics/ (SIM_CCID, ip*): only the temperatures are read there. Gives values, not a verdict.
2026-11-16 10:51:34 +01:00
"""
import argparse
import os
import statistics
import sys
import matplotlib
matplotlib . use ( "Agg" )
import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402
2026-11-16 12:40:03 +01:00
from data import POSE , filter_window , is_reference , iso , load , load_camera , load_temperature , parse_time , rejected # noqa: E402
from metrics import cut , runs , targets # noqa: E402
from plot import bound # noqa: E402
from summary import STATS , duration # noqa: E402
2026-11-16 10:51:34 +01:00
2026-11-16 12:40:03 +01:00
COLUMNS = ( "temperature" ,) + tuple ( "cam_" + c for c in POSE ) + ( "tgt_corr_x" , "tgt_corr_y" , "rej_targets" , "rej_refs" )
2026-11-16 10:51:34 +01:00
2026-11-16 12:40:03 +01:00
def read ( folder , start , end ):
"""Station folder -> ({column: [{'t', 'v'}]}, [measurement times])."""
by_sensor = load ( folder )
series = { "temperature" : filter_window ([{ "t" : t , "v" : v } for t , v in load_temperature ( folder )], start , end )}
for p in filter_window ( load_camera ( folder ), start , end ):
2026-11-16 10:51:34 +01:00
for c in POSE :
if p [ c ] is not None :
series . setdefault ( "cam_" + c , []) . append ({ "t" : p [ "t" ], "v" : p [ c ]})
for c in ( "corr_x" , "corr_y" ):
2026-11-16 12:40:03 +01:00
series [ "tgt_" + c ] = targets ( by_sensor , c , start , end )
rej = {} # t -> [rejected targets, rejected references]
for s , ms in by_sensor . items ():
ref = is_reference ( ms )
for m in filter_window ( ms , start , end ):
2026-11-16 10:51:34 +01:00
n = rej . setdefault ( m [ "t" ], [ 0 , 0 ])
2026-11-16 12:40:03 +01:00
n [ ref ] += rejected ( m )
series [ "rej_targets" ] = [{ "t" : t , "v" : float ( n [ 0 ])} for t , n in sorted ( rej . items ())]
2026-11-16 10:51:34 +01:00
series [ "rej_refs" ] = [{ "t" : t , "v" : float ( n [ 1 ])} for t , n in sorted ( rej . items ())]
return series , sorted ( rej )
def correlation ( a , b ):
try :
return f " { statistics . correlation ( a , b ) : +.2f } "
except ( statistics . StatisticsError , ValueError ):
return "-"
def main ():
a = argparse . ArgumentParser ( description = __doc__ , formatter_class = argparse . RawDescriptionHelpFormatter )
2026-11-16 12:40:03 +01:00
a . add_argument ( "folders" , nargs = "+" , help = "station folders, at least two" )
a . add_argument ( "--columns" , default = "temperature,cam_tilt,cam_rotation,cam_scale,tgt_corr_x,tgt_corr_y" )
a . add_argument ( "--step" , default = "1h" , type = duration )
2026-11-16 10:51:34 +01:00
a . add_argument ( "--stat" , default = "median" , choices = STATS )
2026-11-16 12:40:03 +01:00
a . add_argument ( "--table" , default = "" , metavar = "T1/T2|no" , help = "table printed only between T1 and T2 ( \" no \" : not at all)" )
a . add_argument ( "--start" , type = parse_time )
a . add_argument ( "--end" , type = parse_time )
a . add_argument ( "--out" , default = os . path . join ( "out" , "diagnostic" ), help = "folder of the PNG" )
2026-11-16 10:51:34 +01:00
o = a . parse_args ()
2026-11-16 12:40:03 +01:00
columns = o . columns . split ( "," )
if missing := [ c for c in columns if c not in COLUMNS ]:
sys . exit ( f "Unknown columns: { ', ' . join ( missing ) } . Known: { ', ' . join ( COLUMNS ) } ." )
if o . table in ( "" , "no" ):
in_table = lambda t : o . table == "" # noqa: E731
2026-11-16 10:51:34 +01:00
else :
2026-11-16 12:40:03 +01:00
limits = [ parse_time ( x ) for x in o . table . split ( "/" )]
if len ( limits ) != 2 :
sys . exit ( "--table expects T1/T2 or no." )
in_table = lambda t : limits [ 0 ] <= t <= limits [ 1 ] # noqa: E731
names = [ os . path . basename ( os . path . normpath ( d )) for d in o . folders ]
by_station , measured = {}, {}
for n , d in zip ( names , o . folders ):
by_station [ n ], measured [ n ] = read ( d , o . start , o . end )
times = [ p [ "t" ] for s in by_station . values () for c in s . values () for p in c ]
if not times :
sys . exit ( "No value in the requested window." )
t0 = ( o . start or min ( times )) . replace ( minute = 0 , second = 0 , microsecond = 0 )
t1 = o . end or max ( times )
grid = [ t0 + i * o . step for i in range ( int (( t1 - t0 ) / o . step ) + 1 )]
cell = lambda t : t0 + (( t - t0 ) // o . step ) * o . step # noqa: E731
print ( "Stations:" )
for n in names :
ts = measured [ n ]
2026-11-16 10:51:34 +01:00
if not ts :
2026-11-16 12:40:03 +01:00
print ( f " { n } : no measurement in the window" )
2026-11-16 10:51:34 +01:00
continue
cad = statistics . median ( b - a_ for a_ , b in zip ( ts , ts [ 1 :])) if len ( ts ) > 1 else None
2026-11-16 12:40:03 +01:00
print ( f " { n } : measurements { iso ( ts [ 0 ]) } → { iso ( ts [ - 1 ]) } , { len ( ts ) } times, cadence { round ( cad . total_seconds () / 60 ) if cad else '-' } min; "
+ ", " . join ( f " { c } { len ( by_station [ n ] . get ( c , [])) } " for c in columns ))
# gaps: steps without a measurement, without temperature, and the stations concerned
with_meas = { n : { cell ( t ) for t in measured [ n ]} for n in names }
with_temp = { n : { cell ( p [ "t" ]) for p in by_station [ n ][ "temperature" ]} for n in names }
print ( f " \n Gaps (steps of { int ( o . step . total_seconds () // 60 ) } min without a measurement or without temperature), with the stations where they are missing:" )
keys = {}
for k in grid :
m = tuple ( n for n in names if k not in with_meas [ n ])
t = tuple ( n for n in names if k not in with_temp [ n ])
2026-11-16 10:51:34 +01:00
if m or t :
2026-11-16 12:40:03 +01:00
keys . setdefault (( m , t ), []) . append ( k )
lines = []
for ( m , t ), ks in keys . items ():
for d , f , nb in runs ( ks , o . step ):
lines . append (( d , f , nb , m , t ))
for d , f , nb , m , t in sorted ( lines ):
print ( f " { iso ( d ) } → { iso ( f ) } ( { nb } steps): no measurement { ',' . join ( m ) or '-' } ; no temperature { ',' . join ( t ) or '-' } " )
if not lines :
print ( " (none)" )
2026-11-16 10:51:34 +01:00
stat = STATS [ o . stat ]
2026-11-16 12:40:03 +01:00
table = {} # (station, column) -> {step: value}
for n in names :
for c in columns :
cells = {}
for p in by_station [ n ] . get ( c , []):
cells . setdefault ( cell ( p [ "t" ]), []) . append ( p [ "v" ])
table [ n , c ] = { k : stat ( v ) for k , v in cells . items ()}
header = [ f " { n } : { c } " for n in names for c in columns ]
print ( f " \n { o . stat } per step of { int ( o . step . total_seconds () // 60 ) } min (start of the step, UTC); \" - \" : no value"
+ ( f " (table: { o . table } )" if o . table else "" ) + ":" )
if o . table != "no" :
print ( " \t " . join ([ "time" ] + header ))
for k in grid :
if in_table ( k ):
v = [ table [ n , c ] . get ( k ) for n in names for c in columns ]
2026-11-16 10:51:34 +01:00
if any ( x is not None for x in v ):
print ( " \t " . join ([ iso ( k )] + [ "-" if x is None else f " { x : .6g } " for x in v ]))
2026-11-16 12:40:03 +01:00
print ( " \n Correlation between stations, r of the values, then r of the steps from one value to the next (n: steps where both have a value):" )
for c in columns :
for i , s1 in enumerate ( names ):
for s2 in names [ i + 1 :]:
2026-11-16 10:51:34 +01:00
ks = sorted ( set ( table [ s1 , c ]) & set ( table [ s2 , c ]))
v1 , v2 = [ table [ s1 , c ][ k ] for k in ks ], [ table [ s2 , c ][ k ] for k in ks ]
d1 , d2 = [ b - a_ for a_ , b in zip ( v1 , v1 [ 1 :])], [ b - a_ for a_ , b in zip ( v2 , v2 [ 1 :])]
2026-11-16 12:40:03 +01:00
print ( f " { c : 12 } { s1 } - { s2 } : r values { correlation ( v1 , v2 ) } , r steps { correlation ( d1 , d2 ) } (n = { len ( ks ) } )" )
2026-11-16 10:51:34 +01:00
2026-11-16 12:40:03 +01:00
fig , axes = plt . subplots ( len ( columns ) + 1 , 1 , sharex = True , figsize = ( 14 , 2.4 * ( len ( columns ) + 1 )), squeeze = False ,
layout = "constrained" , gridspec_kw = { "height_ratios" : [ 3 ] * len ( columns ) + [ 0.3 * len ( names )]})
for ax , c in zip ( axes [: - 1 , 0 ], columns ):
for i , n in enumerate ( names ):
s = by_station [ n ] . get ( c , [])
2026-11-16 10:51:34 +01:00
if not s :
continue
med = 0 if c == "temperature" else statistics . median ( p [ "v" ] for p in s )
2026-11-16 12:40:03 +01:00
ts , vs = cut ([{ "t" : p [ "t" ], "v" : p [ "v" ] - med } for p in s ])
2026-11-16 10:51:34 +01:00
ax . plot ( ts , vs , ".-" , ms = 2 , lw = 0.8 , color = f "C { i % 10 } " , label = n )
2026-11-16 12:40:03 +01:00
ax . set_ylabel ( c if c == "temperature" else f " { c } \n minus its median" , fontsize = 8 )
2026-11-16 10:51:34 +01:00
ax . grid ( alpha = 0.3 )
ax . legend ( loc = "upper left" , bbox_to_anchor = ( 1.0 , 1.0 ), fontsize = 8 )
ax = axes [ - 1 , 0 ]
2026-11-16 12:40:03 +01:00
for i , n in enumerate ( names ):
for k in grid :
if k not in with_meas [ n ]:
ax . axvspan ( k , k + o . step , ymin = i / len ( names ), ymax = ( i + 1 ) / len ( names ), color = f "C { i % 10 } " , lw = 0 )
ax . set_yticks ([( i + 0.5 ) / len ( names ) for i in range ( len ( names ))], names , fontsize = 7 )
2026-11-16 10:51:34 +01:00
ax . set_ylim ( 0 , 1 )
2026-11-16 12:40:03 +01:00
ax . set_ylabel ( "no measurement" , fontsize = 8 )
ax . set_xlim ( grid [ 0 ], grid [ - 1 ] + o . step )
2026-11-16 10:51:34 +01:00
ax . xaxis . set_major_formatter ( mdates . DateFormatter ( " %d /%m \n %H:%MZ" ))
2026-11-16 12:40:03 +01:00
fig . suptitle ( f " { ', ' . join ( names ) } : same timeline (UTC times)" )
os . makedirs ( o . out , exist_ok = True )
path = os . path . join ( o . out , f " { '-' . join ( names ) } _stations_ { bound ( o . start , 'start' ) } _ { bound ( o . end , 'end' ) } .png" )
fig . savefig ( path , dpi = 100 , bbox_inches = "tight" )
print ( f " \n { path } " )
2026-11-16 10:51:34 +01:00
if __name__ == "__main__" :
main ()