"""Dated ranges per sensor and per column, at the native step of the measurements, for the diagnostic agent. Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y] [--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N] [--sensors A,B] [--start ...] [--end ...] For each column, each sensor is compared with its own level over the --start/--end window (median of the non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a long episode skews the median) and with its own noise (median of the differences between two consecutive non-empty measurements, outside --outside). 1. One table per column: level, noise, and the largest jump from one measurement to the next (value, in noises, the two instants). 2. The ranges where the column deviates from its level by more than --k noises: start and end (first and last off-level measurement), number of off-level measurements, extreme value (signed deviation from the level, in units and in noises) and its instant. Two ranges in the same direction less than --merge apart (default 1h) make only one (a deviation downwards then upwards makes two ranges); rejected measurements (empty column) do not cut a range. Column "rejected": how many of the measurements of the range are rejected by the station (error_value above the threshold), and "rejected extreme" if the extreme is; a jump with a rejected end says so. corr_x, corr_y of a rejected measurement are empty, references included (data.load). A range that touches the edge of the window is dated at the edge (the level is the window's: a lasting plateau can make the other half the "range"). With --min-measurements N, only the ranges of at least N off-level measurements are printed (their total number too). With --rolling N (an integer: N non-empty measurements on either side) or --rolling DURATION (6h, 1d: the measurements less than DURATION before and after), the level of each measurement is the median of its neighbours, without it (local level): a long drift or a plateau follows the level, only the deviations shorter than the window remain. The "level" column of table 1 stays the median of the window; the noise does not change. 3. With --instants N: the instants where at least N sensors are off level (on one of the columns), with, for each, the sensors and the deviation in noises (cx corr_x, cy corr_y, ev error_value, otherwise the column name), the sensors rejected at that instant (among --sensors: removing the sensors that are always rejected shortens the list), and each reference of the station, in --sensors or not: error_value ("r" if rejected) / inlier ("-" empty). Each column is read at the measurements, never averaged per step. Gives values, not a verdict. """ import argparse import bisect import statistics import sys from common import COLUMNS, check, duration, outside, own_noise, sensors from data import filter_window, is_reference, iso, parse_time, rejected ABBREV = {"corr_x": "cx", "corr_y": "cy", "error_value": "ev"} def local_levels(ms, g): """Local level of each measurement (t, v): median of the non-empty neighbours, without it; g an integer (measurements on either side) or a timedelta (measurements less than g away); None without a neighbour.""" ts, vs = [t for t, _ in ms], [v for _, v in ms] out = [] for i, t in enumerate(ts): lo, hi = (max(0, i - g), i + g + 1) if isinstance(g, int) else (bisect.bisect_left(ts, t - g), bisect.bisect_right(ts, t + g)) w = vs[lo:i] + vs[i + 1:hi] out.append(statistics.median(w) if w else None) return out def main(): a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a.add_argument("folder") a.add_argument("--columns", default="error_value,corr_x,corr_y") a.add_argument("--k", type=float, default=5, help="deviation from the level that counts, in the sensor's own noises") a.add_argument("--merge", default="1h", type=duration, help="two ranges closer than this make only one") a.add_argument("--min-measurements", type=int, default=1, help="prints only the ranges of at least N off-level measurements") a.add_argument("--outside", help="T1/T2: level and noise are computed outside this moment (ranges are searched everywhere)") a.add_argument("--rolling", type=lambda x: int(x) if x.isdigit() else duration(x), help="N (measurements on either side) or duration (6h, 1d): local level, median of the neighbours without the measurement") a.add_argument("--instants", type=int, help="N: instants where at least N sensors are off level, with rejections and references") a.add_argument("--sensors", help="comma-separated list (default: all)") a.add_argument("--start", type=parse_time) a.add_argument("--end", type=parse_time) o = a.parse_args() by, names = sensors(o.folder, o.sensors) columns = o.columns.split(",") kept = outside(o.outside) check("sensors", names, by) check("columns", columns, COLUMNS) series = {s: filter_window(by[s], o.start, o.end) for s in names} if not any(series.values()): sys.exit("No measurement in the requested window.") roll = (f"{o.rolling} measurements" if isinstance(o.rolling, int) else f"{int(o.rolling.total_seconds() // 60)} min") if o.rolling is not None else "" by_instant = {} # iso(t) -> ["sensor.col+deviation"] (--instants) for col in columns: print(f"\n== {col}: level, noise, largest jump from one measurement to the next ==") print(f"{'sensor':9}{'n':>5}{'level':>10}{'noise':>10}{'jump':>10}{'(noises)':>9} between") ranges = [] for s in names: ms = [(m["t"], m[col]) for m in series[s] if m[col] is not None] rej = {m["t"] for m in series[s] if rejected(m)} if len(ms) < 2: print(f"{s:9}{len(ms):5} (fewer than 2 values)") continue ref = [x for x in ms if kept(x[0])] if len(ref) < 2: print(f"{s:9}{len(ms):5} (fewer than 2 values outside --outside)") continue lvl = statistics.median(v for _, v in ref) jumps = [(b[1] - a_[1], a_[0], b[0]) for a_, b in zip(ms, ms[1:])] noise = own_noise([v for _, v in ref]) d, t1, t2 = max(jumps, key=lambda x: abs(x[0])) in_noises = f"{d / noise:9.1f}" if noise else f"{'-':>9}" flagged = [iso(x) for x in (t1, t2) if x in rej] print(f"{s:9}{len(ms):5}{lvl:10.4g}{noise:10.3g}{d:+10.4g}{in_noises} {iso(t1)} and {iso(t2)}" + (f" (rejected: {', '.join(flagged)})" if flagged else "")) if not noise: continue lv = local_levels(ms, o.rolling) if o.rolling is not None else [lvl] * len(ms) group = [] # (t, deviation from the level) for (t, v), n in zip(ms + [(None, 0)], lv + [None]): if t is not None and n is not None and abs(v - n) > o.k * noise: if group and (t - group[-1][0] > o.merge or (v > n) != (group[-1][1] > 0)): # same direction only ranges.append((s, group, noise, rej)) group = [] group.append((t, v - n)) by_instant.setdefault(iso(t), {}).setdefault(s, []).append(f"{ABBREV.get(col, col)}{(v - n) / noise:+.0f}") elif t is None and group: ranges.append((s, group, noise, rej)) total = len(ranges) ranges = [p for p in ranges if len(p[1]) >= o.min_measurements] print(f"\n{col}: ranges more than {o.k:g} noises from the level{' rolling ±' + roll + ' (median of the neighbours)' if roll else ''}{' (level outside ' + o.outside + ')' if o.outside and not roll else ''}{' (noise outside ' + o.outside + ')' if o.outside and roll else ''} (merge {int(o.merge.total_seconds() // 60)} min)" + (f"; {len(ranges)} of at least {o.min_measurements} measurements out of {total}" if o.min_measurements > 1 else "")) print(f"{'sensor':9}{'start':<18}{'end':<18}{'measurements':>13}{'extreme':>10}{'(noises)':>9} {'at':<17}{'rejected':>9}") for s, g, noise, rej in ranges: t, d = max(g, key=lambda x: abs(x[1])) print(f"{s:9}{iso(g[0][0]):<18}{iso(g[-1][0]):<18}{len(g):13}{d:+10.4g}{d / noise:9.1f} {iso(t):<17}" f"{sum(x[0] in rej for x in g):9}{' rejected extreme' if t in rej else ''}") if not ranges: print("(none)") if o.instants is not None: refs = sorted(s for s in by if is_reference(by[s])) rej_t, ref_t = {}, {s: {} for s in refs} for s in set(names) | set(refs): for m in filter_window(by[s], o.start, o.end): if rejected(m) and s in names: rej_t.setdefault(iso(m["t"]), []).append(s) if s in ref_t: ref_t[s][iso(m["t"])] = m f = lambda v, fmt: "-" if v is None else format(v, fmt) rows = [(t, c) for t, c in sorted(by_instant.items()) if len(c) >= o.instants] print(f"\nInstants where at least {o.instants} sensors are more than {o.k:g} noises from the level{' rolling ±' + roll if roll else ''} " f"({', '.join(columns)}): {len(rows)} of {len(by_instant)} instants with at least one sensor off level") print("instant n sensors off level (deviation in noises) | rejected | references: error_value (r rejected) / inlier") for t, c in rows: r = " ".join(f"{s} {f(ref_t[s][t]['error_value'], '.3g')}{'r' if rejected(ref_t[s][t]) else ''}/{f(ref_t[s][t]['inlier'], 'g')}" for s in refs if t in ref_t[s]) print(f"{t} {len(c):3} {' '.join(s + '.' + ','.join(v) for s, v in c.items())} | {','.join(rej_t.get(t, [])) or '-'} | {r or '-'}") if __name__ == "__main__": main()