124 lines
6.4 KiB
Python
124 lines
6.4 KiB
Python
"""Compact numeric summary of the series of an "advanced" export, for the diagnostic agent.
|
|||
|
|
|
||
|
|
Usage: python3 tools/scripts/summary.py data/PS_DES_COUDRES/ [--column error_value]
|
||
|
|
[--stat median|min|max] [--step 1h] [--level period|window] [--sensors SENSOR_A,SENSOR_B] [--start 2026-01-01T00:00Z] [--end ...]
|
||
|
|
[--instants T1,T2] [--mark]
|
||
|
|
|
||
|
|
1. Usual level of each sensor: median, MAD (median absolute deviation), min, max, over the whole period
|
||
|
|
(default) or, with --level window, over the --start/--end window only (a level change
|
||
|
|
in the window is not compared with weeks of other measurements).
|
||
|
|
Columns: error_value, corr_x, corr_y, relative_x, relative_y, x, y, sigma0, reproj_err, and the six image quality metrics (peak to snr, raw values; empty on
|
||
|
|
old cameras; against a calm reference: quality.py).
|
||
|
|
2. Table: one row per time step (step start, UTC), one column per sensor, the chosen statistic
|
||
|
|
over the measurements of the step. "rej": all measurements of the step are rejected; "empty": measurements,
|
||
|
|
but the column is empty (e.g. reproj_err of a target); "-": no measurement. The sensor header is repeated
|
||
|
|
every 24 rows. With --instants T1,T2: only the steps that contain these instants. With --mark: "*"
|
||
|
|
after the value when a measurement of the step is rejected (x, y, error_value of a rejected measurement stay filled),
|
||
|
|
"o" when a measurement of the step has inlier = 0 (reference set aside by the compensation).
|
||
|
|
Gives values, not a verdict.
|
||
|
|
"""
|
||
|
|
import argparse
|
||
|
|
import re
|
||
|
|
import statistics
|
||
|
|
import sys
|
||
|
|
from datetime import timedelta
|
||
|
|
|
||
|
|
from data import filter_window, is_reference, iso, load, parse_time, rejected
|
||
|
|
|
||
|
|
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
|
||
|
|
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
|
||
|
|
STATS = {"median": statistics.median, "min": min, "max": max}
|
||
|
|
|
||
|
|
|
||
|
|
def duration(text):
|
||
|
|
"""'15min', '1h', '6h', '1d' -> timedelta."""
|
||
|
|
m = re.fullmatch(r"(\d+)(min|h|d)", text)
|
||
|
|
if not m:
|
||
|
|
raise argparse.ArgumentTypeError(f"invalid step: {text!r} (e.g. 15min, 1h, 1d)")
|
||
|
|
n, unit = int(m.group(1)), m.group(2)
|
||
|
|
return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unit]
|
||
|
|
|
||
|
|
|
||
|
|
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
|
||
|
|
|
||
|
|
|
||
|
|
def own_noise(vs):
|
||
|
|
"""Median of the absolute differences between two consecutive values; 0 when it is below ROUNDING (values constant
|
||
|
|
up to rounding: the corr of the references when the compensation goes exactly through them, two references
|
||
|
|
retained), so that a deviation is not written as billions of noises; 0 also with fewer than two values."""
|
||
|
|
d = [abs(b - a) for a, b in zip(vs, vs[1:])]
|
||
|
|
b = statistics.median(d) if d else 0
|
||
|
|
return b if b >= ROUNDING else 0
|
||
|
|
|
||
|
|
|
||
|
|
def values(ms, col):
|
||
|
|
return [m[col] for m in ms if m[col] is not None]
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
|
|
a.add_argument("folder")
|
||
|
|
a.add_argument("--column", default="error_value", choices=COLUMNS + QUALITY)
|
||
|
|
a.add_argument("--stat", default="median", choices=STATS)
|
||
|
|
a.add_argument("--step", default="1h", type=duration)
|
||
|
|
a.add_argument("--level", default="period", choices=("period", "window"),
|
||
|
|
help="period over which median and MAD are computed (default: all measurements)")
|
||
|
|
a.add_argument("--sensors", help="comma-separated list (default: all)")
|
||
|
|
a.add_argument("--instants", help="T1,T2: only the steps that contain these instants")
|
||
|
|
a.add_argument("--mark", action="store_true", help="*: a measurement of the step rejected; o: a measurement of the step with inlier = 0")
|
||
|
|
a.add_argument("--start", type=parse_time)
|
||
|
|
a.add_argument("--end", type=parse_time)
|
||
|
|
o = a.parse_args()
|
||
|
|
|
||
|
|
by = load(o.folder)
|
||
|
|
if not by:
|
||
|
|
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
|
||
|
|
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
||
|
|
unknown = [s for s in names if s not in by]
|
||
|
|
if unknown:
|
||
|
|
sys.exit(f"Missing sensors: {', '.join(unknown)}. Present: {', '.join(sorted(by))}.")
|
||
|
|
|
||
|
|
t0 = min(m["t"] for s in names for m in by[s]).replace(hour=0, minute=0, second=0, microsecond=0)
|
||
|
|
steps = {}
|
||
|
|
for s in names:
|
||
|
|
for m in filter_window(by[s], o.start, o.end):
|
||
|
|
step_start = t0 + ((m["t"] - t0) // o.step) * o.step
|
||
|
|
steps.setdefault(step_start, {}).setdefault(s, []).append(m)
|
||
|
|
if not steps:
|
||
|
|
sys.exit("No measurement in the requested window.")
|
||
|
|
|
||
|
|
over = "the --start/--end window" if o.level == "window" else "the whole period"
|
||
|
|
print(f"Column: {o.column}. Usual level over {over} (non-empty measurements):")
|
||
|
|
print(f"{'sensor':10} {'n':>5} {'median':>10} {'MAD':>10} {'min':>10} {'max':>10}")
|
||
|
|
for s in names:
|
||
|
|
v = values(filter_window(by[s], o.start, o.end) if o.level == "window" else by[s], o.column)
|
||
|
|
if not v:
|
||
|
|
print(f"{s:10} {0:5} (no value)")
|
||
|
|
continue
|
||
|
|
med = statistics.median(v)
|
||
|
|
mad = statistics.median(abs(x - med) for x in v)
|
||
|
|
print(f"{s:10} {len(v):5} {med:10.5g} {mad:10.5g} {min(v):10.5g} {max(v):10.5g}")
|
||
|
|
|
||
|
|
print(f"\n{o.stat} of {o.column} per step of {o.step} (step start, UTC):")
|
||
|
|
print("rej: all measurements of the step rejected; empty: empty column; -: no measurement")
|
||
|
|
if o.mark:
|
||
|
|
print("*: at least one measurement of the step is rejected; o: at least one measurement of the step has inlier = 0")
|
||
|
|
header = f"{'start':17} " + " ".join(f"{s:>9}" for s in names)
|
||
|
|
wanted = {t0 + ((parse_time(x) - t0) // o.step) * o.step for x in o.instants.split(",")} if o.instants else None
|
||
|
|
for i, k in enumerate(k for k in sorted(steps) if wanted is None or k in wanted):
|
||
|
|
if i % 24 == 0:
|
||
|
|
print(header)
|
||
|
|
cells = []
|
||
|
|
for s in names:
|
||
|
|
ms = steps[k].get(s, [])
|
||
|
|
v = values(ms, o.column)
|
||
|
|
if v:
|
||
|
|
mark = ("*" if any(map(rejected, ms)) else "") + ("o" if any(m["inlier"] == 0 for m in ms) else "") if o.mark else ""
|
||
|
|
cells.append(f"{STATS[o.stat](v):.5g}{mark}".rjust(9))
|
||
|
|
else:
|
||
|
|
cells.append(f"{'-' if not ms else 'rej' if all(map(rejected, ms)) else 'empty':>9}")
|
||
|
|
print(f"{iso(k):17} " + " ".join(cells))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|