Files

134 lines
5.4 KiB
Python

"""What the reading tools share beyond reading (data.py): columns, steps, noise, sensor list, runs, file names.
Standard library only: the tools without a figure run with python3, without matplotlib.
"""
import argparse
import re
import statistics
import sys
from datetime import timedelta
from data import filter_window, is_reference, load, parse_time
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err",
"corr_x_mm", "corr_y_mm")
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
STATS = {"median": statistics.median, "min": min, "max": max}
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
def duration(text):
"""'15min', '1h', '6h', '1d' -> timedelta."""
m = re.fullmatch(r"(\d+)(min|h|d)", text)
if not m:
raise argparse.ArgumentTypeError(f"invalid step: {text!r} (e.g. 15min, 1h, 1d)")
n, unit = int(m.group(1)), m.group(2)
return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unit]
def own_noise(*series):
"""Median of the absolute differences between two consecutive values (within each series given); 0 when it is below
ROUNDING (values constant up to rounding: the corr of the references when the compensation goes exactly through them,
two references retained), so that a deviation is not written as billions of noises; 0 also with fewer than two values."""
d = [abs(b - a) for vs in series for a, b in zip(vs, vs[1:])]
b = statistics.median(d) if d else 0
return b if b >= ROUNDING else 0
def sensors(folder, wanted):
"""-> ({sensor: [measurement]}, the --sensors names, or all of them, references first); exits without an export."""
by = load(folder)
if not by:
sys.exit(f'No "advanced" export in {folder} (see inventory.py).')
return by, wanted.split(",") if wanted else sorted(by, key=lambda s: (not is_reference(by[s]), s))
def check(what, requested, known):
"""Exits on the names of requested that known does not have."""
if missing := [x for x in requested if x not in known]:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
def per_step(first, step):
"""Start of the step of each time, steps aligned on the midnight of `first` (hh:00, hh:15…)."""
midnight = first.replace(hour=0, minute=0, second=0, microsecond=0)
return lambda t: midnight + (t - midnight) // step * step
def window_filter(text):
"""--table: '' everything, 'no' nothing, 'T1/T2' between T1 and T2 -> predicate on a time."""
if text in ("", "no"):
return lambda t: text == ""
limits = [parse_time(x) for x in text.split("/")]
if len(limits) != 2:
sys.exit("--table expects T1/T2 or no.")
return lambda t: limits[0] <= t <= limits[1]
def outside(text):
"""--outside T1/T2 -> predicate: is a time outside that moment (always true without --outside)."""
if not text:
return lambda t: True
limits = [parse_time(x) for x in text.split("/")]
if len(limits) != 2:
sys.exit("--outside expects T1/T2.")
return lambda t: not limits[0] <= t <= limits[1]
def group(items, close):
"""Sorted items -> runs (lists) of consecutive items where close(previous, next)."""
out = []
for x in items:
if out and close(out[-1][-1], x):
out[-1].append(x)
else:
out.append([x])
return out
def without_pose(instants, poses):
"""Runs (start, end, n) of consecutive measurement instants without a camera pose (no registration), to the minute."""
posed = {p["t"].replace(second=0, microsecond=0) for p in poses}
lacking = [(i, t) for i, t in enumerate(instants) if t.replace(second=0, microsecond=0) not in posed]
return [(g[0][1], g[-1][1], len(g)) for g in group(lacking, lambda a, b: b[0] == a[0] + 1)]
def extremes(d):
"""{t: value} -> ((t, value) of the minimum, (t, value) of the maximum)."""
return min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
def targets(by_sensor, col, start, end):
"""At each measurement, median over the targets of col minus the target's own level (median over the window)."""
deviations = {}
for s, ms in by_sensor.items():
if is_reference(ms):
continue
ms = [m for m in filter_window(ms, start, end) if m[col] is not None]
if ms:
level = statistics.median(m[col] for m in ms)
for m in ms:
deviations.setdefault(m["t"], []).append(m[col] - level)
return [{"t": t, "v": statistics.median(v)} for t, v in sorted(deviations.items())]
def cut(series):
"""Times and numeric values, NaN inserted over a gap of more than 1.5 times the usual step (the line breaks)."""
pts = [(p["t"], p["v"]) for p in series if isinstance(p["v"], float)]
if len(pts) < 3:
return [t for t, _ in pts], [v for _, v in pts]
step = statistics.median(b[0] - a[0] for a, b in zip(pts, pts[1:]))
ts, vs = [pts[0][0]], [pts[0][1]]
for (a, _), (b, v) in zip(pts, pts[1:]):
if b - a > 1.5 * step:
ts.append(a + (b - a) / 2)
vs.append(float("nan"))
ts.append(b)
vs.append(v)
return ts, vs
def bound(t, default):
"""--start/--end in a file name."""
return t.strftime("%Y%m%dT%H%MZ") if t else default