"""What the reading tools share beyond reading (data.py): columns, steps, noise, sensor list, runs, file names. Standard library only: the tools without a figure run with python3, without matplotlib. """ import argparse import re import statistics import sys from datetime import timedelta from data import filter_window, is_reference, load, parse_time COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err") QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py STATS = {"median": statistics.median, "min": min, "max": max} ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise def duration(text): """'15min', '1h', '6h', '1d' -> timedelta.""" m = re.fullmatch(r"(\d+)(min|h|d)", text) if not m: raise argparse.ArgumentTypeError(f"invalid step: {text!r} (e.g. 15min, 1h, 1d)") n, unit = int(m.group(1)), m.group(2) return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unit] def own_noise(*series): """Median of the absolute differences between two consecutive values (within each series given); 0 when it is below ROUNDING (values constant up to rounding: the corr of the references when the compensation goes exactly through them, two references retained), so that a deviation is not written as billions of noises; 0 also with fewer than two values.""" d = [abs(b - a) for vs in series for a, b in zip(vs, vs[1:])] b = statistics.median(d) if d else 0 return b if b >= ROUNDING else 0 def sensors(folder, wanted): """-> ({sensor: [measurement]}, the --sensors names, or all of them, references first); exits without an export.""" by = load(folder) if not by: sys.exit(f'No "advanced" export in {folder} (see inventory.py).') return by, wanted.split(",") if wanted else sorted(by, key=lambda s: (not is_reference(by[s]), s)) def check(what, requested, known): """Exits on the names of requested that known does not have.""" if missing := [x for x in requested if x not in known]: sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.") def per_step(first, step): """Start of the step of each time, steps aligned on the midnight of `first` (hh:00, hh:15…).""" midnight = first.replace(hour=0, minute=0, second=0, microsecond=0) return lambda t: midnight + (t - midnight) // step * step def window_filter(text): """--table: '' everything, 'no' nothing, 'T1/T2' between T1 and T2 -> predicate on a time.""" if text in ("", "no"): return lambda t: text == "" limits = [parse_time(x) for x in text.split("/")] if len(limits) != 2: sys.exit("--table expects T1/T2 or no.") return lambda t: limits[0] <= t <= limits[1] def outside(text): """--outside T1/T2 -> predicate: is a time outside that moment (always true without --outside).""" if not text: return lambda t: True limits = [parse_time(x) for x in text.split("/")] if len(limits) != 2: sys.exit("--outside expects T1/T2.") return lambda t: not limits[0] <= t <= limits[1] def group(items, close): """Sorted items -> runs (lists) of consecutive items where close(previous, next).""" out = [] for x in items: if out and close(out[-1][-1], x): out[-1].append(x) else: out.append([x]) return out def without_pose(instants, poses): """Runs (start, end, n) of consecutive measurement instants without a camera pose (no registration), to the minute.""" posed = {p["t"].replace(second=0, microsecond=0) for p in poses} lacking = [(i, t) for i, t in enumerate(instants) if t.replace(second=0, microsecond=0) not in posed] return [(g[0][1], g[-1][1], len(g)) for g in group(lacking, lambda a, b: b[0] == a[0] + 1)] def extremes(d): """{t: value} -> ((t, value) of the minimum, (t, value) of the maximum).""" return min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1]) def targets(by_sensor, col, start, end): """At each measurement, median over the targets of col minus the target's own level (median over the window).""" deviations = {} for s, ms in by_sensor.items(): if is_reference(ms): continue ms = [m for m in filter_window(ms, start, end) if m[col] is not None] if ms: level = statistics.median(m[col] for m in ms) for m in ms: deviations.setdefault(m["t"], []).append(m[col] - level) return [{"t": t, "v": statistics.median(v)} for t, v in sorted(deviations.items())] def cut(series): """Times and numeric values, NaN inserted over a gap of more than 1.5 times the usual step (the line breaks).""" pts = [(p["t"], p["v"]) for p in series if isinstance(p["v"], float)] if len(pts) < 3: return [t for t, _ in pts], [v for _, v in pts] step = statistics.median(b[0] - a[0] for a, b in zip(pts, pts[1:])) ts, vs = [pts[0][0]], [pts[0][1]] for (a, _), (b, v) in zip(pts, pts[1:]): if b - a > 1.5 * step: ts.append(a + (b - a) / 2) vs.append(float("nan")) ts.append(b) vs.append(v) return ts, vs def bound(t, default): """--start/--end in a file name.""" return t.strftime("%Y%m%dT%H%MZ") if t else default