Scripts: shared helpers in common.py, no tool imports another; CLI and outputs unchanged
data.py stays the reader (METRIC_RE now also captures the file date, same matches); common.py holds columns, steps, noise, sensor list, runs, file names. Duplicated run grouping, pose gaps, step bucketing, --table/--outside parsing removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c1719cbd1d
commit
6ed4c420e1
13 files changed
+308
-359
No files matched your search
@@ -0,0 +1,132 @@
|
||||
"""What the reading tools share beyond reading (data.py): columns, steps, noise, sensor list, runs, file names.
|
||||
|
||||
Standard library only: the tools without a figure run with python3, without matplotlib.
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
|
||||
from data import filter_window, is_reference, load, parse_time
|
||||
|
||||
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
|
||||
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
|
||||
STATS = {"median": statistics.median, "min": min, "max": max}
|
||||
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
|
||||
|
||||
|
||||
def duration(text):
|
||||
"""'15min', '1h', '6h', '1d' -> timedelta."""
|
||||
m = re.fullmatch(r"(\d+)(min|h|d)", text)
|
||||
if not m:
|
||||
raise argparse.ArgumentTypeError(f"invalid step: {text!r} (e.g. 15min, 1h, 1d)")
|
||||
n, unit = int(m.group(1)), m.group(2)
|
||||
return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unit]
|
||||
|
||||
|
||||
def own_noise(*series):
|
||||
"""Median of the absolute differences between two consecutive values (within each series given); 0 when it is below
|
||||
ROUNDING (values constant up to rounding: the corr of the references when the compensation goes exactly through them,
|
||||
two references retained), so that a deviation is not written as billions of noises; 0 also with fewer than two values."""
|
||||
d = [abs(b - a) for vs in series for a, b in zip(vs, vs[1:])]
|
||||
b = statistics.median(d) if d else 0
|
||||
return b if b >= ROUNDING else 0
|
||||
|
||||
|
||||
def sensors(folder, wanted):
|
||||
"""-> ({sensor: [measurement]}, the --sensors names, or all of them, references first); exits without an export."""
|
||||
by = load(folder)
|
||||
if not by:
|
||||
sys.exit(f'No "advanced" export in {folder} (see inventory.py).')
|
||||
return by, wanted.split(",") if wanted else sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
||||
|
||||
|
||||
def check(what, requested, known):
|
||||
"""Exits on the names of requested that known does not have."""
|
||||
if missing := [x for x in requested if x not in known]:
|
||||
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
|
||||
|
||||
|
||||
def per_step(first, step):
|
||||
"""Start of the step of each time, steps aligned on the midnight of `first` (hh:00, hh:15…)."""
|
||||
midnight = first.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
return lambda t: midnight + (t - midnight) // step * step
|
||||
|
||||
|
||||
def window_filter(text):
|
||||
"""--table: '' everything, 'no' nothing, 'T1/T2' between T1 and T2 -> predicate on a time."""
|
||||
if text in ("", "no"):
|
||||
return lambda t: text == ""
|
||||
limits = [parse_time(x) for x in text.split("/")]
|
||||
if len(limits) != 2:
|
||||
sys.exit("--table expects T1/T2 or no.")
|
||||
return lambda t: limits[0] <= t <= limits[1]
|
||||
|
||||
|
||||
def outside(text):
|
||||
"""--outside T1/T2 -> predicate: is a time outside that moment (always true without --outside)."""
|
||||
if not text:
|
||||
return lambda t: True
|
||||
limits = [parse_time(x) for x in text.split("/")]
|
||||
if len(limits) != 2:
|
||||
sys.exit("--outside expects T1/T2.")
|
||||
return lambda t: not limits[0] <= t <= limits[1]
|
||||
|
||||
|
||||
def group(items, close):
|
||||
"""Sorted items -> runs (lists) of consecutive items where close(previous, next)."""
|
||||
out = []
|
||||
for x in items:
|
||||
if out and close(out[-1][-1], x):
|
||||
out[-1].append(x)
|
||||
else:
|
||||
out.append([x])
|
||||
return out
|
||||
|
||||
|
||||
def without_pose(instants, poses):
|
||||
"""Runs (start, end, n) of consecutive measurement instants without a camera pose (no registration), to the minute."""
|
||||
posed = {p["t"].replace(second=0, microsecond=0) for p in poses}
|
||||
lacking = [(i, t) for i, t in enumerate(instants) if t.replace(second=0, microsecond=0) not in posed]
|
||||
return [(g[0][1], g[-1][1], len(g)) for g in group(lacking, lambda a, b: b[0] == a[0] + 1)]
|
||||
|
||||
|
||||
def extremes(d):
|
||||
"""{t: value} -> ((t, value) of the minimum, (t, value) of the maximum)."""
|
||||
return min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
|
||||
|
||||
|
||||
def targets(by_sensor, col, start, end):
|
||||
"""At each measurement, median over the targets of col minus the target's own level (median over the window)."""
|
||||
deviations = {}
|
||||
for s, ms in by_sensor.items():
|
||||
if is_reference(ms):
|
||||
continue
|
||||
ms = [m for m in filter_window(ms, start, end) if m[col] is not None]
|
||||
if ms:
|
||||
level = statistics.median(m[col] for m in ms)
|
||||
for m in ms:
|
||||
deviations.setdefault(m["t"], []).append(m[col] - level)
|
||||
return [{"t": t, "v": statistics.median(v)} for t, v in sorted(deviations.items())]
|
||||
|
||||
|
||||
def cut(series):
|
||||
"""Times and numeric values, NaN inserted over a gap of more than 1.5 times the usual step (the line breaks)."""
|
||||
pts = [(p["t"], p["v"]) for p in series if isinstance(p["v"], float)]
|
||||
if len(pts) < 3:
|
||||
return [t for t, _ in pts], [v for _, v in pts]
|
||||
step = statistics.median(b[0] - a[0] for a, b in zip(pts, pts[1:]))
|
||||
ts, vs = [pts[0][0]], [pts[0][1]]
|
||||
for (a, _), (b, v) in zip(pts, pts[1:]):
|
||||
if b - a > 1.5 * step:
|
||||
ts.append(a + (b - a) / 2)
|
||||
vs.append(float("nan"))
|
||||
ts.append(b)
|
||||
vs.append(v)
|
||||
return ts, vs
|
||||
|
||||
|
||||
def bound(t, default):
|
||||
"""--start/--end in a file name."""
|
||||
return t.strftime("%Y%m%dT%H%MZ") if t else default
|
||||
Reference in new issue
Block a user