Scripts: shared helpers in common.py, no tool imports another; CLI and outputs unchanged

data.py stays the reader (METRIC_RE now also captures the file date, same matches);
common.py holds columns, steps, noise, sensor list, runs, file names. Duplicated
run grouping, pose gaps, step bucketing, --table/--outside parsing removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 15:48:42 +01:00
1 parent c1719cbd1d
commit 6ed4c420e1
13 files changed
+308 -359

No files matched your search

+132
View File
@@ -0,0 +1,132 @@
"""What the reading tools share beyond reading (data.py): columns, steps, noise, sensor list, runs, file names.
Standard library only: the tools without a figure run with python3, without matplotlib.
"""
import argparse
import re
import statistics
import sys
from datetime import timedelta
from data import filter_window, is_reference, load, parse_time
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
STATS = {"median": statistics.median, "min": min, "max": max}
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
def duration(text):
"""'15min', '1h', '6h', '1d' -> timedelta."""
m = re.fullmatch(r"(\d+)(min|h|d)", text)
if not m:
raise argparse.ArgumentTypeError(f"invalid step: {text!r} (e.g. 15min, 1h, 1d)")
n, unit = int(m.group(1)), m.group(2)
return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unit]
def own_noise(*series):
"""Median of the absolute differences between two consecutive values (within each series given); 0 when it is below
ROUNDING (values constant up to rounding: the corr of the references when the compensation goes exactly through them,
two references retained), so that a deviation is not written as billions of noises; 0 also with fewer than two values."""
d = [abs(b - a) for vs in series for a, b in zip(vs, vs[1:])]
b = statistics.median(d) if d else 0
return b if b >= ROUNDING else 0
def sensors(folder, wanted):
"""-> ({sensor: [measurement]}, the --sensors names, or all of them, references first); exits without an export."""
by = load(folder)
if not by:
sys.exit(f'No "advanced" export in {folder} (see inventory.py).')
return by, wanted.split(",") if wanted else sorted(by, key=lambda s: (not is_reference(by[s]), s))
def check(what, requested, known):
"""Exits on the names of requested that known does not have."""
if missing := [x for x in requested if x not in known]:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
def per_step(first, step):
"""Start of the step of each time, steps aligned on the midnight of `first` (hh:00, hh:15…)."""
midnight = first.replace(hour=0, minute=0, second=0, microsecond=0)
return lambda t: midnight + (t - midnight) // step * step
def window_filter(text):
"""--table: '' everything, 'no' nothing, 'T1/T2' between T1 and T2 -> predicate on a time."""
if text in ("", "no"):
return lambda t: text == ""
limits = [parse_time(x) for x in text.split("/")]
if len(limits) != 2:
sys.exit("--table expects T1/T2 or no.")
return lambda t: limits[0] <= t <= limits[1]
def outside(text):
"""--outside T1/T2 -> predicate: is a time outside that moment (always true without --outside)."""
if not text:
return lambda t: True
limits = [parse_time(x) for x in text.split("/")]
if len(limits) != 2:
sys.exit("--outside expects T1/T2.")
return lambda t: not limits[0] <= t <= limits[1]
def group(items, close):
"""Sorted items -> runs (lists) of consecutive items where close(previous, next)."""
out = []
for x in items:
if out and close(out[-1][-1], x):
out[-1].append(x)
else:
out.append([x])
return out
def without_pose(instants, poses):
"""Runs (start, end, n) of consecutive measurement instants without a camera pose (no registration), to the minute."""
posed = {p["t"].replace(second=0, microsecond=0) for p in poses}
lacking = [(i, t) for i, t in enumerate(instants) if t.replace(second=0, microsecond=0) not in posed]
return [(g[0][1], g[-1][1], len(g)) for g in group(lacking, lambda a, b: b[0] == a[0] + 1)]
def extremes(d):
"""{t: value} -> ((t, value) of the minimum, (t, value) of the maximum)."""
return min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
def targets(by_sensor, col, start, end):
"""At each measurement, median over the targets of col minus the target's own level (median over the window)."""
deviations = {}
for s, ms in by_sensor.items():
if is_reference(ms):
continue
ms = [m for m in filter_window(ms, start, end) if m[col] is not None]
if ms:
level = statistics.median(m[col] for m in ms)
for m in ms:
deviations.setdefault(m["t"], []).append(m[col] - level)
return [{"t": t, "v": statistics.median(v)} for t, v in sorted(deviations.items())]
def cut(series):
"""Times and numeric values, NaN inserted over a gap of more than 1.5 times the usual step (the line breaks)."""
pts = [(p["t"], p["v"]) for p in series if isinstance(p["v"], float)]
if len(pts) < 3:
return [t for t, _ in pts], [v for _, v in pts]
step = statistics.median(b[0] - a[0] for a, b in zip(pts, pts[1:]))
ts, vs = [pts[0][0]], [pts[0][1]]
for (a, _), (b, v) in zip(pts, pts[1:]):
if b - a > 1.5 * step:
ts.append(a + (b - a) / 2)
vs.append(float("nan"))
ts.append(b)
vs.append(v)
return ts, vs
def bound(t, default):
"""--start/--end in a file name."""
return t.strftime("%Y%m%dT%H%MZ") if t else default
+2 -2
View File
@@ -1,4 +1,4 @@
"""Reading a station's data for the diagnostic agent (standard library). """Reading a station folder (standard library only): the one reader of the app, the reading tools and the detectors.
Formats: tools/DATA.md. Every time is UTC. Formats: tools/DATA.md. Every time is UTC.
""" """
@@ -98,7 +98,7 @@ def load_camera(folder):
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
METRIC_RE = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_") METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_") # name, then the date of the file (one CSV per metric and per day)
def load_metrics(folder): def load_metrics(folder):
+38 -46
View File
@@ -55,9 +55,8 @@ import matplotlib
matplotlib.use("Agg") matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402 import matplotlib.pyplot as plt # noqa: E402
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402 from common import COLUMNS, bound, check, duration, own_noise, sensors # noqa: E402
from plot import COLUMNS, bound # noqa: E402 from data import POSE, filter_window, is_reference, iso, load_camera, parse_time, rejected # noqa: E402
from summary import ROUNDING, duration, own_noise # noqa: E402
def hours(text): def hours(text):
@@ -73,6 +72,27 @@ def at_night(t, h1, h2):
return (h1 <= t.hour or t.hour < h2) if h1 > h2 else h1 <= t.hour < h2 return (h1 <= t.hour or t.hour < h2) if h1 > h2 else h1 <= t.hour < h2
def nights_of(pts, h1, h2):
"""[(t, v)] -> {date of the night: [(t, v)]}: the night hours, dated by the day the night starts."""
nights = {}
for t, v in pts:
if at_night(t, h1, h2):
nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
return nights
def frame(ax, values):
"""Frame from the 1st to the 99th percentile widened by half: an isolated peak does not crush the scale; -> the number
of values out of frame."""
v = sorted(values)
if not v:
return 0
q1, q2 = v[len(v) // 100], v[-1 - len(v) // 100]
margin = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - margin, q2 + margin)
return sum(not q1 - margin <= e <= q2 + margin for e in v)
def cut_index(pts, la, lb): def cut_index(pts, la, lb):
"""Index j (1..n-1) where sum|v-la| before j + sum|v-lb| from j on is smallest.""" """Index j (1..n-1) where sum|v-la| before j + sum|v-lb| from j on is smallest."""
before, after = 0.0, sum(abs(v - lb) for _, v in pts) before, after = 0.0, sum(abs(v - lb) for _, v in pts)
@@ -103,15 +123,10 @@ def main():
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs") a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
o = a.parse_args() o = a.parse_args()
by_sensor = load(o.folder) by_sensor, names = sensors(o.folder, o.sensors)
if not by_sensor:
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
columns = o.columns.split(",") columns = o.columns.split(",")
for requested, known, what in ((names, by_sensor, "sensors"), (columns, COLUMNS, "columns")): check("sensors", names, by_sensor)
missing = [x for x in requested if x not in known] check("columns", columns, COLUMNS)
if missing:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
series = {s: [m for m in filter_window(by_sensor[s], o.start, o.end) if not rejected(m)] for s in names} series = {s: [m for m in filter_window(by_sensor[s], o.start, o.end) if not rejected(m)] for s in names}
if not any(series.values()): if not any(series.values()):
sys.exit("No accepted measurement in the requested window.") sys.exit("No accepted measurement in the requested window.")
@@ -133,10 +148,7 @@ def main():
ax.axis("off") ax.axis("off")
continue continue
noise = noises[s] = own_noise([v for _, v in ms]) noise = noises[s] = own_noise([v for _, v in ms])
nights = {} nights = nights_of(ms, h1, h2)
for t, v in ms:
if at_night(t, h1, h2):
nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
lvl = {d: statistics.median(v for _, v in p) for d, p in nights.items()} lvl = {d: statistics.median(v for _, v in p) for d, p in nights.items()}
cells = [] cells = []
for d in sorted(nights): for d in sorted(nights):
@@ -167,11 +179,7 @@ def main():
for i, (d, p) in enumerate(sorted(per_day.items())): for i, (d, p) in enumerate(sorted(per_day.items())):
ax.plot([x for x, _ in p], [e for _, e in p], "-", lw=0.8, color=f"C{i % 10}", label=f"{d:%d/%m}") ax.plot([x for x, _ in p], [e for _, e in p], "-", lw=0.8, color=f"C{i % 10}", label=f"{d:%d/%m}")
ax.axhline(0, color="0.3", ls=":", lw=0.8) ax.axhline(0, color="0.3", ls=":", lw=0.8)
every = sorted(e for p in per_day.values() for _, e in p) out_of_frame = frame(ax, (e for p in per_day.values() for _, e in p))
q1, q2 = every[len(every) // 100], every[-1 - len(every) // 100] # frame from the 1st to the 99th percentile: an isolated peak does not crush the scale
margin = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - margin, q2 + margin)
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in every)
for x0, x1 in ((h1, 24), (0, h2)) if h1 > h2 else ((h1, h2),): for x0, x1 in ((h1, 24), (0, h2)) if h1 > h2 else ((h1, h2),):
ax.axvspan(x0, x1, color="0.9", lw=0) ax.axvspan(x0, x1, color="0.9", lw=0)
ax.set_title(s + (" (ref.)" if is_reference(series[s]) else "") + f" noise {noise:.2g}" ax.set_title(s + (" (ref.)" if is_reference(series[s]) else "") + f" noise {noise:.2g}"
@@ -217,15 +225,6 @@ def main():
day_bounds(o, names, series, columns, station) day_bounds(o, names, series, columns, station)
def night_levels(pts, h1, h2):
"""[(t, v)] -> {date of the night: median}."""
nights = {}
for t, v in pts:
if at_night(t, h1, h2):
nights.setdefault((t - timedelta(hours=h1)).date(), []).append(v)
return {d: statistics.median(v) for d, v in nights.items()}
def day_bounds(o, names, series, columns, station): def day_bounds(o, names, series, columns, station):
"""Section 3: per day, the deviation from the level of the night that precedes it, for the target median and the pose.""" """Section 3: per day, the deviation from the level of the night that precedes it, for the target median and the pose."""
h1, h2 = o.night h1, h2 = o.night
@@ -240,7 +239,10 @@ def day_bounds(o, names, series, columns, station):
t0, t1 = min(every), max(every) t0, t1 = min(every), max(every)
night_of = lambda t: (t - night_length - timedelta(hours=h1)).date() # night that ends before t (the day of t starts at its end) night_of = lambda t: (t - night_length - timedelta(hours=h1)).date() # night that ends before t (the day of t starts at its end)
days = [night_of(t0) + timedelta(days=i) for i in range((night_of(t1) - night_of(t0)).days + 1)] days = [night_of(t0) + timedelta(days=i) for i in range((night_of(t1) - night_of(t0)).days + 1)]
hm = lambda t: f"{t:%d/%m %H:%M}" hm = lambda t: f"{t:%d/%m %H:%M}" # noqa: E731
span = lambda p, i, j: (f"from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) " # noqa: E731
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
exceed = "threshold" if o.fraction else f"{o.k:g} noises"
threshold_txt = f"threshold {o.fraction:g} times the largest |deviation| of the day" if o.fraction else f"threshold {o.k:g} noises" threshold_txt = f"threshold {o.fraction:g} times the largest |deviation| of the day" if o.fraction else f"threshold {o.k:g} noises"
print(f"\n== Bounds of each day: deviation from the level of the night that precedes it ({h1} h → {h2} h UTC), {threshold_txt}" print(f"\n== Bounds of each day: deviation from the level of the night that precedes it ({h1} h → {h2} h UTC), {threshold_txt}"
+ (f", deviation smoothed over {o.smoothing} measurements" if o.smoothing > 1 else "") + " ==") + (f", deviation smoothed over {o.smoothing} measurements" if o.smoothing > 1 else "") + " ==")
@@ -248,7 +250,7 @@ def day_bounds(o, names, series, columns, station):
"exceedance, just after the last; next night: its level minus that of the reference night") "exceedance, just after the last; next night: its level minus that of the reference night")
fig, axes = plt.subplots(len(seqs), 1, sharex=True, figsize=(18, 2.2 * len(seqs)), squeeze=False) fig, axes = plt.subplots(len(seqs), 1, sharex=True, figsize=(18, 2.2 * len(seqs)), squeeze=False)
for ax, (name, short, per_sensor) in zip(axes.flat, seqs): for ax, (name, short, per_sensor) in zip(axes.flat, seqs):
lvl = {s: night_levels(p, h1, h2) for s, p in per_sensor.items()} lvl = {s: {d: statistics.median(v for _, v in n) for d, n in nights_of(p, h1, h2).items()} for s, p in per_sensor.items()}
nights = sorted({d for n in lvl.values() for d in n}) nights = sorted({d for n in lvl.values() for d in n})
devs = [] # (day, ref. night, [(t, deviation)], deviation of the next night) devs = [] # (day, ref. night, [(t, deviation)], deviation of the next night)
for d in days: for d in days:
@@ -264,9 +266,7 @@ def day_bounds(o, names, series, columns, station):
next_n = [lvl[s][d + timedelta(days=1)] - lvl[s][ref] for s in lvl if ref in lvl[s] and d + timedelta(days=1) in lvl[s]] next_n = [lvl[s][d + timedelta(days=1)] - lvl[s][ref] for s in lvl if ref in lvl[s] and d + timedelta(days=1) in lvl[s]]
devs.append((start.date(), d, ref, [(t, statistics.median(e)) for t, e in sorted(per_t.items())], devs.append((start.date(), d, ref, [(t, statistics.median(e)) for t, e in sorted(per_t.items())],
statistics.median(next_n) if next_n else None)) statistics.median(next_n) if next_n else None))
diffs = [abs(b[1] - a_[1]) for *_, p, _ in devs for a_, b in zip(p, p[1:])] noise = own_noise(*[[e for _, e in p] for *_, p, _ in devs])
noise = statistics.median(diffs) if diffs else 0
noise = noise if noise >= ROUNDING else 0
print(f"\n{name}: noise {noise:.3g}") print(f"\n{name}: noise {noise:.3g}")
in_noises = lambda e: f"{e:+.4g} ({e / noise:+.1f})" if noise else f"{e:+.4g}" in_noises = lambda e: f"{e:+.4g} ({e / noise:+.1f})" if noise else f"{e:+.4g}"
h = o.smoothing // 2 h = o.smoothing // 2
@@ -281,11 +281,10 @@ def day_bounds(o, names, series, columns, station):
over = [i for i, (_, e) in enumerate(p) if threshold and abs(e) > threshold] over = [i for i, (_, e) in enumerate(p) if threshold and abs(e) > threshold]
if over: if over:
i, j = over[0], over[-1] i, j = over[0], over[-1]
line += (f"> {'threshold' if o.fraction else f'{o.k:g} noises'}: {len(over)} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) " line += f"> {exceed}: {len(over)} measurements, {span(p, i, j)}"
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
ax.axvspan(p[i][0], p[j][0], color="0.85", lw=0) ax.axvspan(p[i][0], p[j][0], color="0.85", lw=0)
else: else:
line += f"never > {'threshold' if o.fraction else f'{o.k:g} noises'}" line += f"never > {exceed}"
top, bottom = max(p, key=lambda x: x[1]), min(p, key=lambda x: x[1]) top, bottom = max(p, key=lambda x: x[1]), min(p, key=lambda x: x[1])
line += f" | max {in_noises(top[1])} at {hm(top[0])}, min {in_noises(bottom[1])} at {hm(bottom[0])}" line += f" | max {in_noises(top[1])} at {hm(top[0])}, min {in_noises(bottom[1])} at {hm(bottom[0])}"
x = max(range(len(p)), key=lambda i: abs(p[i][1])) x = max(range(len(p)), key=lambda i: abs(p[i][1]))
@@ -296,8 +295,7 @@ def day_bounds(o, names, series, columns, station):
i -= 1 i -= 1
while j + 1 < len(p) and sign * p[j + 1][1] > threshold: while j + 1 < len(p) and sign * p[j + 1][1] > threshold:
j += 1 j += 1
line += (f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) " line += f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, {span(p, i, j)}"
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
ax.axvspan(p[i][0], p[j][0], color="0.7", lw=0) ax.axvspan(p[i][0], p[j][0], color="0.7", lw=0)
line += f" | next night {in_noises(nxt)}" if nxt is not None else " | next night not measured" line += f" | next night {in_noises(nxt)}" if nxt is not None else " | next night not measured"
print(line) print(line)
@@ -309,13 +307,7 @@ def day_bounds(o, names, series, columns, station):
for y in (o.k * noise, -o.k * noise): for y in (o.k * noise, -o.k * noise):
ax.axhline(y, color="0.4", ls=":", lw=0.8) ax.axhline(y, color="0.4", ls=":", lw=0.8)
ax.axhline(0, color="0.3", lw=0.5) ax.axhline(0, color="0.3", lw=0.5)
all_e = sorted(e for *_, p, _ in devs for _, e in p) out_of_frame = frame(ax, (e for *_, p, _ in devs for _, e in p))
out_of_frame = 0
if all_e: # frame from the 1st to the 99th percentile widened by half, as in byhour: an isolated peak does not crush the scale
q1, q2 = all_e[len(all_e) // 100], all_e[-1 - len(all_e) // 100]
margin = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - margin, q2 + margin)
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in all_e)
ax.set_title(f"{name} noise {noise:.2g}" + (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9) ax.set_title(f"{name} noise {noise:.2g}" + (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9)
ax.grid(alpha=0.3) ax.grid(alpha=0.3)
ax.tick_params(labelsize=7) ax.tick_params(labelsize=7)
-1
View File
@@ -17,7 +17,6 @@ from PIL import Image, ImageDraw, ImageFont
from data import find_files, image_time, iso, parse_time from data import find_files, image_time, iso, parse_time
def main(): def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder") a.add_argument("folder")
+21 -44
View File
@@ -20,14 +20,11 @@ import collections
import csv import csv
import glob import glob
import os import os
import re
import statistics import statistics
from datetime import datetime, timedelta, timezone from datetime import datetime, timezone
from data import filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected from common import QUALITY, group, without_pose
from summary import QUALITY from data import METRIC_RE, filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_\d{2}_\d{2}") # NIEDERHORN_BusCurrent_2026_09_22_00_00 to ….csv
def span(a, b, n): def span(a, b, n):
@@ -158,17 +155,17 @@ def images(paths, start=None, end=None):
return return
# Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot): # Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot):
# one line per change of gap, without losing any of the times covered. # one line per change of gap, without losing any of the times covered.
runs = [] # [first, last, gap in min, number of gaps] pairs = [(a, b, round((b - a).total_seconds() / 300) * 5) for a, b in zip(times, times[1:])]
for a, b in zip(times, times[1:]):
e = round((b - a).total_seconds() / 300) * 5
if runs and runs[-1][2] == e:
runs[-1][1], runs[-1][3] = b, runs[-1][3] + 1
else:
runs.append([a, b, e, 1])
what = "over the window" if start or end else "between the first and the last" what = "over the window" if start or end else "between the first and the last"
print(f" gap between two images {what}, per run of equal gaps (within 5 min):") print(f" gap between two images {what}, per run of equal gaps (within 5 min):")
for a, b, e, n in runs: for g in group(pairs, lambda p, q: p[2] == q[2]):
print(f" {iso(a)} → {iso(b)}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min")) n, e = len(g), g[0][2]
print(f" {iso(g[0][0])} → {iso(g[-1][1])}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min"))
def missing(instants, cadence):
"""Gaps between two present instants: [(before, after, number of absent steps)]."""
return [(a, b, n) for a, b in zip(instants, instants[1:]) if (n := round((b - a) / cadence) - 1) > 0]
def measurements(by): def measurements(by):
@@ -179,8 +176,7 @@ def measurements(by):
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:])) cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants") print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants")
print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min") print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min")
gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(instants, instants[1:])] gaps = missing(instants, cadence)
gaps = [x for x in gaps if x[2] > 0]
print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}") print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}")
for a, b, n in gaps: for a, b, n in gaps:
print(f" between {iso(a)} and {iso(b)}: {n}") print(f" between {iso(a)} and {iso(b)}: {n}")
@@ -207,19 +203,11 @@ def camera(folder, by, start=None, end=None):
if not poses: if not poses:
print(f"\nCamera pose {what}: none") print(f"\nCamera pose {what}: none")
return return
with_pose = {p["t"].replace(second=0, microsecond=0) for p in poses}
print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}") print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}")
runs = [] # [start, end, count, rank] of the consecutive measurement instants without a pose runs = without_pose(instants, poses)
for i, t in enumerate(instants):
if t.replace(second=0, microsecond=0) in with_pose:
continue
if runs and runs[-1][3] == i - 1:
runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i
else:
runs.append([t, t, 1, i])
print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}") print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}")
if start or end: if start or end:
for a, b, n, _ in runs: for a, b, n in runs:
print(f" {span(a, b, n)}") print(f" {span(a, b, n)}")
@@ -232,8 +220,7 @@ def window(by, start, end):
print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present") print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present")
names = sorted(by, key=lambda s: (not is_reference(by[s]), s)) names = sorted(by, key=lambda s: (not is_reference(by[s]), s))
print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):") print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):")
gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(inside, inside[1:])] gaps = missing(inside, cadence)
gaps = [x for x in gaps if x[2] > 0]
for a, b, n in gaps: for a, b, n in gaps:
print(f" between {iso(a)} and {iso(b)}: {n}") print(f" between {iso(a)} and {iso(b)}: {n}")
if not gaps: if not gaps:
@@ -248,14 +235,8 @@ def window(by, start, end):
print("Rejected measurements in the window, per sensor (start → end of each run of consecutive steps, count):") print("Rejected measurements in the window, per sensor (start → end of each run of consecutive steps, count):")
for s in names: for s in names:
rej = [t for t in sorted(series[s]) if rejected(series[s][t])] rej = [t for t in sorted(series[s]) if rejected(series[s][t])]
runs = []
for t in rej:
if runs and t - runs[-1][1] <= cadence * 1.5:
runs[-1][1], runs[-1][2] = t, runs[-1][2] + 1
else:
runs.append([t, t, 1])
if rej: if rej:
print(f" {s} ({len(rej)}): " + "; ".join(span(a, b, n) for a, b, n in runs)) print(f" {s} ({len(rej)}): " + "; ".join(span(g[0], g[-1], len(g)) for g in group(rej, lambda a, b: b - a <= cadence * 1.5)))
if not any(rejected(m) for d in series.values() for m in d.values()): if not any(rejected(m) for d in series.values() for m in d.values()):
print(" (none)") print(" (none)")
refs = [s for s in names if is_reference(by[s])] refs = [s for s in names if is_reference(by[s])]
@@ -276,14 +257,10 @@ def window(by, start, end):
+ ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True))) + ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True)))
print("Instants where references are not retained (runs of consecutive steps with the same references out; " print("Instants where references are not retained (runs of consecutive steps with the same references out; "
f"r = rejected, empty = {col} empty, otherwise the inlier value):") f"r = rejected, empty = {col} empty, otherwise the inlier value):")
runs = [] runs = group([(i, t, h) for i, (t, h) in enumerate(out) if h], lambda a, b: b[0] == a[0] + 1 and b[2] == a[2])
for i, (t, h) in enumerate(out): for g in runs:
if h and runs and runs[-1][3] == h and runs[-1][4] == i - 1: h = g[0][2]
runs[-1][1], runs[-1][2], runs[-1][4] = t, runs[-1][2] + 1, i print(f" {span(g[0][1], g[-1][1], len(g))}: {len(refs) - len(h)} retained; out: "
elif h:
runs.append([t, t, 1, h, i])
for a, b, n, h, _ in runs:
print(f" {span(a, b, n)}: {len(refs) - len(h)} retained; out: "
+ ", ".join(f"{s} ({q})" for s, q in h)) + ", ".join(f"{s} ({q})" for s, q in h))
if not runs: if not runs:
print(" (none)") print(" (none)")
+4 -47
View File
@@ -34,10 +34,8 @@ matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402 import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402 import matplotlib.pyplot as plt # noqa: E402
from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time # noqa: E402 from common import STATS, bound, cut, duration, group, targets # noqa: E402
from inventory import METRIC_RE # noqa: E402 from data import METRIC_RE, POSE, filter_window, iso, load, load_camera, load_temperature, parse_time # noqa: E402
from plot import bound # noqa: E402
from summary import STATS, duration # noqa: E402
def is_identifier(name): def is_identifier(name):
@@ -76,20 +74,6 @@ def read_metrics(folder, start, end):
return {n: filter_window([{"t": t, "v": s[t]} for t in sorted(s)], start, end) for n, s in by_name.items()} return {n: filter_window([{"t": t, "v": s[t]} for t in sorted(s)], start, end) for n, s in by_name.items()}
def targets(by_sensor, col, start, end):
"""At each measurement, median over the targets of col minus the target's own level (median over the window)."""
deviations = {}
for s, ms in by_sensor.items():
if is_reference(ms):
continue
ms = [m for m in filter_window(ms, start, end) if m[col] is not None]
if ms:
level = statistics.median(m[col] for m in ms)
for m in ms:
deviations.setdefault(m["t"], []).append(m[col] - level)
return [{"t": t, "v": statistics.median(v)} for t, v in sorted(deviations.items())]
def smooth(series, width): def smooth(series, width):
"""Moving average centred over width (numeric values).""" """Moving average centred over width (numeric values)."""
pts = [p for p in series if isinstance(p["v"], float)] pts = [p for p in series if isinstance(p["v"], float)]
@@ -101,33 +85,6 @@ def fmt(v):
return "-" if v is None else f"{v:.6g}" if isinstance(v, float) else str(v) return "-" if v is None else f"{v:.6g}" if isinstance(v, float) else str(v)
def runs(empty_steps, step):
"""[t] sorted -> [(start, end, n)] of consecutive steps."""
out = []
for t in empty_steps:
if out and t - out[-1][1] == step:
out[-1] = (out[-1][0], t, out[-1][2] + 1)
else:
out.append((t, t, 1))
return out
def cut(series):
"""Times and numeric values, NaN inserted over a gap of more than 1.5 times the usual step (the line breaks)."""
pts = [(p["t"], p["v"]) for p in series if isinstance(p["v"], float)]
if len(pts) < 3:
return [t for t, _ in pts], [v for _, v in pts]
step = statistics.median(b[0] - a[0] for a, b in zip(pts, pts[1:]))
ts, vs = [pts[0][0]], [pts[0][1]]
for (a, _), (b, v) in zip(pts, pts[1:]):
if b - a > 1.5 * step:
ts.append(a + (b - a) / 2)
vs.append(float("nan"))
ts.append(b)
vs.append(v)
return ts, vs
def main(): def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder") a.add_argument("folder")
@@ -229,8 +186,8 @@ def main():
empty = [k for k in grid if all(k not in table.get(c, {}) for c in metric_cols)] empty = [k for k in grid if all(k not in table.get(c, {}) for c in metric_cols)]
print(f"\nSteps without any metric ({', '.join(metric_cols)}): {len(empty)} of {len(grid)}") print(f"\nSteps without any metric ({', '.join(metric_cols)}): {len(empty)} of {len(grid)}")
for d, f, n in runs(empty, step): for g in group(empty, lambda a, b: b - a == step):
print(f" {iso(d)} → {iso(f)} ({n} steps)") print(f" {iso(g[0])} → {iso(g[-1])} ({len(g)} steps)")
panels = {} # title -> columns panels = {} # title -> columns
for c in columns: for c in columns:
+75 -111
View File
@@ -62,9 +62,8 @@ matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402 import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402 import matplotlib.pyplot as plt # noqa: E402
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402 from common import COLUMNS, bound, check, duration, extremes, outside, own_noise, per_step, sensors, window_filter, without_pose # noqa: E402
from plot import COLUMNS, bound # noqa: E402 from data import POSE, filter_window, is_reference, iso, load_camera, parse_time, rejected # noqa: E402
from summary import duration, own_noise # noqa: E402
def level(ms, col): def level(ms, col):
@@ -166,11 +165,9 @@ def rigid(o, names, series, times, base, station):
+ f".\nResidual = raw position minus the position at {iso(t_b)} carried by this movement [px]; rejected measurements left out of the fit, " + f".\nResidual = raw position minus the position at {iso(t_b)} carried by this movement [px]; rejected measurements left out of the fit, "
f"of tgt_med, tgt_max and of the sensor columns (separately in the per-sensor summary); median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:") f"of tgt_med, tgt_max and of the sensor columns (separately in the per-sensor summary); median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
cols = ["rot_deg", "scale", "tx", "ty", "tgt_med", "tgt_max"] + [f"{s}_{c}" for s in refs + followed for c in ("dx", "dy")] cols = ["rot_deg", "scale", "tx", "ty", "tgt_med", "tgt_max"] + [f"{s}_{c}" for s in refs + followed for c in ("dx", "dy")]
midnight = tf[0].replace(hour=0, minute=0, second=0, microsecond=0) key, rows = per_step(tf[0], o.step), {}
t0 = midnight + ((tf[0] - midnight) // o.step) * o.step
per_step = {}
for t in tf: for t in tf:
line = per_step.setdefault(t0 + ((t - t0) // o.step) * o.step, {}) line = rows.setdefault(key(t), {})
for c, v in zip(cols, fit[t]): for c, v in zip(cols, fit[t]):
line.setdefault(c, []).append(v) line.setdefault(c, []).append(v)
for s in refs + followed: for s in refs + followed:
@@ -178,10 +175,10 @@ def rigid(o, names, series, times, base, station):
line.setdefault(f"{s}_dx", []).append(res[s][t][0]), line.setdefault(f"{s}_dy", []).append(res[s][t][1]) line.setdefault(f"{s}_dx", []).append(res[s][t][0]), line.setdefault(f"{s}_dy", []).append(res[s][t][1])
w = max(10, max(len(c) for c in cols) + 1) w = max(10, max(len(c) for c in cols) + 1)
header = f"{'time':17}" + "".join(f"{c:>{w}}" for c in cols) header = f"{'time':17}" + "".join(f"{c:>{w}}" for c in cols)
for i, t in enumerate(t for t in sorted(per_step) if o.in_table(t)): for i, t in enumerate(t for t in sorted(rows) if o.in_table(t)):
if i % 24 == 0: if i % 24 == 0:
print(header) print(header)
print(f"{iso(t):17}" + "".join(f"{statistics.median(per_step[t][c]):{w}.{7 if c == 'scale' else 3}g}" if c in per_step[t] else " " * w for c in cols)) print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):{w}.{7 if c == 'scale' else 3}g}" if c in rows[t] else " " * w for c in cols))
print("\nPer sensor, accepted measurements: min and max of dx, dy, value at the last time; then the rejected ones separately:") print("\nPer sensor, accepted measurements: min and max of dx, dy, value at the last time; then the rejected ones separately:")
for s in names: for s in names:
good = {t: v for t, v in res[s].items() if t in acc[s]} good = {t: v for t, v in res[s].items() if t in acc[s]}
@@ -189,15 +186,15 @@ def rigid(o, names, series, times, base, station):
for k, c in ((0, "dx"), (1, "dy")): for k, c in ((0, "dx"), (1, "dy")):
line = f"{s:9}{' (ref.)' if s in refs else ' '} {c}" line = f"{s:9}{' (ref.)' if s in refs else ' '} {c}"
if good: if good:
lo, hi = min(good.items(), key=lambda x: x[1][k]), max(good.items(), key=lambda x: x[1][k]) lo, hi = extremes({t: v[k] for t, v in good.items()})
last = res[s].get(tf[-1]) last = res[s].get(tf[-1])
line += (f" min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}, at the last time " line += (f" min {lo[1]:+.3g} at {iso(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}, at the last time "
f"{last[k] if last else math.nan:+.3g}{' (rejected)' if last and tf[-1] not in acc[s] else ''}") f"{last[k] if last else math.nan:+.3g}{' (rejected)' if last and tf[-1] not in acc[s] else ''}")
else: else:
line += " no accepted measurement" line += " no accepted measurement"
if rej: if rej:
lo, hi = min(rej.items(), key=lambda x: x[1][k]), max(rej.items(), key=lambda x: x[1][k]) lo, hi = extremes({t: v[k] for t, v in rej.items()})
line += f"; {len(rej)} rejected: min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}" line += f"; {len(rej)} rejected: min {lo[1]:+.3g} at {iso(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}"
if good or rej: if good or rej:
print(line) print(line)
@@ -224,26 +221,14 @@ def main():
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs") a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
o = a.parse_args() o = a.parse_args()
by_sensor = load(o.folder) by_sensor, names = sensors(o.folder, o.sensors)
if not by_sensor:
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
columns = o.columns.split(",") columns = o.columns.split(",")
residuals = [c for c in o.residuals.split(",") if c and c != "no"] # '' or "no": none, as for --table residuals = [c for c in o.residuals.split(",") if c and c != "no"] # '' or "no": none, as for --table
outside = tuple(parse_time(x) for x in o.outside.split("/")) if o.outside else None kept = outside(o.outside)
if outside and len(outside) != 2: o.in_table = window_filter(o.table)
sys.exit("--outside expects T1/T2.") check("sensors", names, by_sensor)
if o.table in ("", "no"): check("sensors of --follow (taken from --sensors)", [s for s in o.follow.split(",") if s], names)
o.in_table = lambda t: o.table == "" check("columns", columns + residuals + [o.column], COLUMNS)
else:
limits = [parse_time(x) for x in o.table.split("/")]
if len(limits) != 2:
sys.exit("--table expects T1/T2 or no.")
o.in_table = lambda t: limits[0] <= t <= limits[1]
for requested, known, what in ((names, by_sensor, "sensors"), ([s for s in o.follow.split(",") if s], names, "sensors of --follow (taken from --sensors)"), (columns + residuals + [o.column], COLUMNS, "columns")):
missing = [x for x in requested if x not in known]
if missing:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
series = {s: filter_window(by_sensor[s], o.start, o.end) for s in names} series = {s: filter_window(by_sensor[s], o.start, o.end) for s in names}
if not any(series.values()): if not any(series.values()):
sys.exit("No measurement in the requested window.") sys.exit("No measurement in the requested window.")
@@ -274,7 +259,7 @@ def main():
ax.plot(cam_t, [pose_t.get(t, math.nan) for t in cam_t], ".-", ms=3, lw=0.5, color="C2", label="camera pose (one point per registration)") ax.plot(cam_t, [pose_t.get(t, math.nan) for t in cam_t], ".-", ms=3, lw=0.5, color="C2", label="camera pose (one point per registration)")
ax.set_ylabel(f"camera: {col}" + (" [deg]" if col in ("pan", "tilt", "rotation") else "")) ax.set_ylabel(f"camera: {col}" + (" [deg]" if col in ("pan", "tilt", "rotation") else ""))
ax.legend(loc="upper left", fontsize=8) ax.legend(loc="upper left", fontsize=8)
commons, refs_per_col, ref_devs = {}, {}, {} commons, refs_per_col, ref_devs = {}, {}, {} # median per measurement of each column; references per column, their deviations
for ax, col in zip(axes[len(pose_cols):, 0], columns): for ax, col in zip(axes[len(pose_cols):, 0], columns):
e, _ = deviations(col) e, _ = deviations(col)
med, low, high, tt = [], [], [], [] med, low, high, tt = [], [], [], []
@@ -329,19 +314,13 @@ def main():
print(common) print(common)
# table of the common median, per step # table of the common median, per step
midnight = times[0].replace(hour=0, minute=0, second=0, microsecond=0) key = per_step(times[0], o.step) # steps aligned on the clock (hh:00, hh:15…), not on the 1st measurement
t0 = midnight + ((times[0] - midnight) // o.step) * o.step # steps aligned on the clock (hh:00, hh:15…), not on the 1st measurement
rows = {}
for col in columns:
for t, v in commons[col].items():
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col, []).append(v)
for col in pose_cols: for col in pose_cols:
commons["cam_" + col] = {p["t"]: p[col] for p in poses if p[col] is not None} commons["cam_" + col] = {p["t"]: p[col] for p in poses if p[col] is not None}
for t, v in commons["cam_" + col].items(): rows = {}
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault("cam_" + col, []).append(v) for col in columns + ["cam_" + c for c in pose_cols] + [c + "_targets" for c in refs_per_col]:
for col in refs_per_col: for t, v in commons[col].items():
for t, v in commons[col + "_targets"].items(): rows.setdefault(key(t), {}).setdefault(col, []).append(v)
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col + "_targets", []).append(v)
table_cols = columns + [c + "_targets" for c in refs_per_col] + ["cam_" + c for c in pose_cols] table_cols = columns + [c + "_targets" for c in refs_per_col] + ["cam_" + c for c in pose_cols]
print(f"\nCommon median (deviation from the median of the window){' and raw camera pose (cam_, empty without registration)' if pose_cols else ''}, " print(f"\nCommon median (deviation from the median of the window){' and raw camera pose (cam_, empty without registration)' if pose_cols else ''}, "
f"median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:") f"median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
@@ -352,15 +331,14 @@ def main():
print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):13.{6 if c.startswith('cam_') else 4}g}" if c in rows[t] else " " * 13 for c in table_cols)) print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):13.{6 if c.startswith('cam_') else 4}g}" if c in rows[t] else " " * 13 for c in table_cols))
for col in table_cols: for col in table_cols:
if commons[col]: if commons[col]:
lo, hi = min(commons[col].items(), key=lambda x: x[1]), max(commons[col].items(), key=lambda x: x[1]) lo, hi = extremes(commons[col])
g = 6 if col.startswith("cam_") else 4 g = 6 if col.startswith("cam_") else 4
print(f"{col} (measurements): min {lo[1]:.{g}g} at {iso(lo[0])}, max {hi[1]:.{g}g} at {iso(hi[0])}") print(f"{col} (measurements): min {lo[1]:.{g}g} at {iso(lo[0])}, max {hi[1]:.{g}g} at {iso(hi[0])}")
for col, refs in refs_per_col.items(): for col, refs in refs_per_col.items():
print(f"\n{col} raw of the global references, deviation from their median of the window (measurements); {col}_targets: median of the targets alone:") print(f"\n{col} raw of the global references, deviation from their median of the window (measurements); {col}_targets: median of the targets alone:")
for s in refs: for s in refs:
d = ref_devs[col][s] lo, hi = extremes(ref_devs[col][s])
lo, hi = min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
rej = {m["t"] for m in series[s] if rejected(m)} rej = {m["t"] for m in series[s] if rejected(m)}
r = lambda t: " (rejected)" if t in rej else "" r = lambda t: " (rejected)" if t in rej else ""
print(f"{s:9} min {lo[1]:+.3g} at {iso(lo[0])}{r(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}{r(hi[0])}" print(f"{s:9} min {lo[1]:+.3g} at {iso(lo[0])}{r(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}{r(hi[0])}"
@@ -368,66 +346,71 @@ def main():
if not pose_cols: if not pose_cols:
print("Camera pose: none in the window (no camera/ in the folder, or no registration).") print("Camera pose: none in the window (no camera/ in the folder, or no registration).")
else: else:
posed = {p["t"].replace(second=0, microsecond=0) for p in poses} runs = without_pose(times, poses)
runs = [] # runs of consecutive measurement times without a pose: [start, end, number]
for i, t in enumerate(times):
if t.replace(second=0, microsecond=0) in posed:
continue
if runs and runs[-1][3] == i - 1:
runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i
else:
runs.append([t, t, 1, i])
print(f"Measurement times without a camera pose (no registration): {sum(x[2] for x in runs)} of {len(times)}" print(f"Measurement times without a camera pose (no registration): {sum(x[2] for x in runs)} of {len(times)}"
+ (": " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n, _ in runs) if runs else "")) + (": " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n in runs) if runs else ""))
# 2. small multiples # 2. small multiples and 3. residuals: one small plot per sensor, one scale each
col = o.column mins = int(o.step.total_seconds() // 60)
n = len(names) nrows = math.ceil(len(names) / 5)
nrows = math.ceil(n / 5)
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False) def sheet():
for ax, s in zip(axes.flat, names): return plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
ms = [m for m in series[s] if m[col] is not None]
ax.plot([m["t"] for m in ms], [m[col] for m in ms], "-", lw=0.4, color="0.7") def small(ax, pts, line, title):
"""Measurements {t: v} in grey, their median per step in blue (returned), a dotted line at `line`."""
steps = {} steps = {}
for m in ms: for t, v in pts.items():
steps.setdefault(t0 + ((m["t"] - t0) // o.step) * o.step, []).append(m[col]) steps.setdefault(key(t), []).append(v)
ax.plot(list(steps), [statistics.median(v) for v in steps.values()], "-", lw=1.1, color="C0") pm = {t: statistics.median(v) for t, v in steps.items()}
lv = level(series[s], col) ax.plot(list(pts), list(pts.values()), "-", lw=0.4, color="0.7")
if lv: ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0")
ax.axhline(lv[0], color="C3", ls=":", lw=0.8) if line is not None:
ax.set_title(s + (" (ref.)" if series[s] and is_reference(series[s]) else ""), fontsize=9) ax.axhline(line, color="C3", ls=":", lw=0.8)
ax.set_title(title, fontsize=9)
ax.grid(alpha=0.3) ax.grid(alpha=0.3)
ax.tick_params(labelsize=7) ax.tick_params(labelsize=7)
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5)) ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh")) ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
for ax in list(axes.flat)[n:]: return pm
ax.axis("off")
fig.suptitle(f"{station}: {col}, median per step of {int(o.step.total_seconds() // 60)} min (blue), measurements (grey), " def save(fig, axes, title, name):
f"median of the window (dotted); one scale per sensor (UTC times)") for ax in list(axes.flat)[len(names):]:
fig.tight_layout() ax.axis("off")
multiples = base.format(f"multiples-{col}") fig.suptitle(title)
fig.savefig(multiples, dpi=100, bbox_inches="tight") fig.tight_layout()
print(multiples) path = base.format(name)
fig.savefig(path, dpi=100, bbox_inches="tight")
plt.close(fig)
print(path)
col = o.column
fig, axes = sheet()
for ax, s in zip(axes.flat, names):
lv = level(series[s], col)
small(ax, {m["t"]: m[col] for m in series[s] if m[col] is not None}, lv and lv[0],
s + (" (ref.)" if series[s] and is_reference(series[s]) else ""))
save(fig, axes, f"{station}: {col}, median per step of {mins} min (blue), measurements (grey), "
f"median of the window (dotted); one scale per sensor (UTC times)", f"multiples-{col}")
def own_level(ms, col): # (median, MAD) outside --outside, or over the window when nothing is left
return level([m for m in ms if kept(m["t"])] or ms, col)
# 3. residuals: each sensor minus its own level
for col, others in [(c, False) for c in residuals] + [(c, True) for c in residuals if o.others]: for col, others in [(c, False) for c in residuals] + [(c, True) for c in residuals if o.others]:
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False) fig, axes = sheet()
dev = {} dev = {}
if others: # {sensor: {t: value - own level}} for the median of the others if others: # {sensor: {t: value - own level}} for the median of the others
for q in names: for q in names:
mq = [m for m in series[q] if m[col] is not None] mq = [m for m in series[q] if m[col] is not None]
rq = [m for m in mq if not outside or not outside[0] <= m["t"] <= outside[1]] or mq if nq := own_level(mq, col):
nq = level(rq, col)
if nq:
dev[q] = {m["t"]: m[col] - nq[0] for m in mq} dev[q] = {m["t"]: m[col] - nq[0] for m in mq}
print(f"\n{col} minus the own level{' and minus the median of the other sensors at the same time' if others else ''} ({'outside ' + o.outside if outside else 'median of the window'}), median per step of " print(f"\n{col} minus the own level{' and minus the median of the other sensors at the same time' if others else ''} ({'outside ' + o.outside if o.outside else 'median of the window'}), median per step of "
f"{int(o.step.total_seconds() // 60)} min:") f"{mins} min:")
slopes = [] slopes = []
print(f"{'sensor':9}{'level':>10}{'noise':>10}{'min dev.':>11} {'at':<17}{'(noises)':>9}{'max dev.':>11} {'at':<17}{'(noises)':>9}") print(f"{'sensor':9}{'level':>10}{'noise':>10}{'min dev.':>11} {'at':<17}{'(noises)':>9}{'max dev.':>11} {'at':<17}{'(noises)':>9}")
for ax, s in zip(axes.flat, names): for ax, s in zip(axes.flat, names):
ms = [m for m in series[s] if m[col] is not None] ms = [m for m in series[s] if m[col] is not None]
ref = [m for m in ms if not outside or not outside[0] <= m["t"] <= outside[1]] or ms lv = own_level(ms, col)
lv = level(ref, col)
if not lv: if not lv:
ax.axis("off") ax.axis("off")
continue continue
@@ -443,20 +426,8 @@ def main():
continue continue
else: else:
res = {m["t"]: m[col] - lv[0] for m in ms} res = {m["t"]: m[col] - lv[0] for m in ms}
steps = {} lo, hi = extremes(small(ax, res, 0, s + (" (ref.)" if is_reference(ms) else "") + f" noise {noise:.2g}"))
for t, v in res.items(): f = lambda v: f"{v / noise:9.1f}" if noise else f"{'-':>9}" # noqa: E731
steps.setdefault(t0 + ((t - t0) // o.step) * o.step, []).append(v)
pm = {t: statistics.median(v) for t, v in steps.items()}
ax.plot(list(res), list(res.values()), "-", lw=0.4, color="0.7")
ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0")
ax.axhline(0, color="C3", ls=":", lw=0.8)
ax.set_title(s + (" (ref.)" if is_reference(ms) else "") + f" noise {noise:.2g}", fontsize=9)
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
lo, hi = min(pm.items(), key=lambda x: x[1]), max(pm.items(), key=lambda x: x[1])
f = lambda v: f"{v / noise:9.1f}" if noise else f"{'-':>9}"
print(f"{s:9}{lv[0]:10.4g}{noise:10.3g}{lo[1]:11.3g} {iso(lo[0]):<17}{f(lo[1])}{hi[1]:11.3g} {iso(hi[0]):<17}{f(hi[1])}") print(f"{s:9}{lv[0]:10.4g}{noise:10.3g}{lo[1]:11.3g} {iso(lo[0]):<17}{f(lo[1])}{hi[1]:11.3g} {iso(hi[0]):<17}{f(hi[1])}")
if others: if others:
for c in columns: for c in columns:
@@ -465,17 +436,10 @@ def main():
xs, ys = [commons[c][t] for t in tt], [res[t] for t in tt] xs, ys = [commons[c][t] for t in tt], [res[t] for t in tt]
slope = statistics.linear_regression(xs, ys).slope slope = statistics.linear_regression(xs, ys).slope
slopes.append(f"{s:9}slope on common {c} {slope:+.4g} per unit, r = {statistics.correlation(xs, ys):+.2f}") slopes.append(f"{s:9}slope on common {c} {slope:+.4g} per unit, r = {statistics.correlation(xs, ys):+.2f}")
for ax in list(axes.flat)[n:]:
ax.axis("off")
if slopes: if slopes:
print("\n".join(slopes)) print("\n".join(slopes))
fig.suptitle(f"{station}: {col} minus the own level{' and the median of the others' if others else ''} of each sensor, median per step of " save(fig, axes, f"{station}: {col} minus the own level{' and the median of the others' if others else ''} of each sensor, median per step of "
f"{int(o.step.total_seconds() // 60)} min (blue), measurements (grey); one scale per sensor (UTC times)") f"{mins} min (blue), measurements (grey); one scale per sensor (UTC times)", f"residuals-{col}" + ("-others" if others else ""))
fig.tight_layout()
path = base.format(f"residuals-{col}" + ("-others" if others else ""))
fig.savefig(path, dpi=100, bbox_inches="tight")
plt.close(fig)
print(path)
if o.rigid: if o.rigid:
rigid(o, names, series, times, base, station) rigid(o, names, series, times, base, station)
+5 -16
View File
@@ -17,10 +17,8 @@ matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402 import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402 import matplotlib.pyplot as plt # noqa: E402
from data import filter_window, is_reference, load, parse_time # noqa: E402 from common import COLUMNS, QUALITY, bound, check, sensors # noqa: E402
from summary import QUALITY # noqa: E402 from data import filter_window, parse_time # noqa: E402
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
def zone(text): def zone(text):
@@ -31,10 +29,6 @@ def zone(text):
return parse_time(start), parse_time(end) return parse_time(start), parse_time(end)
def bound(t, default):
return t.strftime("%Y%m%dT%H%MZ") if t else default
def main(): def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder") a.add_argument("folder")
@@ -46,15 +40,10 @@ def main():
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG") a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG")
o = a.parse_args() o = a.parse_args()
by = load(o.folder) by, names = sensors(o.folder, o.sensors)
if not by:
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
columns = o.columns.split(",") columns = o.columns.split(",")
for requested, known, what in ((names, by, "sensors"), (columns, COLUMNS + QUALITY, "columns")): check("sensors", names, by)
missing = [x for x in requested if x not in known] check("columns", columns, COLUMNS + QUALITY)
if missing:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
series = {s: filter_window(by[s], o.start, o.end) for s in names} series = {s: filter_window(by[s], o.start, o.end) for s in names}
if not any(series.values()): if not any(series.values()):
sys.exit("No measurement in the requested window.") sys.exit("No measurement in the requested window.")
+12 -21
View File
@@ -46,7 +46,7 @@ import sys
from datetime import timedelta from datetime import timedelta
import data import data
from summary import QUALITY as METRICS, duration from common import QUALITY as METRICS, duration, group, per_step, sensors
HIGH_BAD = ("fwhm", "dominance") # for these two, it is the rise that says a degraded image HIGH_BAD = ("fwhm", "dominance") # for these two, it is the rise that says a degraded image
COLOURS = {"Visibility": "tab:blue", "Obstruction": "tab:orange", "Thermal displacement": "tab:red", COLOURS = {"Visibility": "tab:blue", "Obstruction": "tab:orange", "Thermal displacement": "tab:red",
@@ -89,6 +89,11 @@ def affected(r, c, threshold):
return r is not None and ((c in ("flux", "snr") and r < threshold) or (c == "dominance" and r > 1 / threshold)) return r is not None and ((c in ("flux", "snr") and r < threshold) or (c == "dominance" and r > 1 / threshold))
def hit(ref, s, m, threshold):
"""The measurement says a degraded image: flux or snr below threshold, or dominance above 1/threshold."""
return any(affected(ratio(ref, s, c, m), c, threshold) for c in ("flux", "snr", "dominance"))
def med(v): def med(v):
return f"{statistics.median(v):.2f}" if v else "-" return f"{statistics.median(v):.2f}" if v else "-"
@@ -123,14 +128,8 @@ def summary(by_sensor, ref, names, refs, pmax):
for c in METRICS)) for c in METRICS))
def per_step(by_sensor, names, step):
step = duration(step)
start = min(m["t"] for s in names for m in by_sensor[s]).replace(hour=0, minute=0, second=0, microsecond=0)
return lambda t: start + (t - start) // step * step
def table(by_sensor, ref, names, c, step, threshold): def table(by_sensor, ref, names, c, step, threshold):
key, cells = per_step(by_sensor, names, step), {} key, cells = per_step(min(m["t"] for s in names for m in by_sensor[s]), duration(step)), {}
for s in names: for s in names:
for m in by_sensor[s]: for m in by_sensor[s]:
r = ratio(ref, s, c, m) r = ratio(ref, s, c, m)
@@ -158,7 +157,7 @@ def table(by_sensor, ref, names, c, step, threshold):
def common(by_sensor, ref, names, refs, step): def common(by_sensor, ref, names, refs, step):
"""Per step: median of the ratios of the targets, then of the references, for each metric.""" """Per step: median of the ratios of the targets, then of the references, for each metric."""
key, cells = per_step(by_sensor, names, step), {} key, cells = per_step(min(m["t"] for s in names for m in by_sensor[s]), duration(step)), {}
for s in names: for s in names:
for m in by_sensor[s]: for m in by_sensor[s]:
d = cells.setdefault(key(m["t"]), {c: ([], []) for c in METRICS}) d = cells.setdefault(key(m["t"]), {c: ([], []) for c in METRICS})
@@ -204,7 +203,7 @@ def labels(a, every, ref, refs, names, start, end):
worst = [] worst = []
for s in names: for s in names:
ms = [m for m in win[s] if t0 <= m["t"] <= t1] ms = [m for m in win[s] if t0 <= m["t"] <= t1]
if any(affected(ratio(ref, s, c, m), c, a.threshold) for m in ms for c in ("flux", "snr", "dominance")): if any(hit(ref, s, m, a.threshold) for m in ms):
fl = [x for x in (ratio(ref, s, "flux", m) for m in ms) if x is not None] fl = [x for x in (ratio(ref, s, "flux", m) for m in ms) if x is not None]
worst.append((min(fl) if fl else 1, s + ("r" if s in refs else ""))) worst.append((min(fl) if fl else 1, s + ("r" if s in refs else "")))
worst.sort() worst.sort()
@@ -236,14 +235,9 @@ def outside(a, every, ref, refs, names, start, end):
for m in data.filter_window(every[s], start, end): for m in data.filter_window(every[s], start, end):
if within(m["t"], covered): if within(m["t"], covered):
continue continue
if any(affected(ratio(ref, s, c, m), c, a.threshold) for c in ("flux", "snr", "dominance")): if hit(ref, s, m, a.threshold):
inst.setdefault(m["t"], {})[s + ("r" if s in refs else "")] = ratio(ref, s, "flux", m) or 0 inst.setdefault(m["t"], {})[s + ("r" if s in refs else "")] = ratio(ref, s, "flux", m) or 0
runs = [] runs = group(sorted(t for t in inst if len(inst[t]) >= a.min), lambda x, y: y - x <= timedelta(minutes=15))
for t in sorted(t for t in inst if len(inst[t]) >= a.min):
if runs and t - runs[-1][-1] <= timedelta(minutes=15):
runs[-1].append(t)
else:
runs.append([t])
print(f"Outside the labels (except {a.except_}): times when at least {a.min} sensors are affected; {len(runs)} runs") print(f"Outside the labels (except {a.except_}): times when at least {a.min} sensors are affected; {len(runs)} runs")
for ru in runs: for ru in runs:
worst = {} worst = {}
@@ -369,13 +363,10 @@ def main():
if unknown: if unknown:
p.error(f"unknown metrics: {', '.join(unknown)}; known: {', '.join(METRICS)}") p.error(f"unknown metrics: {', '.join(unknown)}; known: {', '.join(METRICS)}")
every = data.load(a.folder) every, names = sensors(a.folder, a.sensors)
if not every:
sys.exit(f"No \"advanced\" export in {a.folder} (see inventory.py).")
if not any(m[c] is not None for ms in every.values() for m in ms for c in METRICS): if not any(m[c] is not None for ms in every.values() for m in ms for c in METRICS):
sys.exit(f"No quality metrics in {a.folder}: peak to snr empty (old camera; " sys.exit(f"No quality metrics in {a.folder}: peak to snr empty (old camera; "
"tools/DATA.md).") "tools/DATA.md).")
names = a.sensors.split(",") if a.sensors else sorted(every, key=lambda s: (not data.is_reference(every[s]), s))
missing = [s for s in names if s not in every] missing = [s for s in names if s not in every]
if missing: if missing:
sys.exit(f"Missing sensors: {', '.join(missing)}. Present: {', '.join(sorted(every))}.") sys.exit(f"Missing sensors: {', '.join(missing)}. Present: {', '.join(sorted(every))}.")
+8 -15
View File
@@ -34,8 +34,8 @@ import bisect
import statistics import statistics
import sys import sys
from data import filter_window, is_reference, iso, load, parse_time, rejected from common import COLUMNS, check, duration, outside, own_noise, sensors
from summary import COLUMNS, duration, own_noise from data import filter_window, is_reference, iso, parse_time, rejected
ABBREV = {"corr_x": "cx", "corr_y": "cy", "error_value": "ev"} ABBREV = {"corr_x": "cx", "corr_y": "cy", "error_value": "ev"}
@@ -68,18 +68,11 @@ def main():
a.add_argument("--end", type=parse_time) a.add_argument("--end", type=parse_time)
o = a.parse_args() o = a.parse_args()
by = load(o.folder) by, names = sensors(o.folder, o.sensors)
if not by:
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
columns = o.columns.split(",") columns = o.columns.split(",")
outside = tuple(parse_time(x) for x in o.outside.split("/")) if o.outside else None kept = outside(o.outside)
if outside and len(outside) != 2: check("sensors", names, by)
sys.exit("--outside expects T1/T2.") check("columns", columns, COLUMNS)
for requested, known, what in ((names, by, "sensors"), (columns, COLUMNS, "columns")):
missing = [x for x in requested if x not in known]
if missing:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
series = {s: filter_window(by[s], o.start, o.end) for s in names} series = {s: filter_window(by[s], o.start, o.end) for s in names}
if not any(series.values()): if not any(series.values()):
sys.exit("No measurement in the requested window.") sys.exit("No measurement in the requested window.")
@@ -96,7 +89,7 @@ def main():
if len(ms) < 2: if len(ms) < 2:
print(f"{s:9}{len(ms):5} (fewer than 2 values)") print(f"{s:9}{len(ms):5} (fewer than 2 values)")
continue continue
ref = [x for x in ms if not outside or not outside[0] <= x[0] <= outside[1]] ref = [x for x in ms if kept(x[0])]
if len(ref) < 2: if len(ref) < 2:
print(f"{s:9}{len(ms):5} (fewer than 2 values outside --outside)") print(f"{s:9}{len(ms):5} (fewer than 2 values outside --outside)")
continue continue
@@ -123,7 +116,7 @@ def main():
ranges.append((s, group, noise, rej)) ranges.append((s, group, noise, rej))
total = len(ranges) total = len(ranges)
ranges = [p for p in ranges if len(p[1]) >= o.min_measurements] ranges = [p for p in ranges if len(p[1]) >= o.min_measurements]
print(f"\n{col}: ranges more than {o.k:g} noises from the level{' rolling ±' + roll + ' (median of the neighbours)' if roll else ''}{' (level outside ' + o.outside + ')' if outside and not roll else ''}{' (noise outside ' + o.outside + ')' if outside and roll else ''} (merge {int(o.merge.total_seconds() // 60)} min)" print(f"\n{col}: ranges more than {o.k:g} noises from the level{' rolling ±' + roll + ' (median of the neighbours)' if roll else ''}{' (level outside ' + o.outside + ')' if o.outside and not roll else ''}{' (noise outside ' + o.outside + ')' if o.outside and roll else ''} (merge {int(o.merge.total_seconds() // 60)} min)"
+ (f"; {len(ranges)} of at least {o.min_measurements} measurements out of {total}" if o.min_measurements > 1 else "")) + (f"; {len(ranges)} of at least {o.min_measurements} measurements out of {total}" if o.min_measurements > 1 else ""))
print(f"{'sensor':9}{'start':<18}{'end':<18}{'measurements':>13}{'extreme':>10}{'(noises)':>9} {'at':<17}{'rejected':>9}") print(f"{'sensor':9}{'start':<18}{'end':<18}{'measurements':>13}{'extreme':>10}{'(noises)':>9} {'at':<17}{'rejected':>9}")
for s, g, noise, rej in ranges: for s, g, noise, rej in ranges:
+3 -14
View File
@@ -33,10 +33,8 @@ matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402 import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402 import matplotlib.pyplot as plt # noqa: E402
from common import STATS, bound, cut, duration, group, targets, window_filter # noqa: E402
from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time, rejected # noqa: E402 from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time, rejected # noqa: E402
from metrics import cut, runs, targets # noqa: E402
from plot import bound # noqa: E402
from summary import STATS, duration # noqa: E402
COLUMNS = ("temperature",) + tuple("cam_" + c for c in POSE) + ("tgt_corr_x", "tgt_corr_y", "rej_targets", "rej_refs") COLUMNS = ("temperature",) + tuple("cam_" + c for c in POSE) + ("tgt_corr_x", "tgt_corr_y", "rej_targets", "rej_refs")
@@ -83,13 +81,7 @@ def main():
columns = o.columns.split(",") columns = o.columns.split(",")
if missing := [c for c in columns if c not in COLUMNS]: if missing := [c for c in columns if c not in COLUMNS]:
sys.exit(f"Unknown columns: {', '.join(missing)}. Known: {', '.join(COLUMNS)}.") sys.exit(f"Unknown columns: {', '.join(missing)}. Known: {', '.join(COLUMNS)}.")
if o.table in ("", "no"): in_table = window_filter(o.table)
in_table = lambda t: o.table == "" # noqa: E731
else:
limits = [parse_time(x) for x in o.table.split("/")]
if len(limits) != 2:
sys.exit("--table expects T1/T2 or no.")
in_table = lambda t: limits[0] <= t <= limits[1] # noqa: E731
names = [os.path.basename(os.path.normpath(d)) for d in o.folders] names = [os.path.basename(os.path.normpath(d)) for d in o.folders]
by_station, measured = {}, {} by_station, measured = {}, {}
@@ -123,10 +115,7 @@ def main():
t = tuple(n for n in names if k not in with_temp[n]) t = tuple(n for n in names if k not in with_temp[n])
if m or t: if m or t:
keys.setdefault((m, t), []).append(k) keys.setdefault((m, t), []).append(k)
lines = [] lines = [(g[0], g[-1], len(g), m, t) for (m, t), ks in keys.items() for g in group(ks, lambda a, b: b - a == o.step)]
for (m, t), ks in keys.items():
for d, f, nb in runs(ks, o.step):
lines.append((d, f, nb, m, t))
for d, f, nb, m, t in sorted(lines): for d, f, nb, m, t in sorted(lines):
print(f" {iso(d)} → {iso(f)} ({nb} steps): no measurement {','.join(m) or '-'}; no temperature {','.join(t) or '-'}") print(f" {iso(d)} → {iso(f)} ({nb} steps): no measurement {','.join(m) or '-'}; no temperature {','.join(t) or '-'}")
if not lines: if not lines:
+6 -36
View File
@@ -18,37 +18,11 @@ Usage: python3 tools/scripts/summary.py data/PS_DES_COUDRES/ [--column error_val
Gives values, not a verdict. Gives values, not a verdict.
""" """
import argparse import argparse
import re
import statistics import statistics
import sys import sys
from datetime import timedelta
from data import filter_window, is_reference, iso, load, parse_time, rejected from common import COLUMNS, QUALITY, STATS, duration, per_step, sensors
from data import filter_window, iso, parse_time, rejected
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
STATS = {"median": statistics.median, "min": min, "max": max}
def duration(text):
"""'15min', '1h', '6h', '1d' -> timedelta."""
m = re.fullmatch(r"(\d+)(min|h|d)", text)
if not m:
raise argparse.ArgumentTypeError(f"invalid step: {text!r} (e.g. 15min, 1h, 1d)")
n, unit = int(m.group(1)), m.group(2)
return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unit]
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
def own_noise(vs):
"""Median of the absolute differences between two consecutive values; 0 when it is below ROUNDING (values constant
up to rounding: the corr of the references when the compensation goes exactly through them, two references
retained), so that a deviation is not written as billions of noises; 0 also with fewer than two values."""
d = [abs(b - a) for a, b in zip(vs, vs[1:])]
b = statistics.median(d) if d else 0
return b if b >= ROUNDING else 0
def values(ms, col): def values(ms, col):
@@ -70,20 +44,16 @@ def main():
a.add_argument("--end", type=parse_time) a.add_argument("--end", type=parse_time)
o = a.parse_args() o = a.parse_args()
by = load(o.folder) by, names = sensors(o.folder, o.sensors)
if not by:
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
unknown = [s for s in names if s not in by] unknown = [s for s in names if s not in by]
if unknown: if unknown:
sys.exit(f"Missing sensors: {', '.join(unknown)}. Present: {', '.join(sorted(by))}.") sys.exit(f"Missing sensors: {', '.join(unknown)}. Present: {', '.join(sorted(by))}.")
t0 = min(m["t"] for s in names for m in by[s]).replace(hour=0, minute=0, second=0, microsecond=0) key = per_step(min(m["t"] for s in names for m in by[s]), o.step)
steps = {} steps = {}
for s in names: for s in names:
for m in filter_window(by[s], o.start, o.end): for m in filter_window(by[s], o.start, o.end):
step_start = t0 + ((m["t"] - t0) // o.step) * o.step steps.setdefault(key(m["t"]), {}).setdefault(s, []).append(m)
steps.setdefault(step_start, {}).setdefault(s, []).append(m)
if not steps: if not steps:
sys.exit("No measurement in the requested window.") sys.exit("No measurement in the requested window.")
@@ -104,7 +74,7 @@ def main():
if o.mark: if o.mark:
print("*: at least one measurement of the step is rejected; o: at least one measurement of the step has inlier = 0") print("*: at least one measurement of the step is rejected; o: at least one measurement of the step has inlier = 0")
header = f"{'start':17} " + " ".join(f"{s:>9}" for s in names) header = f"{'start':17} " + " ".join(f"{s:>9}" for s in names)
wanted = {t0 + ((parse_time(x) - t0) // o.step) * o.step for x in o.instants.split(",")} if o.instants else None wanted = {key(parse_time(x)) for x in o.instants.split(",")} if o.instants else None
for i, k in enumerate(k for k in sorted(steps) if wanted is None or k in wanted): for i, k in enumerate(k for k in sorted(steps) if wanted is None or k in wanted):
if i % 24 == 0: if i % 24 == 0:
print(header) print(header)
+2 -6
View File
@@ -32,18 +32,16 @@ import matplotlib
matplotlib.use("Agg") matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402 import matplotlib.pyplot as plt # noqa: E402
from common import check # noqa: E402
from data import find_files, image_time, iso, load, parse_time # noqa: E402 from data import find_files, image_time, iso, load, parse_time # noqa: E402
def percentile(grey, p): def percentile(grey, p):
"""Percentile p (0 to 100) of the pixels of a thumbnail, without interpolation.""" """Percentile p (0 to 100) of the pixels of a thumbnail, without interpolation."""
v = sorted(grey.ravel().tolist()) v = sorted(grey.ravel().tolist())
return v[min(int(p / 100 * len(v)), len(v) - 1)] return v[min(int(p / 100 * len(v)), len(v) - 1)]
def main(): def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder") a.add_argument("folder")
@@ -62,9 +60,7 @@ def main():
zone = tuple(map(int, o.zone.split(","))) if o.zone else None zone = tuple(map(int, o.zone.split(","))) if o.zone else None
if not names and not zone: if not names and not zone:
sys.exit("Give --sensors, or --zone X,Y,W,H.") sys.exit("Give --sensors, or --zone X,Y,W,H.")
missing = [s for s in names if s not in by] check("sensors", names, by)
if missing:
sys.exit(f"Unknown sensors: {', '.join(missing)}. Known: {', '.join(sorted(by))}.")
known = sorted(h for h in images if h) known = sorted(h for h in images if h)
if not known: if not known:
sys.exit(f"No image in {o.folder}.") sys.exit(f"No image in {o.folder}.")