Scripts: shared helpers in common.py, no tool imports another; CLI and outputs unchanged
data.py stays the reader (METRIC_RE now also captures the file date, same matches); common.py holds columns, steps, noise, sensor list, runs, file names. Duplicated run grouping, pose gaps, step bucketing, --table/--outside parsing removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c1719cbd1d
commit
6ed4c420e1
13 files changed
+308
-359
No files matched your search
@@ -0,0 +1,132 @@
|
|||||||
|
"""What the reading tools share beyond reading (data.py): columns, steps, noise, sensor list, runs, file names.
|
||||||
|
|
||||||
|
Standard library only: the tools without a figure run with python3, without matplotlib.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import re
|
||||||
|
import statistics
|
||||||
|
import sys
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
from data import filter_window, is_reference, load, parse_time
|
||||||
|
|
||||||
|
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
|
||||||
|
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
|
||||||
|
STATS = {"median": statistics.median, "min": min, "max": max}
|
||||||
|
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
|
||||||
|
|
||||||
|
|
||||||
|
def duration(text):
|
||||||
|
"""'15min', '1h', '6h', '1d' -> timedelta."""
|
||||||
|
m = re.fullmatch(r"(\d+)(min|h|d)", text)
|
||||||
|
if not m:
|
||||||
|
raise argparse.ArgumentTypeError(f"invalid step: {text!r} (e.g. 15min, 1h, 1d)")
|
||||||
|
n, unit = int(m.group(1)), m.group(2)
|
||||||
|
return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unit]
|
||||||
|
|
||||||
|
|
||||||
|
def own_noise(*series):
|
||||||
|
"""Median of the absolute differences between two consecutive values (within each series given); 0 when it is below
|
||||||
|
ROUNDING (values constant up to rounding: the corr of the references when the compensation goes exactly through them,
|
||||||
|
two references retained), so that a deviation is not written as billions of noises; 0 also with fewer than two values."""
|
||||||
|
d = [abs(b - a) for vs in series for a, b in zip(vs, vs[1:])]
|
||||||
|
b = statistics.median(d) if d else 0
|
||||||
|
return b if b >= ROUNDING else 0
|
||||||
|
|
||||||
|
|
||||||
|
def sensors(folder, wanted):
|
||||||
|
"""-> ({sensor: [measurement]}, the --sensors names, or all of them, references first); exits without an export."""
|
||||||
|
by = load(folder)
|
||||||
|
if not by:
|
||||||
|
sys.exit(f'No "advanced" export in {folder} (see inventory.py).')
|
||||||
|
return by, wanted.split(",") if wanted else sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
||||||
|
|
||||||
|
|
||||||
|
def check(what, requested, known):
|
||||||
|
"""Exits on the names of requested that known does not have."""
|
||||||
|
if missing := [x for x in requested if x not in known]:
|
||||||
|
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
|
||||||
|
|
||||||
|
|
||||||
|
def per_step(first, step):
|
||||||
|
"""Start of the step of each time, steps aligned on the midnight of `first` (hh:00, hh:15…)."""
|
||||||
|
midnight = first.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
return lambda t: midnight + (t - midnight) // step * step
|
||||||
|
|
||||||
|
|
||||||
|
def window_filter(text):
|
||||||
|
"""--table: '' everything, 'no' nothing, 'T1/T2' between T1 and T2 -> predicate on a time."""
|
||||||
|
if text in ("", "no"):
|
||||||
|
return lambda t: text == ""
|
||||||
|
limits = [parse_time(x) for x in text.split("/")]
|
||||||
|
if len(limits) != 2:
|
||||||
|
sys.exit("--table expects T1/T2 or no.")
|
||||||
|
return lambda t: limits[0] <= t <= limits[1]
|
||||||
|
|
||||||
|
|
||||||
|
def outside(text):
|
||||||
|
"""--outside T1/T2 -> predicate: is a time outside that moment (always true without --outside)."""
|
||||||
|
if not text:
|
||||||
|
return lambda t: True
|
||||||
|
limits = [parse_time(x) for x in text.split("/")]
|
||||||
|
if len(limits) != 2:
|
||||||
|
sys.exit("--outside expects T1/T2.")
|
||||||
|
return lambda t: not limits[0] <= t <= limits[1]
|
||||||
|
|
||||||
|
|
||||||
|
def group(items, close):
|
||||||
|
"""Sorted items -> runs (lists) of consecutive items where close(previous, next)."""
|
||||||
|
out = []
|
||||||
|
for x in items:
|
||||||
|
if out and close(out[-1][-1], x):
|
||||||
|
out[-1].append(x)
|
||||||
|
else:
|
||||||
|
out.append([x])
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def without_pose(instants, poses):
|
||||||
|
"""Runs (start, end, n) of consecutive measurement instants without a camera pose (no registration), to the minute."""
|
||||||
|
posed = {p["t"].replace(second=0, microsecond=0) for p in poses}
|
||||||
|
lacking = [(i, t) for i, t in enumerate(instants) if t.replace(second=0, microsecond=0) not in posed]
|
||||||
|
return [(g[0][1], g[-1][1], len(g)) for g in group(lacking, lambda a, b: b[0] == a[0] + 1)]
|
||||||
|
|
||||||
|
|
||||||
|
def extremes(d):
|
||||||
|
"""{t: value} -> ((t, value) of the minimum, (t, value) of the maximum)."""
|
||||||
|
return min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
|
||||||
|
|
||||||
|
|
||||||
|
def targets(by_sensor, col, start, end):
|
||||||
|
"""At each measurement, median over the targets of col minus the target's own level (median over the window)."""
|
||||||
|
deviations = {}
|
||||||
|
for s, ms in by_sensor.items():
|
||||||
|
if is_reference(ms):
|
||||||
|
continue
|
||||||
|
ms = [m for m in filter_window(ms, start, end) if m[col] is not None]
|
||||||
|
if ms:
|
||||||
|
level = statistics.median(m[col] for m in ms)
|
||||||
|
for m in ms:
|
||||||
|
deviations.setdefault(m["t"], []).append(m[col] - level)
|
||||||
|
return [{"t": t, "v": statistics.median(v)} for t, v in sorted(deviations.items())]
|
||||||
|
|
||||||
|
|
||||||
|
def cut(series):
|
||||||
|
"""Times and numeric values, NaN inserted over a gap of more than 1.5 times the usual step (the line breaks)."""
|
||||||
|
pts = [(p["t"], p["v"]) for p in series if isinstance(p["v"], float)]
|
||||||
|
if len(pts) < 3:
|
||||||
|
return [t for t, _ in pts], [v for _, v in pts]
|
||||||
|
step = statistics.median(b[0] - a[0] for a, b in zip(pts, pts[1:]))
|
||||||
|
ts, vs = [pts[0][0]], [pts[0][1]]
|
||||||
|
for (a, _), (b, v) in zip(pts, pts[1:]):
|
||||||
|
if b - a > 1.5 * step:
|
||||||
|
ts.append(a + (b - a) / 2)
|
||||||
|
vs.append(float("nan"))
|
||||||
|
ts.append(b)
|
||||||
|
vs.append(v)
|
||||||
|
return ts, vs
|
||||||
|
|
||||||
|
|
||||||
|
def bound(t, default):
|
||||||
|
"""--start/--end in a file name."""
|
||||||
|
return t.strftime("%Y%m%dT%H%MZ") if t else default
|
||||||
+2
-2
@@ -1,4 +1,4 @@
|
|||||||
"""Reading a station's data for the diagnostic agent (standard library).
|
"""Reading a station folder (standard library only): the one reader of the app, the reading tools and the detectors.
|
||||||
|
|
||||||
Formats: tools/DATA.md. Every time is UTC.
|
Formats: tools/DATA.md. Every time is UTC.
|
||||||
"""
|
"""
|
||||||
@@ -98,7 +98,7 @@ def load_camera(folder):
|
|||||||
|
|
||||||
|
|
||||||
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
|
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
|
||||||
METRIC_RE = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
|
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_") # name, then the date of the file (one CSV per metric and per day)
|
||||||
|
|
||||||
|
|
||||||
def load_metrics(folder):
|
def load_metrics(folder):
|
||||||
|
|||||||
+38
-46
@@ -55,9 +55,8 @@ import matplotlib
|
|||||||
matplotlib.use("Agg")
|
matplotlib.use("Agg")
|
||||||
import matplotlib.pyplot as plt # noqa: E402
|
import matplotlib.pyplot as plt # noqa: E402
|
||||||
|
|
||||||
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402
|
from common import COLUMNS, bound, check, duration, own_noise, sensors # noqa: E402
|
||||||
from plot import COLUMNS, bound # noqa: E402
|
from data import POSE, filter_window, is_reference, iso, load_camera, parse_time, rejected # noqa: E402
|
||||||
from summary import ROUNDING, duration, own_noise # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
def hours(text):
|
def hours(text):
|
||||||
@@ -73,6 +72,27 @@ def at_night(t, h1, h2):
|
|||||||
return (h1 <= t.hour or t.hour < h2) if h1 > h2 else h1 <= t.hour < h2
|
return (h1 <= t.hour or t.hour < h2) if h1 > h2 else h1 <= t.hour < h2
|
||||||
|
|
||||||
|
|
||||||
|
def nights_of(pts, h1, h2):
|
||||||
|
"""[(t, v)] -> {date of the night: [(t, v)]}: the night hours, dated by the day the night starts."""
|
||||||
|
nights = {}
|
||||||
|
for t, v in pts:
|
||||||
|
if at_night(t, h1, h2):
|
||||||
|
nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
|
||||||
|
return nights
|
||||||
|
|
||||||
|
|
||||||
|
def frame(ax, values):
|
||||||
|
"""Frame from the 1st to the 99th percentile widened by half: an isolated peak does not crush the scale; -> the number
|
||||||
|
of values out of frame."""
|
||||||
|
v = sorted(values)
|
||||||
|
if not v:
|
||||||
|
return 0
|
||||||
|
q1, q2 = v[len(v) // 100], v[-1 - len(v) // 100]
|
||||||
|
margin = (q2 - q1) * 0.5 or 1
|
||||||
|
ax.set_ylim(q1 - margin, q2 + margin)
|
||||||
|
return sum(not q1 - margin <= e <= q2 + margin for e in v)
|
||||||
|
|
||||||
|
|
||||||
def cut_index(pts, la, lb):
|
def cut_index(pts, la, lb):
|
||||||
"""Index j (1..n-1) where sum|v-la| before j + sum|v-lb| from j on is smallest."""
|
"""Index j (1..n-1) where sum|v-la| before j + sum|v-lb| from j on is smallest."""
|
||||||
before, after = 0.0, sum(abs(v - lb) for _, v in pts)
|
before, after = 0.0, sum(abs(v - lb) for _, v in pts)
|
||||||
@@ -103,15 +123,10 @@ def main():
|
|||||||
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
|
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
|
||||||
o = a.parse_args()
|
o = a.parse_args()
|
||||||
|
|
||||||
by_sensor = load(o.folder)
|
by_sensor, names = sensors(o.folder, o.sensors)
|
||||||
if not by_sensor:
|
|
||||||
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
|
|
||||||
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
|
|
||||||
columns = o.columns.split(",")
|
columns = o.columns.split(",")
|
||||||
for requested, known, what in ((names, by_sensor, "sensors"), (columns, COLUMNS, "columns")):
|
check("sensors", names, by_sensor)
|
||||||
missing = [x for x in requested if x not in known]
|
check("columns", columns, COLUMNS)
|
||||||
if missing:
|
|
||||||
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
|
|
||||||
series = {s: [m for m in filter_window(by_sensor[s], o.start, o.end) if not rejected(m)] for s in names}
|
series = {s: [m for m in filter_window(by_sensor[s], o.start, o.end) if not rejected(m)] for s in names}
|
||||||
if not any(series.values()):
|
if not any(series.values()):
|
||||||
sys.exit("No accepted measurement in the requested window.")
|
sys.exit("No accepted measurement in the requested window.")
|
||||||
@@ -133,10 +148,7 @@ def main():
|
|||||||
ax.axis("off")
|
ax.axis("off")
|
||||||
continue
|
continue
|
||||||
noise = noises[s] = own_noise([v for _, v in ms])
|
noise = noises[s] = own_noise([v for _, v in ms])
|
||||||
nights = {}
|
nights = nights_of(ms, h1, h2)
|
||||||
for t, v in ms:
|
|
||||||
if at_night(t, h1, h2):
|
|
||||||
nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
|
|
||||||
lvl = {d: statistics.median(v for _, v in p) for d, p in nights.items()}
|
lvl = {d: statistics.median(v for _, v in p) for d, p in nights.items()}
|
||||||
cells = []
|
cells = []
|
||||||
for d in sorted(nights):
|
for d in sorted(nights):
|
||||||
@@ -167,11 +179,7 @@ def main():
|
|||||||
for i, (d, p) in enumerate(sorted(per_day.items())):
|
for i, (d, p) in enumerate(sorted(per_day.items())):
|
||||||
ax.plot([x for x, _ in p], [e for _, e in p], "-", lw=0.8, color=f"C{i % 10}", label=f"{d:%d/%m}")
|
ax.plot([x for x, _ in p], [e for _, e in p], "-", lw=0.8, color=f"C{i % 10}", label=f"{d:%d/%m}")
|
||||||
ax.axhline(0, color="0.3", ls=":", lw=0.8)
|
ax.axhline(0, color="0.3", ls=":", lw=0.8)
|
||||||
every = sorted(e for p in per_day.values() for _, e in p)
|
out_of_frame = frame(ax, (e for p in per_day.values() for _, e in p))
|
||||||
q1, q2 = every[len(every) // 100], every[-1 - len(every) // 100] # frame from the 1st to the 99th percentile: an isolated peak does not crush the scale
|
|
||||||
margin = (q2 - q1) * 0.5 or 1
|
|
||||||
ax.set_ylim(q1 - margin, q2 + margin)
|
|
||||||
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in every)
|
|
||||||
for x0, x1 in ((h1, 24), (0, h2)) if h1 > h2 else ((h1, h2),):
|
for x0, x1 in ((h1, 24), (0, h2)) if h1 > h2 else ((h1, h2),):
|
||||||
ax.axvspan(x0, x1, color="0.9", lw=0)
|
ax.axvspan(x0, x1, color="0.9", lw=0)
|
||||||
ax.set_title(s + (" (ref.)" if is_reference(series[s]) else "") + f" noise {noise:.2g}"
|
ax.set_title(s + (" (ref.)" if is_reference(series[s]) else "") + f" noise {noise:.2g}"
|
||||||
@@ -217,15 +225,6 @@ def main():
|
|||||||
day_bounds(o, names, series, columns, station)
|
day_bounds(o, names, series, columns, station)
|
||||||
|
|
||||||
|
|
||||||
def night_levels(pts, h1, h2):
|
|
||||||
"""[(t, v)] -> {date of the night: median}."""
|
|
||||||
nights = {}
|
|
||||||
for t, v in pts:
|
|
||||||
if at_night(t, h1, h2):
|
|
||||||
nights.setdefault((t - timedelta(hours=h1)).date(), []).append(v)
|
|
||||||
return {d: statistics.median(v) for d, v in nights.items()}
|
|
||||||
|
|
||||||
|
|
||||||
def day_bounds(o, names, series, columns, station):
|
def day_bounds(o, names, series, columns, station):
|
||||||
"""Section 3: per day, the deviation from the level of the night that precedes it, for the target median and the pose."""
|
"""Section 3: per day, the deviation from the level of the night that precedes it, for the target median and the pose."""
|
||||||
h1, h2 = o.night
|
h1, h2 = o.night
|
||||||
@@ -240,7 +239,10 @@ def day_bounds(o, names, series, columns, station):
|
|||||||
t0, t1 = min(every), max(every)
|
t0, t1 = min(every), max(every)
|
||||||
night_of = lambda t: (t - night_length - timedelta(hours=h1)).date() # night that ends before t (the day of t starts at its end)
|
night_of = lambda t: (t - night_length - timedelta(hours=h1)).date() # night that ends before t (the day of t starts at its end)
|
||||||
days = [night_of(t0) + timedelta(days=i) for i in range((night_of(t1) - night_of(t0)).days + 1)]
|
days = [night_of(t0) + timedelta(days=i) for i in range((night_of(t1) - night_of(t0)).days + 1)]
|
||||||
hm = lambda t: f"{t:%d/%m %H:%M}"
|
hm = lambda t: f"{t:%d/%m %H:%M}" # noqa: E731
|
||||||
|
span = lambda p, i, j: (f"from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) " # noqa: E731
|
||||||
|
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
|
||||||
|
exceed = "threshold" if o.fraction else f"{o.k:g} noises"
|
||||||
threshold_txt = f"threshold {o.fraction:g} times the largest |deviation| of the day" if o.fraction else f"threshold {o.k:g} noises"
|
threshold_txt = f"threshold {o.fraction:g} times the largest |deviation| of the day" if o.fraction else f"threshold {o.k:g} noises"
|
||||||
print(f"\n== Bounds of each day: deviation from the level of the night that precedes it ({h1} h → {h2} h UTC), {threshold_txt}"
|
print(f"\n== Bounds of each day: deviation from the level of the night that precedes it ({h1} h → {h2} h UTC), {threshold_txt}"
|
||||||
+ (f", deviation smoothed over {o.smoothing} measurements" if o.smoothing > 1 else "") + " ==")
|
+ (f", deviation smoothed over {o.smoothing} measurements" if o.smoothing > 1 else "") + " ==")
|
||||||
@@ -248,7 +250,7 @@ def day_bounds(o, names, series, columns, station):
|
|||||||
"exceedance, just after the last; next night: its level minus that of the reference night")
|
"exceedance, just after the last; next night: its level minus that of the reference night")
|
||||||
fig, axes = plt.subplots(len(seqs), 1, sharex=True, figsize=(18, 2.2 * len(seqs)), squeeze=False)
|
fig, axes = plt.subplots(len(seqs), 1, sharex=True, figsize=(18, 2.2 * len(seqs)), squeeze=False)
|
||||||
for ax, (name, short, per_sensor) in zip(axes.flat, seqs):
|
for ax, (name, short, per_sensor) in zip(axes.flat, seqs):
|
||||||
lvl = {s: night_levels(p, h1, h2) for s, p in per_sensor.items()}
|
lvl = {s: {d: statistics.median(v for _, v in n) for d, n in nights_of(p, h1, h2).items()} for s, p in per_sensor.items()}
|
||||||
nights = sorted({d for n in lvl.values() for d in n})
|
nights = sorted({d for n in lvl.values() for d in n})
|
||||||
devs = [] # (day, ref. night, [(t, deviation)], deviation of the next night)
|
devs = [] # (day, ref. night, [(t, deviation)], deviation of the next night)
|
||||||
for d in days:
|
for d in days:
|
||||||
@@ -264,9 +266,7 @@ def day_bounds(o, names, series, columns, station):
|
|||||||
next_n = [lvl[s][d + timedelta(days=1)] - lvl[s][ref] for s in lvl if ref in lvl[s] and d + timedelta(days=1) in lvl[s]]
|
next_n = [lvl[s][d + timedelta(days=1)] - lvl[s][ref] for s in lvl if ref in lvl[s] and d + timedelta(days=1) in lvl[s]]
|
||||||
devs.append((start.date(), d, ref, [(t, statistics.median(e)) for t, e in sorted(per_t.items())],
|
devs.append((start.date(), d, ref, [(t, statistics.median(e)) for t, e in sorted(per_t.items())],
|
||||||
statistics.median(next_n) if next_n else None))
|
statistics.median(next_n) if next_n else None))
|
||||||
diffs = [abs(b[1] - a_[1]) for *_, p, _ in devs for a_, b in zip(p, p[1:])]
|
noise = own_noise(*[[e for _, e in p] for *_, p, _ in devs])
|
||||||
noise = statistics.median(diffs) if diffs else 0
|
|
||||||
noise = noise if noise >= ROUNDING else 0
|
|
||||||
print(f"\n{name}: noise {noise:.3g}")
|
print(f"\n{name}: noise {noise:.3g}")
|
||||||
in_noises = lambda e: f"{e:+.4g} ({e / noise:+.1f})" if noise else f"{e:+.4g}"
|
in_noises = lambda e: f"{e:+.4g} ({e / noise:+.1f})" if noise else f"{e:+.4g}"
|
||||||
h = o.smoothing // 2
|
h = o.smoothing // 2
|
||||||
@@ -281,11 +281,10 @@ def day_bounds(o, names, series, columns, station):
|
|||||||
over = [i for i, (_, e) in enumerate(p) if threshold and abs(e) > threshold]
|
over = [i for i, (_, e) in enumerate(p) if threshold and abs(e) > threshold]
|
||||||
if over:
|
if over:
|
||||||
i, j = over[0], over[-1]
|
i, j = over[0], over[-1]
|
||||||
line += (f"> {'threshold' if o.fraction else f'{o.k:g} noises'}: {len(over)} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) "
|
line += f"> {exceed}: {len(over)} measurements, {span(p, i, j)}"
|
||||||
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
|
|
||||||
ax.axvspan(p[i][0], p[j][0], color="0.85", lw=0)
|
ax.axvspan(p[i][0], p[j][0], color="0.85", lw=0)
|
||||||
else:
|
else:
|
||||||
line += f"never > {'threshold' if o.fraction else f'{o.k:g} noises'}"
|
line += f"never > {exceed}"
|
||||||
top, bottom = max(p, key=lambda x: x[1]), min(p, key=lambda x: x[1])
|
top, bottom = max(p, key=lambda x: x[1]), min(p, key=lambda x: x[1])
|
||||||
line += f" | max {in_noises(top[1])} at {hm(top[0])}, min {in_noises(bottom[1])} at {hm(bottom[0])}"
|
line += f" | max {in_noises(top[1])} at {hm(top[0])}, min {in_noises(bottom[1])} at {hm(bottom[0])}"
|
||||||
x = max(range(len(p)), key=lambda i: abs(p[i][1]))
|
x = max(range(len(p)), key=lambda i: abs(p[i][1]))
|
||||||
@@ -296,8 +295,7 @@ def day_bounds(o, names, series, columns, station):
|
|||||||
i -= 1
|
i -= 1
|
||||||
while j + 1 < len(p) and sign * p[j + 1][1] > threshold:
|
while j + 1 < len(p) and sign * p[j + 1][1] > threshold:
|
||||||
j += 1
|
j += 1
|
||||||
line += (f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) "
|
line += f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, {span(p, i, j)}"
|
||||||
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
|
|
||||||
ax.axvspan(p[i][0], p[j][0], color="0.7", lw=0)
|
ax.axvspan(p[i][0], p[j][0], color="0.7", lw=0)
|
||||||
line += f" | next night {in_noises(nxt)}" if nxt is not None else " | next night not measured"
|
line += f" | next night {in_noises(nxt)}" if nxt is not None else " | next night not measured"
|
||||||
print(line)
|
print(line)
|
||||||
@@ -309,13 +307,7 @@ def day_bounds(o, names, series, columns, station):
|
|||||||
for y in (o.k * noise, -o.k * noise):
|
for y in (o.k * noise, -o.k * noise):
|
||||||
ax.axhline(y, color="0.4", ls=":", lw=0.8)
|
ax.axhline(y, color="0.4", ls=":", lw=0.8)
|
||||||
ax.axhline(0, color="0.3", lw=0.5)
|
ax.axhline(0, color="0.3", lw=0.5)
|
||||||
all_e = sorted(e for *_, p, _ in devs for _, e in p)
|
out_of_frame = frame(ax, (e for *_, p, _ in devs for _, e in p))
|
||||||
out_of_frame = 0
|
|
||||||
if all_e: # frame from the 1st to the 99th percentile widened by half, as in byhour: an isolated peak does not crush the scale
|
|
||||||
q1, q2 = all_e[len(all_e) // 100], all_e[-1 - len(all_e) // 100]
|
|
||||||
margin = (q2 - q1) * 0.5 or 1
|
|
||||||
ax.set_ylim(q1 - margin, q2 + margin)
|
|
||||||
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in all_e)
|
|
||||||
ax.set_title(f"{name} noise {noise:.2g}" + (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9)
|
ax.set_title(f"{name} noise {noise:.2g}" + (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9)
|
||||||
ax.grid(alpha=0.3)
|
ax.grid(alpha=0.3)
|
||||||
ax.tick_params(labelsize=7)
|
ax.tick_params(labelsize=7)
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ from PIL import Image, ImageDraw, ImageFont
|
|||||||
from data import find_files, image_time, iso, parse_time
|
from data import find_files, image_time, iso, parse_time
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
a.add_argument("folder")
|
a.add_argument("folder")
|
||||||
|
|||||||
+21
-44
@@ -20,14 +20,11 @@ import collections
|
|||||||
import csv
|
import csv
|
||||||
import glob
|
import glob
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import statistics
|
import statistics
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from data import filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected
|
from common import QUALITY, group, without_pose
|
||||||
from summary import QUALITY
|
from data import METRIC_RE, filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected
|
||||||
|
|
||||||
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_\d{2}_\d{2}") # NIEDERHORN_BusCurrent_2026_09_22_00_00 to ….csv
|
|
||||||
|
|
||||||
|
|
||||||
def span(a, b, n):
|
def span(a, b, n):
|
||||||
@@ -158,17 +155,17 @@ def images(paths, start=None, end=None):
|
|||||||
return
|
return
|
||||||
# Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot):
|
# Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot):
|
||||||
# one line per change of gap, without losing any of the times covered.
|
# one line per change of gap, without losing any of the times covered.
|
||||||
runs = [] # [first, last, gap in min, number of gaps]
|
pairs = [(a, b, round((b - a).total_seconds() / 300) * 5) for a, b in zip(times, times[1:])]
|
||||||
for a, b in zip(times, times[1:]):
|
|
||||||
e = round((b - a).total_seconds() / 300) * 5
|
|
||||||
if runs and runs[-1][2] == e:
|
|
||||||
runs[-1][1], runs[-1][3] = b, runs[-1][3] + 1
|
|
||||||
else:
|
|
||||||
runs.append([a, b, e, 1])
|
|
||||||
what = "over the window" if start or end else "between the first and the last"
|
what = "over the window" if start or end else "between the first and the last"
|
||||||
print(f" gap between two images {what}, per run of equal gaps (within 5 min):")
|
print(f" gap between two images {what}, per run of equal gaps (within 5 min):")
|
||||||
for a, b, e, n in runs:
|
for g in group(pairs, lambda p, q: p[2] == q[2]):
|
||||||
print(f" {iso(a)} → {iso(b)}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min"))
|
n, e = len(g), g[0][2]
|
||||||
|
print(f" {iso(g[0][0])} → {iso(g[-1][1])}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min"))
|
||||||
|
|
||||||
|
|
||||||
|
def missing(instants, cadence):
|
||||||
|
"""Gaps between two present instants: [(before, after, number of absent steps)]."""
|
||||||
|
return [(a, b, n) for a, b in zip(instants, instants[1:]) if (n := round((b - a) / cadence) - 1) > 0]
|
||||||
|
|
||||||
|
|
||||||
def measurements(by):
|
def measurements(by):
|
||||||
@@ -179,8 +176,7 @@ def measurements(by):
|
|||||||
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
|
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
|
||||||
print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants")
|
print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants")
|
||||||
print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min")
|
print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min")
|
||||||
gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(instants, instants[1:])]
|
gaps = missing(instants, cadence)
|
||||||
gaps = [x for x in gaps if x[2] > 0]
|
|
||||||
print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}")
|
print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}")
|
||||||
for a, b, n in gaps:
|
for a, b, n in gaps:
|
||||||
print(f" between {iso(a)} and {iso(b)}: {n}")
|
print(f" between {iso(a)} and {iso(b)}: {n}")
|
||||||
@@ -207,19 +203,11 @@ def camera(folder, by, start=None, end=None):
|
|||||||
if not poses:
|
if not poses:
|
||||||
print(f"\nCamera pose {what}: none")
|
print(f"\nCamera pose {what}: none")
|
||||||
return
|
return
|
||||||
with_pose = {p["t"].replace(second=0, microsecond=0) for p in poses}
|
|
||||||
print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}")
|
print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}")
|
||||||
runs = [] # [start, end, count, rank] of the consecutive measurement instants without a pose
|
runs = without_pose(instants, poses)
|
||||||
for i, t in enumerate(instants):
|
|
||||||
if t.replace(second=0, microsecond=0) in with_pose:
|
|
||||||
continue
|
|
||||||
if runs and runs[-1][3] == i - 1:
|
|
||||||
runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i
|
|
||||||
else:
|
|
||||||
runs.append([t, t, 1, i])
|
|
||||||
print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}")
|
print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}")
|
||||||
if start or end:
|
if start or end:
|
||||||
for a, b, n, _ in runs:
|
for a, b, n in runs:
|
||||||
print(f" {span(a, b, n)}")
|
print(f" {span(a, b, n)}")
|
||||||
|
|
||||||
|
|
||||||
@@ -232,8 +220,7 @@ def window(by, start, end):
|
|||||||
print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present")
|
print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present")
|
||||||
names = sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
names = sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
||||||
print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):")
|
print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):")
|
||||||
gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(inside, inside[1:])]
|
gaps = missing(inside, cadence)
|
||||||
gaps = [x for x in gaps if x[2] > 0]
|
|
||||||
for a, b, n in gaps:
|
for a, b, n in gaps:
|
||||||
print(f" between {iso(a)} and {iso(b)}: {n}")
|
print(f" between {iso(a)} and {iso(b)}: {n}")
|
||||||
if not gaps:
|
if not gaps:
|
||||||
@@ -248,14 +235,8 @@ def window(by, start, end):
|
|||||||
print("Rejected measurements in the window, per sensor (start → end of each run of consecutive steps, count):")
|
print("Rejected measurements in the window, per sensor (start → end of each run of consecutive steps, count):")
|
||||||
for s in names:
|
for s in names:
|
||||||
rej = [t for t in sorted(series[s]) if rejected(series[s][t])]
|
rej = [t for t in sorted(series[s]) if rejected(series[s][t])]
|
||||||
runs = []
|
|
||||||
for t in rej:
|
|
||||||
if runs and t - runs[-1][1] <= cadence * 1.5:
|
|
||||||
runs[-1][1], runs[-1][2] = t, runs[-1][2] + 1
|
|
||||||
else:
|
|
||||||
runs.append([t, t, 1])
|
|
||||||
if rej:
|
if rej:
|
||||||
print(f" {s} ({len(rej)}): " + "; ".join(span(a, b, n) for a, b, n in runs))
|
print(f" {s} ({len(rej)}): " + "; ".join(span(g[0], g[-1], len(g)) for g in group(rej, lambda a, b: b - a <= cadence * 1.5)))
|
||||||
if not any(rejected(m) for d in series.values() for m in d.values()):
|
if not any(rejected(m) for d in series.values() for m in d.values()):
|
||||||
print(" (none)")
|
print(" (none)")
|
||||||
refs = [s for s in names if is_reference(by[s])]
|
refs = [s for s in names if is_reference(by[s])]
|
||||||
@@ -276,14 +257,10 @@ def window(by, start, end):
|
|||||||
+ ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True)))
|
+ ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True)))
|
||||||
print("Instants where references are not retained (runs of consecutive steps with the same references out; "
|
print("Instants where references are not retained (runs of consecutive steps with the same references out; "
|
||||||
f"r = rejected, empty = {col} empty, otherwise the inlier value):")
|
f"r = rejected, empty = {col} empty, otherwise the inlier value):")
|
||||||
runs = []
|
runs = group([(i, t, h) for i, (t, h) in enumerate(out) if h], lambda a, b: b[0] == a[0] + 1 and b[2] == a[2])
|
||||||
for i, (t, h) in enumerate(out):
|
for g in runs:
|
||||||
if h and runs and runs[-1][3] == h and runs[-1][4] == i - 1:
|
h = g[0][2]
|
||||||
runs[-1][1], runs[-1][2], runs[-1][4] = t, runs[-1][2] + 1, i
|
print(f" {span(g[0][1], g[-1][1], len(g))}: {len(refs) - len(h)} retained; out: "
|
||||||
elif h:
|
|
||||||
runs.append([t, t, 1, h, i])
|
|
||||||
for a, b, n, h, _ in runs:
|
|
||||||
print(f" {span(a, b, n)}: {len(refs) - len(h)} retained; out: "
|
|
||||||
+ ", ".join(f"{s} ({q})" for s, q in h))
|
+ ", ".join(f"{s} ({q})" for s, q in h))
|
||||||
if not runs:
|
if not runs:
|
||||||
print(" (none)")
|
print(" (none)")
|
||||||
|
|||||||
+4
-47
@@ -34,10 +34,8 @@ matplotlib.use("Agg")
|
|||||||
import matplotlib.dates as mdates # noqa: E402
|
import matplotlib.dates as mdates # noqa: E402
|
||||||
import matplotlib.pyplot as plt # noqa: E402
|
import matplotlib.pyplot as plt # noqa: E402
|
||||||
|
|
||||||
from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time # noqa: E402
|
from common import STATS, bound, cut, duration, group, targets # noqa: E402
|
||||||
from inventory import METRIC_RE # noqa: E402
|
from data import METRIC_RE, POSE, filter_window, iso, load, load_camera, load_temperature, parse_time # noqa: E402
|
||||||
from plot import bound # noqa: E402
|
|
||||||
from summary import STATS, duration # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
def is_identifier(name):
|
def is_identifier(name):
|
||||||
@@ -76,20 +74,6 @@ def read_metrics(folder, start, end):
|
|||||||
return {n: filter_window([{"t": t, "v": s[t]} for t in sorted(s)], start, end) for n, s in by_name.items()}
|
return {n: filter_window([{"t": t, "v": s[t]} for t in sorted(s)], start, end) for n, s in by_name.items()}
|
||||||
|
|
||||||
|
|
||||||
def targets(by_sensor, col, start, end):
|
|
||||||
"""At each measurement, median over the targets of col minus the target's own level (median over the window)."""
|
|
||||||
deviations = {}
|
|
||||||
for s, ms in by_sensor.items():
|
|
||||||
if is_reference(ms):
|
|
||||||
continue
|
|
||||||
ms = [m for m in filter_window(ms, start, end) if m[col] is not None]
|
|
||||||
if ms:
|
|
||||||
level = statistics.median(m[col] for m in ms)
|
|
||||||
for m in ms:
|
|
||||||
deviations.setdefault(m["t"], []).append(m[col] - level)
|
|
||||||
return [{"t": t, "v": statistics.median(v)} for t, v in sorted(deviations.items())]
|
|
||||||
|
|
||||||
|
|
||||||
def smooth(series, width):
|
def smooth(series, width):
|
||||||
"""Moving average centred over width (numeric values)."""
|
"""Moving average centred over width (numeric values)."""
|
||||||
pts = [p for p in series if isinstance(p["v"], float)]
|
pts = [p for p in series if isinstance(p["v"], float)]
|
||||||
@@ -101,33 +85,6 @@ def fmt(v):
|
|||||||
return "-" if v is None else f"{v:.6g}" if isinstance(v, float) else str(v)
|
return "-" if v is None else f"{v:.6g}" if isinstance(v, float) else str(v)
|
||||||
|
|
||||||
|
|
||||||
def runs(empty_steps, step):
|
|
||||||
"""[t] sorted -> [(start, end, n)] of consecutive steps."""
|
|
||||||
out = []
|
|
||||||
for t in empty_steps:
|
|
||||||
if out and t - out[-1][1] == step:
|
|
||||||
out[-1] = (out[-1][0], t, out[-1][2] + 1)
|
|
||||||
else:
|
|
||||||
out.append((t, t, 1))
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def cut(series):
|
|
||||||
"""Times and numeric values, NaN inserted over a gap of more than 1.5 times the usual step (the line breaks)."""
|
|
||||||
pts = [(p["t"], p["v"]) for p in series if isinstance(p["v"], float)]
|
|
||||||
if len(pts) < 3:
|
|
||||||
return [t for t, _ in pts], [v for _, v in pts]
|
|
||||||
step = statistics.median(b[0] - a[0] for a, b in zip(pts, pts[1:]))
|
|
||||||
ts, vs = [pts[0][0]], [pts[0][1]]
|
|
||||||
for (a, _), (b, v) in zip(pts, pts[1:]):
|
|
||||||
if b - a > 1.5 * step:
|
|
||||||
ts.append(a + (b - a) / 2)
|
|
||||||
vs.append(float("nan"))
|
|
||||||
ts.append(b)
|
|
||||||
vs.append(v)
|
|
||||||
return ts, vs
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
a.add_argument("folder")
|
a.add_argument("folder")
|
||||||
@@ -229,8 +186,8 @@ def main():
|
|||||||
|
|
||||||
empty = [k for k in grid if all(k not in table.get(c, {}) for c in metric_cols)]
|
empty = [k for k in grid if all(k not in table.get(c, {}) for c in metric_cols)]
|
||||||
print(f"\nSteps without any metric ({', '.join(metric_cols)}): {len(empty)} of {len(grid)}")
|
print(f"\nSteps without any metric ({', '.join(metric_cols)}): {len(empty)} of {len(grid)}")
|
||||||
for d, f, n in runs(empty, step):
|
for g in group(empty, lambda a, b: b - a == step):
|
||||||
print(f" {iso(d)} → {iso(f)} ({n} steps)")
|
print(f" {iso(g[0])} → {iso(g[-1])} ({len(g)} steps)")
|
||||||
|
|
||||||
panels = {} # title -> columns
|
panels = {} # title -> columns
|
||||||
for c in columns:
|
for c in columns:
|
||||||
|
|||||||
+75
-111
@@ -62,9 +62,8 @@ matplotlib.use("Agg")
|
|||||||
import matplotlib.dates as mdates # noqa: E402
|
import matplotlib.dates as mdates # noqa: E402
|
||||||
import matplotlib.pyplot as plt # noqa: E402
|
import matplotlib.pyplot as plt # noqa: E402
|
||||||
|
|
||||||
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402
|
from common import COLUMNS, bound, check, duration, extremes, outside, own_noise, per_step, sensors, window_filter, without_pose # noqa: E402
|
||||||
from plot import COLUMNS, bound # noqa: E402
|
from data import POSE, filter_window, is_reference, iso, load_camera, parse_time, rejected # noqa: E402
|
||||||
from summary import duration, own_noise # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
def level(ms, col):
|
def level(ms, col):
|
||||||
@@ -166,11 +165,9 @@ def rigid(o, names, series, times, base, station):
|
|||||||
+ f".\nResidual = raw position minus the position at {iso(t_b)} carried by this movement [px]; rejected measurements left out of the fit, "
|
+ f".\nResidual = raw position minus the position at {iso(t_b)} carried by this movement [px]; rejected measurements left out of the fit, "
|
||||||
f"of tgt_med, tgt_max and of the sensor columns (separately in the per-sensor summary); median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
|
f"of tgt_med, tgt_max and of the sensor columns (separately in the per-sensor summary); median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
|
||||||
cols = ["rot_deg", "scale", "tx", "ty", "tgt_med", "tgt_max"] + [f"{s}_{c}" for s in refs + followed for c in ("dx", "dy")]
|
cols = ["rot_deg", "scale", "tx", "ty", "tgt_med", "tgt_max"] + [f"{s}_{c}" for s in refs + followed for c in ("dx", "dy")]
|
||||||
midnight = tf[0].replace(hour=0, minute=0, second=0, microsecond=0)
|
key, rows = per_step(tf[0], o.step), {}
|
||||||
t0 = midnight + ((tf[0] - midnight) // o.step) * o.step
|
|
||||||
per_step = {}
|
|
||||||
for t in tf:
|
for t in tf:
|
||||||
line = per_step.setdefault(t0 + ((t - t0) // o.step) * o.step, {})
|
line = rows.setdefault(key(t), {})
|
||||||
for c, v in zip(cols, fit[t]):
|
for c, v in zip(cols, fit[t]):
|
||||||
line.setdefault(c, []).append(v)
|
line.setdefault(c, []).append(v)
|
||||||
for s in refs + followed:
|
for s in refs + followed:
|
||||||
@@ -178,10 +175,10 @@ def rigid(o, names, series, times, base, station):
|
|||||||
line.setdefault(f"{s}_dx", []).append(res[s][t][0]), line.setdefault(f"{s}_dy", []).append(res[s][t][1])
|
line.setdefault(f"{s}_dx", []).append(res[s][t][0]), line.setdefault(f"{s}_dy", []).append(res[s][t][1])
|
||||||
w = max(10, max(len(c) for c in cols) + 1)
|
w = max(10, max(len(c) for c in cols) + 1)
|
||||||
header = f"{'time':17}" + "".join(f"{c:>{w}}" for c in cols)
|
header = f"{'time':17}" + "".join(f"{c:>{w}}" for c in cols)
|
||||||
for i, t in enumerate(t for t in sorted(per_step) if o.in_table(t)):
|
for i, t in enumerate(t for t in sorted(rows) if o.in_table(t)):
|
||||||
if i % 24 == 0:
|
if i % 24 == 0:
|
||||||
print(header)
|
print(header)
|
||||||
print(f"{iso(t):17}" + "".join(f"{statistics.median(per_step[t][c]):{w}.{7 if c == 'scale' else 3}g}" if c in per_step[t] else " " * w for c in cols))
|
print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):{w}.{7 if c == 'scale' else 3}g}" if c in rows[t] else " " * w for c in cols))
|
||||||
print("\nPer sensor, accepted measurements: min and max of dx, dy, value at the last time; then the rejected ones separately:")
|
print("\nPer sensor, accepted measurements: min and max of dx, dy, value at the last time; then the rejected ones separately:")
|
||||||
for s in names:
|
for s in names:
|
||||||
good = {t: v for t, v in res[s].items() if t in acc[s]}
|
good = {t: v for t, v in res[s].items() if t in acc[s]}
|
||||||
@@ -189,15 +186,15 @@ def rigid(o, names, series, times, base, station):
|
|||||||
for k, c in ((0, "dx"), (1, "dy")):
|
for k, c in ((0, "dx"), (1, "dy")):
|
||||||
line = f"{s:9}{' (ref.)' if s in refs else ' '} {c}"
|
line = f"{s:9}{' (ref.)' if s in refs else ' '} {c}"
|
||||||
if good:
|
if good:
|
||||||
lo, hi = min(good.items(), key=lambda x: x[1][k]), max(good.items(), key=lambda x: x[1][k])
|
lo, hi = extremes({t: v[k] for t, v in good.items()})
|
||||||
last = res[s].get(tf[-1])
|
last = res[s].get(tf[-1])
|
||||||
line += (f" min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}, at the last time "
|
line += (f" min {lo[1]:+.3g} at {iso(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}, at the last time "
|
||||||
f"{last[k] if last else math.nan:+.3g}{' (rejected)' if last and tf[-1] not in acc[s] else ''}")
|
f"{last[k] if last else math.nan:+.3g}{' (rejected)' if last and tf[-1] not in acc[s] else ''}")
|
||||||
else:
|
else:
|
||||||
line += " no accepted measurement"
|
line += " no accepted measurement"
|
||||||
if rej:
|
if rej:
|
||||||
lo, hi = min(rej.items(), key=lambda x: x[1][k]), max(rej.items(), key=lambda x: x[1][k])
|
lo, hi = extremes({t: v[k] for t, v in rej.items()})
|
||||||
line += f"; {len(rej)} rejected: min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}"
|
line += f"; {len(rej)} rejected: min {lo[1]:+.3g} at {iso(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}"
|
||||||
if good or rej:
|
if good or rej:
|
||||||
print(line)
|
print(line)
|
||||||
|
|
||||||
@@ -224,26 +221,14 @@ def main():
|
|||||||
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
|
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
|
||||||
o = a.parse_args()
|
o = a.parse_args()
|
||||||
|
|
||||||
by_sensor = load(o.folder)
|
by_sensor, names = sensors(o.folder, o.sensors)
|
||||||
if not by_sensor:
|
|
||||||
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
|
|
||||||
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
|
|
||||||
columns = o.columns.split(",")
|
columns = o.columns.split(",")
|
||||||
residuals = [c for c in o.residuals.split(",") if c and c != "no"] # '' or "no": none, as for --table
|
residuals = [c for c in o.residuals.split(",") if c and c != "no"] # '' or "no": none, as for --table
|
||||||
outside = tuple(parse_time(x) for x in o.outside.split("/")) if o.outside else None
|
kept = outside(o.outside)
|
||||||
if outside and len(outside) != 2:
|
o.in_table = window_filter(o.table)
|
||||||
sys.exit("--outside expects T1/T2.")
|
check("sensors", names, by_sensor)
|
||||||
if o.table in ("", "no"):
|
check("sensors of --follow (taken from --sensors)", [s for s in o.follow.split(",") if s], names)
|
||||||
o.in_table = lambda t: o.table == ""
|
check("columns", columns + residuals + [o.column], COLUMNS)
|
||||||
else:
|
|
||||||
limits = [parse_time(x) for x in o.table.split("/")]
|
|
||||||
if len(limits) != 2:
|
|
||||||
sys.exit("--table expects T1/T2 or no.")
|
|
||||||
o.in_table = lambda t: limits[0] <= t <= limits[1]
|
|
||||||
for requested, known, what in ((names, by_sensor, "sensors"), ([s for s in o.follow.split(",") if s], names, "sensors of --follow (taken from --sensors)"), (columns + residuals + [o.column], COLUMNS, "columns")):
|
|
||||||
missing = [x for x in requested if x not in known]
|
|
||||||
if missing:
|
|
||||||
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
|
|
||||||
series = {s: filter_window(by_sensor[s], o.start, o.end) for s in names}
|
series = {s: filter_window(by_sensor[s], o.start, o.end) for s in names}
|
||||||
if not any(series.values()):
|
if not any(series.values()):
|
||||||
sys.exit("No measurement in the requested window.")
|
sys.exit("No measurement in the requested window.")
|
||||||
@@ -274,7 +259,7 @@ def main():
|
|||||||
ax.plot(cam_t, [pose_t.get(t, math.nan) for t in cam_t], ".-", ms=3, lw=0.5, color="C2", label="camera pose (one point per registration)")
|
ax.plot(cam_t, [pose_t.get(t, math.nan) for t in cam_t], ".-", ms=3, lw=0.5, color="C2", label="camera pose (one point per registration)")
|
||||||
ax.set_ylabel(f"camera: {col}" + (" [deg]" if col in ("pan", "tilt", "rotation") else ""))
|
ax.set_ylabel(f"camera: {col}" + (" [deg]" if col in ("pan", "tilt", "rotation") else ""))
|
||||||
ax.legend(loc="upper left", fontsize=8)
|
ax.legend(loc="upper left", fontsize=8)
|
||||||
commons, refs_per_col, ref_devs = {}, {}, {}
|
commons, refs_per_col, ref_devs = {}, {}, {} # median per measurement of each column; references per column, their deviations
|
||||||
for ax, col in zip(axes[len(pose_cols):, 0], columns):
|
for ax, col in zip(axes[len(pose_cols):, 0], columns):
|
||||||
e, _ = deviations(col)
|
e, _ = deviations(col)
|
||||||
med, low, high, tt = [], [], [], []
|
med, low, high, tt = [], [], [], []
|
||||||
@@ -329,19 +314,13 @@ def main():
|
|||||||
print(common)
|
print(common)
|
||||||
|
|
||||||
# table of the common median, per step
|
# table of the common median, per step
|
||||||
midnight = times[0].replace(hour=0, minute=0, second=0, microsecond=0)
|
key = per_step(times[0], o.step) # steps aligned on the clock (hh:00, hh:15…), not on the 1st measurement
|
||||||
t0 = midnight + ((times[0] - midnight) // o.step) * o.step # steps aligned on the clock (hh:00, hh:15…), not on the 1st measurement
|
|
||||||
rows = {}
|
|
||||||
for col in columns:
|
|
||||||
for t, v in commons[col].items():
|
|
||||||
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col, []).append(v)
|
|
||||||
for col in pose_cols:
|
for col in pose_cols:
|
||||||
commons["cam_" + col] = {p["t"]: p[col] for p in poses if p[col] is not None}
|
commons["cam_" + col] = {p["t"]: p[col] for p in poses if p[col] is not None}
|
||||||
for t, v in commons["cam_" + col].items():
|
rows = {}
|
||||||
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault("cam_" + col, []).append(v)
|
for col in columns + ["cam_" + c for c in pose_cols] + [c + "_targets" for c in refs_per_col]:
|
||||||
for col in refs_per_col:
|
for t, v in commons[col].items():
|
||||||
for t, v in commons[col + "_targets"].items():
|
rows.setdefault(key(t), {}).setdefault(col, []).append(v)
|
||||||
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col + "_targets", []).append(v)
|
|
||||||
table_cols = columns + [c + "_targets" for c in refs_per_col] + ["cam_" + c for c in pose_cols]
|
table_cols = columns + [c + "_targets" for c in refs_per_col] + ["cam_" + c for c in pose_cols]
|
||||||
print(f"\nCommon median (deviation from the median of the window){' and raw camera pose (cam_, empty without registration)' if pose_cols else ''}, "
|
print(f"\nCommon median (deviation from the median of the window){' and raw camera pose (cam_, empty without registration)' if pose_cols else ''}, "
|
||||||
f"median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
|
f"median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
|
||||||
@@ -352,15 +331,14 @@ def main():
|
|||||||
print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):13.{6 if c.startswith('cam_') else 4}g}" if c in rows[t] else " " * 13 for c in table_cols))
|
print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):13.{6 if c.startswith('cam_') else 4}g}" if c in rows[t] else " " * 13 for c in table_cols))
|
||||||
for col in table_cols:
|
for col in table_cols:
|
||||||
if commons[col]:
|
if commons[col]:
|
||||||
lo, hi = min(commons[col].items(), key=lambda x: x[1]), max(commons[col].items(), key=lambda x: x[1])
|
lo, hi = extremes(commons[col])
|
||||||
g = 6 if col.startswith("cam_") else 4
|
g = 6 if col.startswith("cam_") else 4
|
||||||
print(f"{col} (measurements): min {lo[1]:.{g}g} at {iso(lo[0])}, max {hi[1]:.{g}g} at {iso(hi[0])}")
|
print(f"{col} (measurements): min {lo[1]:.{g}g} at {iso(lo[0])}, max {hi[1]:.{g}g} at {iso(hi[0])}")
|
||||||
|
|
||||||
for col, refs in refs_per_col.items():
|
for col, refs in refs_per_col.items():
|
||||||
print(f"\n{col} raw of the global references, deviation from their median of the window (measurements); {col}_targets: median of the targets alone:")
|
print(f"\n{col} raw of the global references, deviation from their median of the window (measurements); {col}_targets: median of the targets alone:")
|
||||||
for s in refs:
|
for s in refs:
|
||||||
d = ref_devs[col][s]
|
lo, hi = extremes(ref_devs[col][s])
|
||||||
lo, hi = min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
|
|
||||||
rej = {m["t"] for m in series[s] if rejected(m)}
|
rej = {m["t"] for m in series[s] if rejected(m)}
|
||||||
r = lambda t: " (rejected)" if t in rej else ""
|
r = lambda t: " (rejected)" if t in rej else ""
|
||||||
print(f"{s:9} min {lo[1]:+.3g} at {iso(lo[0])}{r(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}{r(hi[0])}"
|
print(f"{s:9} min {lo[1]:+.3g} at {iso(lo[0])}{r(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}{r(hi[0])}"
|
||||||
@@ -368,66 +346,71 @@ def main():
|
|||||||
if not pose_cols:
|
if not pose_cols:
|
||||||
print("Camera pose: none in the window (no camera/ in the folder, or no registration).")
|
print("Camera pose: none in the window (no camera/ in the folder, or no registration).")
|
||||||
else:
|
else:
|
||||||
posed = {p["t"].replace(second=0, microsecond=0) for p in poses}
|
runs = without_pose(times, poses)
|
||||||
runs = [] # runs of consecutive measurement times without a pose: [start, end, number]
|
|
||||||
for i, t in enumerate(times):
|
|
||||||
if t.replace(second=0, microsecond=0) in posed:
|
|
||||||
continue
|
|
||||||
if runs and runs[-1][3] == i - 1:
|
|
||||||
runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i
|
|
||||||
else:
|
|
||||||
runs.append([t, t, 1, i])
|
|
||||||
print(f"Measurement times without a camera pose (no registration): {sum(x[2] for x in runs)} of {len(times)}"
|
print(f"Measurement times without a camera pose (no registration): {sum(x[2] for x in runs)} of {len(times)}"
|
||||||
+ (": " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n, _ in runs) if runs else ""))
|
+ (": " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n in runs) if runs else ""))
|
||||||
|
|
||||||
# 2. small multiples
|
# 2. small multiples and 3. residuals: one small plot per sensor, one scale each
|
||||||
col = o.column
|
mins = int(o.step.total_seconds() // 60)
|
||||||
n = len(names)
|
nrows = math.ceil(len(names) / 5)
|
||||||
nrows = math.ceil(n / 5)
|
|
||||||
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
|
def sheet():
|
||||||
for ax, s in zip(axes.flat, names):
|
return plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
|
||||||
ms = [m for m in series[s] if m[col] is not None]
|
|
||||||
ax.plot([m["t"] for m in ms], [m[col] for m in ms], "-", lw=0.4, color="0.7")
|
def small(ax, pts, line, title):
|
||||||
|
"""Measurements {t: v} in grey, their median per step in blue (returned), a dotted line at `line`."""
|
||||||
steps = {}
|
steps = {}
|
||||||
for m in ms:
|
for t, v in pts.items():
|
||||||
steps.setdefault(t0 + ((m["t"] - t0) // o.step) * o.step, []).append(m[col])
|
steps.setdefault(key(t), []).append(v)
|
||||||
ax.plot(list(steps), [statistics.median(v) for v in steps.values()], "-", lw=1.1, color="C0")
|
pm = {t: statistics.median(v) for t, v in steps.items()}
|
||||||
lv = level(series[s], col)
|
ax.plot(list(pts), list(pts.values()), "-", lw=0.4, color="0.7")
|
||||||
if lv:
|
ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0")
|
||||||
ax.axhline(lv[0], color="C3", ls=":", lw=0.8)
|
if line is not None:
|
||||||
ax.set_title(s + (" (ref.)" if series[s] and is_reference(series[s]) else ""), fontsize=9)
|
ax.axhline(line, color="C3", ls=":", lw=0.8)
|
||||||
|
ax.set_title(title, fontsize=9)
|
||||||
ax.grid(alpha=0.3)
|
ax.grid(alpha=0.3)
|
||||||
ax.tick_params(labelsize=7)
|
ax.tick_params(labelsize=7)
|
||||||
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
|
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
|
||||||
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
|
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
|
||||||
for ax in list(axes.flat)[n:]:
|
return pm
|
||||||
ax.axis("off")
|
|
||||||
fig.suptitle(f"{station}: {col}, median per step of {int(o.step.total_seconds() // 60)} min (blue), measurements (grey), "
|
def save(fig, axes, title, name):
|
||||||
f"median of the window (dotted); one scale per sensor (UTC times)")
|
for ax in list(axes.flat)[len(names):]:
|
||||||
fig.tight_layout()
|
ax.axis("off")
|
||||||
multiples = base.format(f"multiples-{col}")
|
fig.suptitle(title)
|
||||||
fig.savefig(multiples, dpi=100, bbox_inches="tight")
|
fig.tight_layout()
|
||||||
print(multiples)
|
path = base.format(name)
|
||||||
|
fig.savefig(path, dpi=100, bbox_inches="tight")
|
||||||
|
plt.close(fig)
|
||||||
|
print(path)
|
||||||
|
|
||||||
|
col = o.column
|
||||||
|
fig, axes = sheet()
|
||||||
|
for ax, s in zip(axes.flat, names):
|
||||||
|
lv = level(series[s], col)
|
||||||
|
small(ax, {m["t"]: m[col] for m in series[s] if m[col] is not None}, lv and lv[0],
|
||||||
|
s + (" (ref.)" if series[s] and is_reference(series[s]) else ""))
|
||||||
|
save(fig, axes, f"{station}: {col}, median per step of {mins} min (blue), measurements (grey), "
|
||||||
|
f"median of the window (dotted); one scale per sensor (UTC times)", f"multiples-{col}")
|
||||||
|
|
||||||
|
def own_level(ms, col): # (median, MAD) outside --outside, or over the window when nothing is left
|
||||||
|
return level([m for m in ms if kept(m["t"])] or ms, col)
|
||||||
|
|
||||||
# 3. residuals: each sensor minus its own level
|
|
||||||
for col, others in [(c, False) for c in residuals] + [(c, True) for c in residuals if o.others]:
|
for col, others in [(c, False) for c in residuals] + [(c, True) for c in residuals if o.others]:
|
||||||
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
|
fig, axes = sheet()
|
||||||
dev = {}
|
dev = {}
|
||||||
if others: # {sensor: {t: value - own level}} for the median of the others
|
if others: # {sensor: {t: value - own level}} for the median of the others
|
||||||
for q in names:
|
for q in names:
|
||||||
mq = [m for m in series[q] if m[col] is not None]
|
mq = [m for m in series[q] if m[col] is not None]
|
||||||
rq = [m for m in mq if not outside or not outside[0] <= m["t"] <= outside[1]] or mq
|
if nq := own_level(mq, col):
|
||||||
nq = level(rq, col)
|
|
||||||
if nq:
|
|
||||||
dev[q] = {m["t"]: m[col] - nq[0] for m in mq}
|
dev[q] = {m["t"]: m[col] - nq[0] for m in mq}
|
||||||
print(f"\n{col} minus the own level{' and minus the median of the other sensors at the same time' if others else ''} ({'outside ' + o.outside if outside else 'median of the window'}), median per step of "
|
print(f"\n{col} minus the own level{' and minus the median of the other sensors at the same time' if others else ''} ({'outside ' + o.outside if o.outside else 'median of the window'}), median per step of "
|
||||||
f"{int(o.step.total_seconds() // 60)} min:")
|
f"{mins} min:")
|
||||||
slopes = []
|
slopes = []
|
||||||
print(f"{'sensor':9}{'level':>10}{'noise':>10}{'min dev.':>11} {'at':<17}{'(noises)':>9}{'max dev.':>11} {'at':<17}{'(noises)':>9}")
|
print(f"{'sensor':9}{'level':>10}{'noise':>10}{'min dev.':>11} {'at':<17}{'(noises)':>9}{'max dev.':>11} {'at':<17}{'(noises)':>9}")
|
||||||
for ax, s in zip(axes.flat, names):
|
for ax, s in zip(axes.flat, names):
|
||||||
ms = [m for m in series[s] if m[col] is not None]
|
ms = [m for m in series[s] if m[col] is not None]
|
||||||
ref = [m for m in ms if not outside or not outside[0] <= m["t"] <= outside[1]] or ms
|
lv = own_level(ms, col)
|
||||||
lv = level(ref, col)
|
|
||||||
if not lv:
|
if not lv:
|
||||||
ax.axis("off")
|
ax.axis("off")
|
||||||
continue
|
continue
|
||||||
@@ -443,20 +426,8 @@ def main():
|
|||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
res = {m["t"]: m[col] - lv[0] for m in ms}
|
res = {m["t"]: m[col] - lv[0] for m in ms}
|
||||||
steps = {}
|
lo, hi = extremes(small(ax, res, 0, s + (" (ref.)" if is_reference(ms) else "") + f" noise {noise:.2g}"))
|
||||||
for t, v in res.items():
|
f = lambda v: f"{v / noise:9.1f}" if noise else f"{'-':>9}" # noqa: E731
|
||||||
steps.setdefault(t0 + ((t - t0) // o.step) * o.step, []).append(v)
|
|
||||||
pm = {t: statistics.median(v) for t, v in steps.items()}
|
|
||||||
ax.plot(list(res), list(res.values()), "-", lw=0.4, color="0.7")
|
|
||||||
ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0")
|
|
||||||
ax.axhline(0, color="C3", ls=":", lw=0.8)
|
|
||||||
ax.set_title(s + (" (ref.)" if is_reference(ms) else "") + f" noise {noise:.2g}", fontsize=9)
|
|
||||||
ax.grid(alpha=0.3)
|
|
||||||
ax.tick_params(labelsize=7)
|
|
||||||
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
|
|
||||||
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
|
|
||||||
lo, hi = min(pm.items(), key=lambda x: x[1]), max(pm.items(), key=lambda x: x[1])
|
|
||||||
f = lambda v: f"{v / noise:9.1f}" if noise else f"{'-':>9}"
|
|
||||||
print(f"{s:9}{lv[0]:10.4g}{noise:10.3g}{lo[1]:11.3g} {iso(lo[0]):<17}{f(lo[1])}{hi[1]:11.3g} {iso(hi[0]):<17}{f(hi[1])}")
|
print(f"{s:9}{lv[0]:10.4g}{noise:10.3g}{lo[1]:11.3g} {iso(lo[0]):<17}{f(lo[1])}{hi[1]:11.3g} {iso(hi[0]):<17}{f(hi[1])}")
|
||||||
if others:
|
if others:
|
||||||
for c in columns:
|
for c in columns:
|
||||||
@@ -465,17 +436,10 @@ def main():
|
|||||||
xs, ys = [commons[c][t] for t in tt], [res[t] for t in tt]
|
xs, ys = [commons[c][t] for t in tt], [res[t] for t in tt]
|
||||||
slope = statistics.linear_regression(xs, ys).slope
|
slope = statistics.linear_regression(xs, ys).slope
|
||||||
slopes.append(f"{s:9}slope on common {c} {slope:+.4g} per unit, r = {statistics.correlation(xs, ys):+.2f}")
|
slopes.append(f"{s:9}slope on common {c} {slope:+.4g} per unit, r = {statistics.correlation(xs, ys):+.2f}")
|
||||||
for ax in list(axes.flat)[n:]:
|
|
||||||
ax.axis("off")
|
|
||||||
if slopes:
|
if slopes:
|
||||||
print("\n".join(slopes))
|
print("\n".join(slopes))
|
||||||
fig.suptitle(f"{station}: {col} minus the own level{' and the median of the others' if others else ''} of each sensor, median per step of "
|
save(fig, axes, f"{station}: {col} minus the own level{' and the median of the others' if others else ''} of each sensor, median per step of "
|
||||||
f"{int(o.step.total_seconds() // 60)} min (blue), measurements (grey); one scale per sensor (UTC times)")
|
f"{mins} min (blue), measurements (grey); one scale per sensor (UTC times)", f"residuals-{col}" + ("-others" if others else ""))
|
||||||
fig.tight_layout()
|
|
||||||
path = base.format(f"residuals-{col}" + ("-others" if others else ""))
|
|
||||||
fig.savefig(path, dpi=100, bbox_inches="tight")
|
|
||||||
plt.close(fig)
|
|
||||||
print(path)
|
|
||||||
|
|
||||||
if o.rigid:
|
if o.rigid:
|
||||||
rigid(o, names, series, times, base, station)
|
rigid(o, names, series, times, base, station)
|
||||||
|
|||||||
+5
-16
@@ -17,10 +17,8 @@ matplotlib.use("Agg")
|
|||||||
import matplotlib.dates as mdates # noqa: E402
|
import matplotlib.dates as mdates # noqa: E402
|
||||||
import matplotlib.pyplot as plt # noqa: E402
|
import matplotlib.pyplot as plt # noqa: E402
|
||||||
|
|
||||||
from data import filter_window, is_reference, load, parse_time # noqa: E402
|
from common import COLUMNS, QUALITY, bound, check, sensors # noqa: E402
|
||||||
from summary import QUALITY # noqa: E402
|
from data import filter_window, parse_time # noqa: E402
|
||||||
|
|
||||||
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
|
|
||||||
|
|
||||||
|
|
||||||
def zone(text):
|
def zone(text):
|
||||||
@@ -31,10 +29,6 @@ def zone(text):
|
|||||||
return parse_time(start), parse_time(end)
|
return parse_time(start), parse_time(end)
|
||||||
|
|
||||||
|
|
||||||
def bound(t, default):
|
|
||||||
return t.strftime("%Y%m%dT%H%MZ") if t else default
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
a.add_argument("folder")
|
a.add_argument("folder")
|
||||||
@@ -46,15 +40,10 @@ def main():
|
|||||||
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG")
|
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG")
|
||||||
o = a.parse_args()
|
o = a.parse_args()
|
||||||
|
|
||||||
by = load(o.folder)
|
by, names = sensors(o.folder, o.sensors)
|
||||||
if not by:
|
|
||||||
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
|
|
||||||
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
|
||||||
columns = o.columns.split(",")
|
columns = o.columns.split(",")
|
||||||
for requested, known, what in ((names, by, "sensors"), (columns, COLUMNS + QUALITY, "columns")):
|
check("sensors", names, by)
|
||||||
missing = [x for x in requested if x not in known]
|
check("columns", columns, COLUMNS + QUALITY)
|
||||||
if missing:
|
|
||||||
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
|
|
||||||
series = {s: filter_window(by[s], o.start, o.end) for s in names}
|
series = {s: filter_window(by[s], o.start, o.end) for s in names}
|
||||||
if not any(series.values()):
|
if not any(series.values()):
|
||||||
sys.exit("No measurement in the requested window.")
|
sys.exit("No measurement in the requested window.")
|
||||||
|
|||||||
+12
-21
@@ -46,7 +46,7 @@ import sys
|
|||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
import data
|
import data
|
||||||
from summary import QUALITY as METRICS, duration
|
from common import QUALITY as METRICS, duration, group, per_step, sensors
|
||||||
|
|
||||||
HIGH_BAD = ("fwhm", "dominance") # for these two, it is the rise that says a degraded image
|
HIGH_BAD = ("fwhm", "dominance") # for these two, it is the rise that says a degraded image
|
||||||
COLOURS = {"Visibility": "tab:blue", "Obstruction": "tab:orange", "Thermal displacement": "tab:red",
|
COLOURS = {"Visibility": "tab:blue", "Obstruction": "tab:orange", "Thermal displacement": "tab:red",
|
||||||
@@ -89,6 +89,11 @@ def affected(r, c, threshold):
|
|||||||
return r is not None and ((c in ("flux", "snr") and r < threshold) or (c == "dominance" and r > 1 / threshold))
|
return r is not None and ((c in ("flux", "snr") and r < threshold) or (c == "dominance" and r > 1 / threshold))
|
||||||
|
|
||||||
|
|
||||||
|
def hit(ref, s, m, threshold):
|
||||||
|
"""The measurement says a degraded image: flux or snr below threshold, or dominance above 1/threshold."""
|
||||||
|
return any(affected(ratio(ref, s, c, m), c, threshold) for c in ("flux", "snr", "dominance"))
|
||||||
|
|
||||||
|
|
||||||
def med(v):
|
def med(v):
|
||||||
return f"{statistics.median(v):.2f}" if v else "-"
|
return f"{statistics.median(v):.2f}" if v else "-"
|
||||||
|
|
||||||
@@ -123,14 +128,8 @@ def summary(by_sensor, ref, names, refs, pmax):
|
|||||||
for c in METRICS))
|
for c in METRICS))
|
||||||
|
|
||||||
|
|
||||||
def per_step(by_sensor, names, step):
|
|
||||||
step = duration(step)
|
|
||||||
start = min(m["t"] for s in names for m in by_sensor[s]).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
||||||
return lambda t: start + (t - start) // step * step
|
|
||||||
|
|
||||||
|
|
||||||
def table(by_sensor, ref, names, c, step, threshold):
|
def table(by_sensor, ref, names, c, step, threshold):
|
||||||
key, cells = per_step(by_sensor, names, step), {}
|
key, cells = per_step(min(m["t"] for s in names for m in by_sensor[s]), duration(step)), {}
|
||||||
for s in names:
|
for s in names:
|
||||||
for m in by_sensor[s]:
|
for m in by_sensor[s]:
|
||||||
r = ratio(ref, s, c, m)
|
r = ratio(ref, s, c, m)
|
||||||
@@ -158,7 +157,7 @@ def table(by_sensor, ref, names, c, step, threshold):
|
|||||||
|
|
||||||
def common(by_sensor, ref, names, refs, step):
|
def common(by_sensor, ref, names, refs, step):
|
||||||
"""Per step: median of the ratios of the targets, then of the references, for each metric."""
|
"""Per step: median of the ratios of the targets, then of the references, for each metric."""
|
||||||
key, cells = per_step(by_sensor, names, step), {}
|
key, cells = per_step(min(m["t"] for s in names for m in by_sensor[s]), duration(step)), {}
|
||||||
for s in names:
|
for s in names:
|
||||||
for m in by_sensor[s]:
|
for m in by_sensor[s]:
|
||||||
d = cells.setdefault(key(m["t"]), {c: ([], []) for c in METRICS})
|
d = cells.setdefault(key(m["t"]), {c: ([], []) for c in METRICS})
|
||||||
@@ -204,7 +203,7 @@ def labels(a, every, ref, refs, names, start, end):
|
|||||||
worst = []
|
worst = []
|
||||||
for s in names:
|
for s in names:
|
||||||
ms = [m for m in win[s] if t0 <= m["t"] <= t1]
|
ms = [m for m in win[s] if t0 <= m["t"] <= t1]
|
||||||
if any(affected(ratio(ref, s, c, m), c, a.threshold) for m in ms for c in ("flux", "snr", "dominance")):
|
if any(hit(ref, s, m, a.threshold) for m in ms):
|
||||||
fl = [x for x in (ratio(ref, s, "flux", m) for m in ms) if x is not None]
|
fl = [x for x in (ratio(ref, s, "flux", m) for m in ms) if x is not None]
|
||||||
worst.append((min(fl) if fl else 1, s + ("r" if s in refs else "")))
|
worst.append((min(fl) if fl else 1, s + ("r" if s in refs else "")))
|
||||||
worst.sort()
|
worst.sort()
|
||||||
@@ -236,14 +235,9 @@ def outside(a, every, ref, refs, names, start, end):
|
|||||||
for m in data.filter_window(every[s], start, end):
|
for m in data.filter_window(every[s], start, end):
|
||||||
if within(m["t"], covered):
|
if within(m["t"], covered):
|
||||||
continue
|
continue
|
||||||
if any(affected(ratio(ref, s, c, m), c, a.threshold) for c in ("flux", "snr", "dominance")):
|
if hit(ref, s, m, a.threshold):
|
||||||
inst.setdefault(m["t"], {})[s + ("r" if s in refs else "")] = ratio(ref, s, "flux", m) or 0
|
inst.setdefault(m["t"], {})[s + ("r" if s in refs else "")] = ratio(ref, s, "flux", m) or 0
|
||||||
runs = []
|
runs = group(sorted(t for t in inst if len(inst[t]) >= a.min), lambda x, y: y - x <= timedelta(minutes=15))
|
||||||
for t in sorted(t for t in inst if len(inst[t]) >= a.min):
|
|
||||||
if runs and t - runs[-1][-1] <= timedelta(minutes=15):
|
|
||||||
runs[-1].append(t)
|
|
||||||
else:
|
|
||||||
runs.append([t])
|
|
||||||
print(f"Outside the labels (except {a.except_}): times when at least {a.min} sensors are affected; {len(runs)} runs")
|
print(f"Outside the labels (except {a.except_}): times when at least {a.min} sensors are affected; {len(runs)} runs")
|
||||||
for ru in runs:
|
for ru in runs:
|
||||||
worst = {}
|
worst = {}
|
||||||
@@ -369,13 +363,10 @@ def main():
|
|||||||
if unknown:
|
if unknown:
|
||||||
p.error(f"unknown metrics: {', '.join(unknown)}; known: {', '.join(METRICS)}")
|
p.error(f"unknown metrics: {', '.join(unknown)}; known: {', '.join(METRICS)}")
|
||||||
|
|
||||||
every = data.load(a.folder)
|
every, names = sensors(a.folder, a.sensors)
|
||||||
if not every:
|
|
||||||
sys.exit(f"No \"advanced\" export in {a.folder} (see inventory.py).")
|
|
||||||
if not any(m[c] is not None for ms in every.values() for m in ms for c in METRICS):
|
if not any(m[c] is not None for ms in every.values() for m in ms for c in METRICS):
|
||||||
sys.exit(f"No quality metrics in {a.folder}: peak to snr empty (old camera; "
|
sys.exit(f"No quality metrics in {a.folder}: peak to snr empty (old camera; "
|
||||||
"tools/DATA.md).")
|
"tools/DATA.md).")
|
||||||
names = a.sensors.split(",") if a.sensors else sorted(every, key=lambda s: (not data.is_reference(every[s]), s))
|
|
||||||
missing = [s for s in names if s not in every]
|
missing = [s for s in names if s not in every]
|
||||||
if missing:
|
if missing:
|
||||||
sys.exit(f"Missing sensors: {', '.join(missing)}. Present: {', '.join(sorted(every))}.")
|
sys.exit(f"Missing sensors: {', '.join(missing)}. Present: {', '.join(sorted(every))}.")
|
||||||
|
|||||||
+8
-15
@@ -34,8 +34,8 @@ import bisect
|
|||||||
import statistics
|
import statistics
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from data import filter_window, is_reference, iso, load, parse_time, rejected
|
from common import COLUMNS, check, duration, outside, own_noise, sensors
|
||||||
from summary import COLUMNS, duration, own_noise
|
from data import filter_window, is_reference, iso, parse_time, rejected
|
||||||
|
|
||||||
ABBREV = {"corr_x": "cx", "corr_y": "cy", "error_value": "ev"}
|
ABBREV = {"corr_x": "cx", "corr_y": "cy", "error_value": "ev"}
|
||||||
|
|
||||||
@@ -68,18 +68,11 @@ def main():
|
|||||||
a.add_argument("--end", type=parse_time)
|
a.add_argument("--end", type=parse_time)
|
||||||
o = a.parse_args()
|
o = a.parse_args()
|
||||||
|
|
||||||
by = load(o.folder)
|
by, names = sensors(o.folder, o.sensors)
|
||||||
if not by:
|
|
||||||
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
|
|
||||||
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
|
||||||
columns = o.columns.split(",")
|
columns = o.columns.split(",")
|
||||||
outside = tuple(parse_time(x) for x in o.outside.split("/")) if o.outside else None
|
kept = outside(o.outside)
|
||||||
if outside and len(outside) != 2:
|
check("sensors", names, by)
|
||||||
sys.exit("--outside expects T1/T2.")
|
check("columns", columns, COLUMNS)
|
||||||
for requested, known, what in ((names, by, "sensors"), (columns, COLUMNS, "columns")):
|
|
||||||
missing = [x for x in requested if x not in known]
|
|
||||||
if missing:
|
|
||||||
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
|
|
||||||
series = {s: filter_window(by[s], o.start, o.end) for s in names}
|
series = {s: filter_window(by[s], o.start, o.end) for s in names}
|
||||||
if not any(series.values()):
|
if not any(series.values()):
|
||||||
sys.exit("No measurement in the requested window.")
|
sys.exit("No measurement in the requested window.")
|
||||||
@@ -96,7 +89,7 @@ def main():
|
|||||||
if len(ms) < 2:
|
if len(ms) < 2:
|
||||||
print(f"{s:9}{len(ms):5} (fewer than 2 values)")
|
print(f"{s:9}{len(ms):5} (fewer than 2 values)")
|
||||||
continue
|
continue
|
||||||
ref = [x for x in ms if not outside or not outside[0] <= x[0] <= outside[1]]
|
ref = [x for x in ms if kept(x[0])]
|
||||||
if len(ref) < 2:
|
if len(ref) < 2:
|
||||||
print(f"{s:9}{len(ms):5} (fewer than 2 values outside --outside)")
|
print(f"{s:9}{len(ms):5} (fewer than 2 values outside --outside)")
|
||||||
continue
|
continue
|
||||||
@@ -123,7 +116,7 @@ def main():
|
|||||||
ranges.append((s, group, noise, rej))
|
ranges.append((s, group, noise, rej))
|
||||||
total = len(ranges)
|
total = len(ranges)
|
||||||
ranges = [p for p in ranges if len(p[1]) >= o.min_measurements]
|
ranges = [p for p in ranges if len(p[1]) >= o.min_measurements]
|
||||||
print(f"\n{col}: ranges more than {o.k:g} noises from the level{' rolling ±' + roll + ' (median of the neighbours)' if roll else ''}{' (level outside ' + o.outside + ')' if outside and not roll else ''}{' (noise outside ' + o.outside + ')' if outside and roll else ''} (merge {int(o.merge.total_seconds() // 60)} min)"
|
print(f"\n{col}: ranges more than {o.k:g} noises from the level{' rolling ±' + roll + ' (median of the neighbours)' if roll else ''}{' (level outside ' + o.outside + ')' if o.outside and not roll else ''}{' (noise outside ' + o.outside + ')' if o.outside and roll else ''} (merge {int(o.merge.total_seconds() // 60)} min)"
|
||||||
+ (f"; {len(ranges)} of at least {o.min_measurements} measurements out of {total}" if o.min_measurements > 1 else ""))
|
+ (f"; {len(ranges)} of at least {o.min_measurements} measurements out of {total}" if o.min_measurements > 1 else ""))
|
||||||
print(f"{'sensor':9}{'start':<18}{'end':<18}{'measurements':>13}{'extreme':>10}{'(noises)':>9} {'at':<17}{'rejected':>9}")
|
print(f"{'sensor':9}{'start':<18}{'end':<18}{'measurements':>13}{'extreme':>10}{'(noises)':>9} {'at':<17}{'rejected':>9}")
|
||||||
for s, g, noise, rej in ranges:
|
for s, g, noise, rej in ranges:
|
||||||
|
|||||||
+3
-14
@@ -33,10 +33,8 @@ matplotlib.use("Agg")
|
|||||||
import matplotlib.dates as mdates # noqa: E402
|
import matplotlib.dates as mdates # noqa: E402
|
||||||
import matplotlib.pyplot as plt # noqa: E402
|
import matplotlib.pyplot as plt # noqa: E402
|
||||||
|
|
||||||
|
from common import STATS, bound, cut, duration, group, targets, window_filter # noqa: E402
|
||||||
from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time, rejected # noqa: E402
|
from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time, rejected # noqa: E402
|
||||||
from metrics import cut, runs, targets # noqa: E402
|
|
||||||
from plot import bound # noqa: E402
|
|
||||||
from summary import STATS, duration # noqa: E402
|
|
||||||
|
|
||||||
COLUMNS = ("temperature",) + tuple("cam_" + c for c in POSE) + ("tgt_corr_x", "tgt_corr_y", "rej_targets", "rej_refs")
|
COLUMNS = ("temperature",) + tuple("cam_" + c for c in POSE) + ("tgt_corr_x", "tgt_corr_y", "rej_targets", "rej_refs")
|
||||||
|
|
||||||
@@ -83,13 +81,7 @@ def main():
|
|||||||
columns = o.columns.split(",")
|
columns = o.columns.split(",")
|
||||||
if missing := [c for c in columns if c not in COLUMNS]:
|
if missing := [c for c in columns if c not in COLUMNS]:
|
||||||
sys.exit(f"Unknown columns: {', '.join(missing)}. Known: {', '.join(COLUMNS)}.")
|
sys.exit(f"Unknown columns: {', '.join(missing)}. Known: {', '.join(COLUMNS)}.")
|
||||||
if o.table in ("", "no"):
|
in_table = window_filter(o.table)
|
||||||
in_table = lambda t: o.table == "" # noqa: E731
|
|
||||||
else:
|
|
||||||
limits = [parse_time(x) for x in o.table.split("/")]
|
|
||||||
if len(limits) != 2:
|
|
||||||
sys.exit("--table expects T1/T2 or no.")
|
|
||||||
in_table = lambda t: limits[0] <= t <= limits[1] # noqa: E731
|
|
||||||
|
|
||||||
names = [os.path.basename(os.path.normpath(d)) for d in o.folders]
|
names = [os.path.basename(os.path.normpath(d)) for d in o.folders]
|
||||||
by_station, measured = {}, {}
|
by_station, measured = {}, {}
|
||||||
@@ -123,10 +115,7 @@ def main():
|
|||||||
t = tuple(n for n in names if k not in with_temp[n])
|
t = tuple(n for n in names if k not in with_temp[n])
|
||||||
if m or t:
|
if m or t:
|
||||||
keys.setdefault((m, t), []).append(k)
|
keys.setdefault((m, t), []).append(k)
|
||||||
lines = []
|
lines = [(g[0], g[-1], len(g), m, t) for (m, t), ks in keys.items() for g in group(ks, lambda a, b: b - a == o.step)]
|
||||||
for (m, t), ks in keys.items():
|
|
||||||
for d, f, nb in runs(ks, o.step):
|
|
||||||
lines.append((d, f, nb, m, t))
|
|
||||||
for d, f, nb, m, t in sorted(lines):
|
for d, f, nb, m, t in sorted(lines):
|
||||||
print(f" {iso(d)} → {iso(f)} ({nb} steps): no measurement {','.join(m) or '-'}; no temperature {','.join(t) or '-'}")
|
print(f" {iso(d)} → {iso(f)} ({nb} steps): no measurement {','.join(m) or '-'}; no temperature {','.join(t) or '-'}")
|
||||||
if not lines:
|
if not lines:
|
||||||
|
|||||||
+6
-36
@@ -18,37 +18,11 @@ Usage: python3 tools/scripts/summary.py data/PS_DES_COUDRES/ [--column error_val
|
|||||||
Gives values, not a verdict.
|
Gives values, not a verdict.
|
||||||
"""
|
"""
|
||||||
import argparse
|
import argparse
|
||||||
import re
|
|
||||||
import statistics
|
import statistics
|
||||||
import sys
|
import sys
|
||||||
from datetime import timedelta
|
|
||||||
|
|
||||||
from data import filter_window, is_reference, iso, load, parse_time, rejected
|
from common import COLUMNS, QUALITY, STATS, duration, per_step, sensors
|
||||||
|
from data import filter_window, iso, parse_time, rejected
|
||||||
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
|
|
||||||
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
|
|
||||||
STATS = {"median": statistics.median, "min": min, "max": max}
|
|
||||||
|
|
||||||
|
|
||||||
def duration(text):
|
|
||||||
"""'15min', '1h', '6h', '1d' -> timedelta."""
|
|
||||||
m = re.fullmatch(r"(\d+)(min|h|d)", text)
|
|
||||||
if not m:
|
|
||||||
raise argparse.ArgumentTypeError(f"invalid step: {text!r} (e.g. 15min, 1h, 1d)")
|
|
||||||
n, unit = int(m.group(1)), m.group(2)
|
|
||||||
return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unit]
|
|
||||||
|
|
||||||
|
|
||||||
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
|
|
||||||
|
|
||||||
|
|
||||||
def own_noise(vs):
|
|
||||||
"""Median of the absolute differences between two consecutive values; 0 when it is below ROUNDING (values constant
|
|
||||||
up to rounding: the corr of the references when the compensation goes exactly through them, two references
|
|
||||||
retained), so that a deviation is not written as billions of noises; 0 also with fewer than two values."""
|
|
||||||
d = [abs(b - a) for a, b in zip(vs, vs[1:])]
|
|
||||||
b = statistics.median(d) if d else 0
|
|
||||||
return b if b >= ROUNDING else 0
|
|
||||||
|
|
||||||
|
|
||||||
def values(ms, col):
|
def values(ms, col):
|
||||||
@@ -70,20 +44,16 @@ def main():
|
|||||||
a.add_argument("--end", type=parse_time)
|
a.add_argument("--end", type=parse_time)
|
||||||
o = a.parse_args()
|
o = a.parse_args()
|
||||||
|
|
||||||
by = load(o.folder)
|
by, names = sensors(o.folder, o.sensors)
|
||||||
if not by:
|
|
||||||
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
|
|
||||||
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
|
|
||||||
unknown = [s for s in names if s not in by]
|
unknown = [s for s in names if s not in by]
|
||||||
if unknown:
|
if unknown:
|
||||||
sys.exit(f"Missing sensors: {', '.join(unknown)}. Present: {', '.join(sorted(by))}.")
|
sys.exit(f"Missing sensors: {', '.join(unknown)}. Present: {', '.join(sorted(by))}.")
|
||||||
|
|
||||||
t0 = min(m["t"] for s in names for m in by[s]).replace(hour=0, minute=0, second=0, microsecond=0)
|
key = per_step(min(m["t"] for s in names for m in by[s]), o.step)
|
||||||
steps = {}
|
steps = {}
|
||||||
for s in names:
|
for s in names:
|
||||||
for m in filter_window(by[s], o.start, o.end):
|
for m in filter_window(by[s], o.start, o.end):
|
||||||
step_start = t0 + ((m["t"] - t0) // o.step) * o.step
|
steps.setdefault(key(m["t"]), {}).setdefault(s, []).append(m)
|
||||||
steps.setdefault(step_start, {}).setdefault(s, []).append(m)
|
|
||||||
if not steps:
|
if not steps:
|
||||||
sys.exit("No measurement in the requested window.")
|
sys.exit("No measurement in the requested window.")
|
||||||
|
|
||||||
@@ -104,7 +74,7 @@ def main():
|
|||||||
if o.mark:
|
if o.mark:
|
||||||
print("*: at least one measurement of the step is rejected; o: at least one measurement of the step has inlier = 0")
|
print("*: at least one measurement of the step is rejected; o: at least one measurement of the step has inlier = 0")
|
||||||
header = f"{'start':17} " + " ".join(f"{s:>9}" for s in names)
|
header = f"{'start':17} " + " ".join(f"{s:>9}" for s in names)
|
||||||
wanted = {t0 + ((parse_time(x) - t0) // o.step) * o.step for x in o.instants.split(",")} if o.instants else None
|
wanted = {key(parse_time(x)) for x in o.instants.split(",")} if o.instants else None
|
||||||
for i, k in enumerate(k for k in sorted(steps) if wanted is None or k in wanted):
|
for i, k in enumerate(k for k in sorted(steps) if wanted is None or k in wanted):
|
||||||
if i % 24 == 0:
|
if i % 24 == 0:
|
||||||
print(header)
|
print(header)
|
||||||
|
|||||||
@@ -32,18 +32,16 @@ import matplotlib
|
|||||||
matplotlib.use("Agg")
|
matplotlib.use("Agg")
|
||||||
import matplotlib.pyplot as plt # noqa: E402
|
import matplotlib.pyplot as plt # noqa: E402
|
||||||
|
|
||||||
|
from common import check # noqa: E402
|
||||||
from data import find_files, image_time, iso, load, parse_time # noqa: E402
|
from data import find_files, image_time, iso, load, parse_time # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def percentile(grey, p):
|
def percentile(grey, p):
|
||||||
"""Percentile p (0 to 100) of the pixels of a thumbnail, without interpolation."""
|
"""Percentile p (0 to 100) of the pixels of a thumbnail, without interpolation."""
|
||||||
v = sorted(grey.ravel().tolist())
|
v = sorted(grey.ravel().tolist())
|
||||||
return v[min(int(p / 100 * len(v)), len(v) - 1)]
|
return v[min(int(p / 100 * len(v)), len(v) - 1)]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
a.add_argument("folder")
|
a.add_argument("folder")
|
||||||
@@ -62,9 +60,7 @@ def main():
|
|||||||
zone = tuple(map(int, o.zone.split(","))) if o.zone else None
|
zone = tuple(map(int, o.zone.split(","))) if o.zone else None
|
||||||
if not names and not zone:
|
if not names and not zone:
|
||||||
sys.exit("Give --sensors, or --zone X,Y,W,H.")
|
sys.exit("Give --sensors, or --zone X,Y,W,H.")
|
||||||
missing = [s for s in names if s not in by]
|
check("sensors", names, by)
|
||||||
if missing:
|
|
||||||
sys.exit(f"Unknown sensors: {', '.join(missing)}. Known: {', '.join(sorted(by))}.")
|
|
||||||
known = sorted(h for h in images if h)
|
known = sorted(h for h in images if h)
|
||||||
if not known:
|
if not known:
|
||||||
sys.exit(f"No image in {o.folder}.")
|
sys.exit(f"No image in {o.folder}.")
|
||||||
|
|||||||
Reference in new issue
Block a user