491 lines
31 KiB
Python
491 lines
31 KiB
Python
"""Overview of all the sensors of an "advanced" export: what the sensors do together.
|
||||
|
|
|
|||
|
|
Usage: .venv/bin/python tools/scripts/overview.py data/PS_DES_COUDRES/
|
|||
|
|
[--columns x,y,sigma0] [--column error_value] [--step 15min] [--k 5] [--min 5]
|
|||
|
|
[--residuals corr_x,corr_y,error_value] [--outside T1/T2] [--others] [--rigid BASE [--without A,B] [--follow A,B]] [--table T1/T2|no] [--sensors A,B] [--start ...] [--end ...] [--out FOLDER]
|
|||
|
|
|
|||
|
|
Each sensor is compared with its own level over the window --start/--end (median and MAD of the window).
|
|||
|
|
Two PNGs, whose paths are printed:
|
|||
|
|
1. "common": first, if the folder has camera movements (camera/), one panel per pose quantity
|
|||
|
|
(pan, tilt, rotation [deg], scale, sigma0 of the registration), raw values, one point per registration; then one panel per column of --columns (default x,y,sigma0: raw position and sigma0 of the
|
|||
|
|
camera): median, over the sensors, of each sensor's deviation from its median; grey band = 10th to 90th
|
|||
|
|
percentile between sensors (the spread); on each panel except sigma0, in addition, each global reference separately
|
|||
|
|
(its deviation from its median, one line per reference) and the median of the targets alone (black line): one sees whether the
|
|||
|
|
references move in the image with the targets or without them; last panel: number of sensors whose --column
|
|||
|
|
(default error_value) exceeds its median by more than --k MAD at each step.
|
|||
|
|
2. "multiples": one small plot per sensor (references first) of --column, median per step of --step,
|
|||
|
|
dotted line = median of the window; each plot has its own scale, a plateau shows.
|
|||
|
|
3. "residuals-<column>" (one PNG per column of --residuals, default corr_x,corr_y,error_value): one small plot per
|
|||
|
|
sensor of the column minus the sensor's own level (its median over the window, or outside --outside T1/T2 if
|
|||
|
|
given: the level is then that of the rest of the window, not that of the moment being read), one scale per sensor, zero
|
|||
|
|
dotted; a deviation of a few hundredths on a `corr` column that the parallel bands of `plot.py` hide
|
|||
|
|
shows there. Prints, per sensor, the level, the own noise (median of the differences between two consecutive measurements) and
|
|||
|
|
the minimum and maximum deviation of the median per step of --step, with their time, in units of the column and in noises
|
|||
|
|
(noise 0 and "-" in noises: values constant up to rounding, 1e-9; --residuals no: no residuals PNG).
|
|||
|
|
With --others, each sensor is in addition brought back to the deviation from the median of the OTHER sensors at the same time (one
|
|||
|
|
subtracts, at each measurement, the median of the other sensors of the same column minus their own level): what is
|
|||
|
|
shared by all (the remainder of the compensation of a common movement) cancels out, what is specific to the sensor remains.
|
|||
|
|
PNG "residuals-<column>-others"; the table also prints, per sensor, the slope of this residual on each
|
|||
|
|
column of --columns (common median, e.g. y) and the correlation r: a residual proportional to the common movement
|
|||
|
|
(even if it is not shared by all) has a non-zero slope and a high r.
|
|||
|
|
4. "rigid" (with --rigid BASE): at each measurement, the overall movement of the targets since time BASE
|
|||
|
|
(similarity: translation, rotation, scale, estimated by medians over the targets, except --without); for each
|
|||
|
|
sensor, the residual dx, dy = raw position minus its position at BASE carried by this movement [px]. Panels:
|
|||
|
|
dx and dy of each global reference (one line each) against the median and the 10th-90th percentile of the targets;
|
|||
|
|
median and 90th percentile of |residual| of the targets; fitted rotation and scale; reproj_err of the references. Prints
|
|||
|
|
the table per step of --step (rot_deg, scale, tx, ty, tgt_med, tgt_max, then dx, dy of each reference and of each
|
|||
|
|
sensor of --follow A,B), then per sensor the min and the max of dx, dy with their time and the value at the last
|
|||
|
|
time. Rejected measurements enter neither the fit, nor the band and the median of the targets, tgt_med,
|
|||
|
|
tgt_max, nor the columns of the table; on the PNG, those of the references are crosses, which do not set the scale; in the per-sensor
|
|||
|
|
summary, they come separately (number, min and max).
|
|||
|
|
Also prints the table of the common median (one line per step of --step: time, deviation from the median of the window
|
|||
|
|
for each column of --columns, then <column>_targets, the median of the targets alone, then the raw camera pose, empty without registration) then the minimum and the maximum of each column with their time, and those of each global reference; to date
|
|||
|
|
a start or an end, the table is at the step of --step (default 15min, the measurement cadence; a wider step,
|
|||
|
|
1h for example, flattens a peak of one to four steps and dates to the hour).
|
|||
|
|
Also prints, for each time when at least --min sensors exceed their median by more than --k MAD,
|
|||
|
|
the number and the sensors (--min 1: also the times when a single sensor leaves its level), and the
|
|||
|
|
measurement times without a camera pose (no registration).
|
|||
|
|
The per-step tables (common median, --rigid) repeat their header every 24 lines. With --table T1/T2, they
|
|||
|
|
are printed only between T1 and T2 ("no": not at all); levels, minimum and maximum, PNGs and the rest always cover
|
|||
|
|
the whole window --start/--end: over a month, one reads all the rest, then the table of one day at the same levels.
|
|||
|
|
Gives values, not a verdict.
|
|||
|
|
"""
|
|||
|
|
import argparse
|
|||
|
|
import math
|
|||
|
|
import os
|
|||
|
|
import statistics
|
|||
|
|
import sys
|
|||
|
|
|
|||
|
|
import matplotlib
|
|||
|
|
|
|||
|
|
matplotlib.use("Agg")
|
|||
|
|
import matplotlib.dates as mdates # noqa: E402
|
|||
|
|
import matplotlib.pyplot as plt # noqa: E402
|
|||
|
|
|
|||
|
|
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402
|
|||
|
|
from plot import COLUMNS, bound # noqa: E402
|
|||
|
|
from summary import duration, own_noise # noqa: E402
|
|||
|
|
|
|||
|
|
|
|||
|
|
def level(ms, col):
|
|||
|
|
"""(median, MAD) of the non-empty values; None if there are none."""
|
|||
|
|
v = [m[col] for m in ms if m[col] is not None]
|
|||
|
|
if not v:
|
|||
|
|
return None
|
|||
|
|
med = statistics.median(v)
|
|||
|
|
return med, statistics.median(abs(x - med) for x in v)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def similarity(p, q):
|
|||
|
|
"""Similarity (a, b, tx, ty) that sends the points p onto q: q ≈ (a·x − b·y + tx, b·x + a·y + ty).
|
|||
|
|
|
|||
|
|
Estimation by medians (a target that jumps alone does not pull the fit): rotation and scale, median over
|
|||
|
|
all pairs of targets of the complex ratio (qi − qj) / (pi − pj); translation, median of q − (rotation, scale)·p.
|
|||
|
|
"""
|
|||
|
|
p, q = [complex(*u) for u in p], [complex(*u) for u in q]
|
|||
|
|
z = [(q[i] - q[j]) / (p[i] - p[j]) for i in range(len(p)) for j in range(i) if p[i] != p[j]]
|
|||
|
|
r = complex(statistics.median(v.real for v in z), statistics.median(v.imag for v in z))
|
|||
|
|
d = [w - r * u for u, w in zip(p, q)]
|
|||
|
|
return r.real, r.imag, statistics.median(v.real for v in d), statistics.median(v.imag for v in d)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def rigid(o, names, series, times, base, station):
|
|||
|
|
"""4. Each sensor against the overall movement of the targets since time --rigid: PNG "rigid" and table."""
|
|||
|
|
t_b = min(times, key=lambda t: abs(t - o.rigid))
|
|||
|
|
pos = {s: {m["t"]: (m["x"], m["y"]) for m in series[s] if m["x"] is not None and m["y"] is not None} for s in names}
|
|||
|
|
without = set(o.without.split(",")) - {""}
|
|||
|
|
acc = {s: {m["t"] for m in series[s] if not rejected(m)} for s in names} # accepted measurements: the only ones to enter the fit and the target statistics
|
|||
|
|
refs = [s for s in names if series[s] and is_reference(series[s])]
|
|||
|
|
tgt = [s for s in names if s not in refs and t_b in pos[s]]
|
|||
|
|
fitted = [s for s in tgt if s not in without and t_b in acc[s]]
|
|||
|
|
followed = [s for s in o.follow.split(",") if s and s not in refs]
|
|||
|
|
res, fit = {s: {} for s in names}, {} # res: {sensor: {t: (dx, dy)}}; fit: {t: (rotation deg, scale, tx, ty, median |r|, max of the targets)}
|
|||
|
|
for t in times:
|
|||
|
|
ok = [s for s in fitted if t in pos[s] and t in acc[s]]
|
|||
|
|
if len(ok) < 2:
|
|||
|
|
continue
|
|||
|
|
a, b, tx, ty = similarity([pos[s][t_b] for s in ok], [pos[s][t] for s in ok])
|
|||
|
|
for s in names:
|
|||
|
|
if t in pos[s] and t_b in pos[s]:
|
|||
|
|
(x0, y0), (x, y) = pos[s][t_b], pos[s][t]
|
|||
|
|
res[s][t] = (x - (a * x0 - b * y0 + tx), y - (b * x0 + a * y0 + ty))
|
|||
|
|
r = [math.hypot(*res[s][t]) for s in tgt if t in res[s] and t in acc[s]]
|
|||
|
|
fit[t] = (math.degrees(math.atan2(b, a)), math.hypot(a, b), tx, ty, statistics.median(r), max(r))
|
|||
|
|
if not fit:
|
|||
|
|
print(f"\n--rigid: fewer than two targets with x, y at {iso(t_b)} and after.")
|
|||
|
|
return
|
|||
|
|
tf = sorted(fit)
|
|||
|
|
reproj = {s: {m["t"]: m["reproj_err"] for m in series[s] if m["reproj_err"] is not None} for s in refs}
|
|||
|
|
panels = [("dx", 0), ("dy", 1)]
|
|||
|
|
fig, axes = plt.subplots(6, 1, sharex=True, figsize=(14, 18), squeeze=False)
|
|||
|
|
for ax, (name, k) in zip(axes[:2, 0], panels):
|
|||
|
|
band = [sorted(res[s][t][k] for s in tgt if t in res[s] and t in acc[s]) for t in tf]
|
|||
|
|
q = [statistics.quantiles(v, n=10, method="inclusive") if len(v) >= 2 else [v[0]] * 9 for v in band]
|
|||
|
|
ax.fill_between(tf, [x[0] for x in q], [x[-1] for x in q], color="0.85", label="targets: 10th-90th percentile")
|
|||
|
|
ax.plot(tf, [statistics.median(v) for v in band], "-", lw=1.2, color="k", label="targets: median")
|
|||
|
|
for i, s in enumerate(refs):
|
|||
|
|
ta = [t for t in res[s] if t in acc[s]]
|
|||
|
|
ax.plot(ta, [res[s][t][k] for t in ta], "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (ref.)")
|
|||
|
|
frame = ax.get_ylim() # the scale of the accepted measurements: a reference rejected far away is out of frame
|
|||
|
|
for i, s in enumerate(refs):
|
|||
|
|
tr = [t for t in res[s] if t not in acc[s]]
|
|||
|
|
if tr: # rejected measurements: separately, one cross each, off the line
|
|||
|
|
ax.plot(tr, [res[s][t][k] for t in tr], "x", ms=3, color=f"C{(i + 1) % 10}", label=f"{s} rejected")
|
|||
|
|
ax.set_ylim(frame)
|
|||
|
|
ax.set_ylabel(f"{name} [px]\nraw position minus\nthe targets' movement")
|
|||
|
|
ax.legend(loc="upper left", fontsize=8, ncol=1 + len(refs) // 4)
|
|||
|
|
axes[2, 0].plot(tf, [fit[t][4] for t in tf], "-", lw=1, color="k", label="targets: median of |residual|")
|
|||
|
|
p90 = {t: statistics.quantiles([math.hypot(*res[s][t]) for s in tgt if t in res[s] and t in acc[s]], n=10, method="inclusive")[-1] for t in tf}
|
|||
|
|
axes[2, 0].plot(tf, [p90[t] for t in tf], "-", lw=0.8, color="C3", label="targets: 90th percentile of |residual| (the max is in the table)")
|
|||
|
|
axes[2, 0].set_ylabel("|residual| of the targets [px]")
|
|||
|
|
axes[2, 0].legend(loc="upper left", fontsize=8)
|
|||
|
|
axes[3, 0].plot(tf, [fit[t][0] for t in tf], "-", lw=1, color="C2")
|
|||
|
|
axes[3, 0].set_ylabel("fitted rotation [deg]")
|
|||
|
|
axes[4, 0].plot(tf, [fit[t][1] for t in tf], "-", lw=1, color="C2")
|
|||
|
|
axes[4, 0].set_ylabel("fitted scale")
|
|||
|
|
for i, s in enumerate(refs):
|
|||
|
|
axes[5, 0].plot(list(reproj[s]), list(reproj[s].values()), "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (ref.)")
|
|||
|
|
axes[5, 0].set_ylabel("reproj_err\nof the references")
|
|||
|
|
if refs:
|
|||
|
|
axes[5, 0].legend(loc="upper left", fontsize=8, ncol=1 + len(refs) // 4)
|
|||
|
|
for ax in axes[:, 0]:
|
|||
|
|
ax.grid(alpha=0.4)
|
|||
|
|
ax.axvline(t_b, color="C3", ls=":", lw=0.8)
|
|||
|
|
axes[-1, 0].set_xlim(tf[0], tf[-1])
|
|||
|
|
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
|
|||
|
|
fig.suptitle(f"{station}: overall movement (translation, rotation, scale) fitted on {len(fitted)} targets since {iso(t_b)} (dotted), "
|
|||
|
|
"residual of each sensor, accepted measurements (cross: rejected reference) (UTC times)")
|
|||
|
|
fig.tight_layout(rect=(0, 0, 1, 1 - 0.4 / fig.get_figheight()))
|
|||
|
|
path = base.format("rigid")
|
|||
|
|
fig.savefig(path, dpi=100, bbox_inches="tight")
|
|||
|
|
plt.close(fig)
|
|||
|
|
print(path)
|
|||
|
|
|
|||
|
|
print(f"\nOverall movement fitted on the targets since {iso(t_b)}: {', '.join(fitted)}"
|
|||
|
|
+ (f" (outside the fit: {', '.join(sorted(without & set(tgt)))})" if without & set(tgt) else "")
|
|||
|
|
+ f".\nResidual = raw position minus the position at {iso(t_b)} carried by this movement [px]; rejected measurements left out of the fit, "
|
|||
|
|
f"of tgt_med, tgt_max and of the sensor columns (separately in the per-sensor summary); median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
|
|||
|
|
cols = ["rot_deg", "scale", "tx", "ty", "tgt_med", "tgt_max"] + [f"{s}_{c}" for s in refs + followed for c in ("dx", "dy")]
|
|||
|
|
midnight = tf[0].replace(hour=0, minute=0, second=0, microsecond=0)
|
|||
|
|
t0 = midnight + ((tf[0] - midnight) // o.step) * o.step
|
|||
|
|
per_step = {}
|
|||
|
|
for t in tf:
|
|||
|
|
line = per_step.setdefault(t0 + ((t - t0) // o.step) * o.step, {})
|
|||
|
|
for c, v in zip(cols, fit[t]):
|
|||
|
|
line.setdefault(c, []).append(v)
|
|||
|
|
for s in refs + followed:
|
|||
|
|
if t in res[s] and t in acc[s]:
|
|||
|
|
line.setdefault(f"{s}_dx", []).append(res[s][t][0]), line.setdefault(f"{s}_dy", []).append(res[s][t][1])
|
|||
|
|
w = max(10, max(len(c) for c in cols) + 1)
|
|||
|
|
header = f"{'time':17}" + "".join(f"{c:>{w}}" for c in cols)
|
|||
|
|
for i, t in enumerate(t for t in sorted(per_step) if o.in_table(t)):
|
|||
|
|
if i % 24 == 0:
|
|||
|
|
print(header)
|
|||
|
|
print(f"{iso(t):17}" + "".join(f"{statistics.median(per_step[t][c]):{w}.{7 if c == 'scale' else 3}g}" if c in per_step[t] else " " * w for c in cols))
|
|||
|
|
print("\nPer sensor, accepted measurements: min and max of dx, dy, value at the last time; then the rejected ones separately:")
|
|||
|
|
for s in names:
|
|||
|
|
good = {t: v for t, v in res[s].items() if t in acc[s]}
|
|||
|
|
rej = {t: v for t, v in res[s].items() if t not in acc[s]}
|
|||
|
|
for k, c in ((0, "dx"), (1, "dy")):
|
|||
|
|
line = f"{s:9}{' (ref.)' if s in refs else ' '} {c}"
|
|||
|
|
if good:
|
|||
|
|
lo, hi = min(good.items(), key=lambda x: x[1][k]), max(good.items(), key=lambda x: x[1][k])
|
|||
|
|
last = res[s].get(tf[-1])
|
|||
|
|
line += (f" min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}, at the last time "
|
|||
|
|
f"{last[k] if last else math.nan:+.3g}{' (rejected)' if last and tf[-1] not in acc[s] else ''}")
|
|||
|
|
else:
|
|||
|
|
line += " no accepted measurement"
|
|||
|
|
if rej:
|
|||
|
|
lo, hi = min(rej.items(), key=lambda x: x[1][k]), max(rej.items(), key=lambda x: x[1][k])
|
|||
|
|
line += f"; {len(rej)} rejected: min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}"
|
|||
|
|
if good or rej:
|
|||
|
|
print(line)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|||
|
|
a.add_argument("folder")
|
|||
|
|
a.add_argument("--columns", default="x,y,sigma0")
|
|||
|
|
a.add_argument("--column", default="error_value", choices=COLUMNS)
|
|||
|
|
a.add_argument("--step", default="15min", type=duration, help="step of the tables and of the blue curves (default 15min, the measurement cadence: a wider step flattens and shifts short events)")
|
|||
|
|
a.add_argument("--k", type=float, default=5, help="deviation counted, in MAD of the window")
|
|||
|
|
a.add_argument("--min", type=int, default=5, help="number of sensors from which a time is printed")
|
|||
|
|
a.add_argument("--residuals", default="corr_x,corr_y,error_value", help="columns plotted minus each sensor's own level (\"no\" or '': none)")
|
|||
|
|
a.add_argument("--outside", help="T1/T2: the own level of the residuals is computed outside this moment")
|
|||
|
|
a.add_argument("--others", action="store_true", help="residuals also brought back to the median of the other sensors at the same time, with slope and correlation on the common movement")
|
|||
|
|
a.add_argument("--rigid", type=parse_time, metavar="BASE", help="residuals against the overall movement of the targets (translation, rotation, scale) since time BASE")
|
|||
|
|
a.add_argument("--without", default="", help="A,B: targets left out of the fit of --rigid (their residual is still plotted)")
|
|||
|
|
a.add_argument("--follow", default="", help="A,B: with --rigid, dx, dy of these sensors added to the per-step table (the references are already there)")
|
|||
|
|
a.add_argument("--table", default="", metavar="T1/T2|no",
|
|||
|
|
help="per-step tables printed only between T1 and T2 (\"no\": not at all); the rest over the whole window")
|
|||
|
|
a.add_argument("--sensors", help="comma-separated list (default: all)")
|
|||
|
|
a.add_argument("--start", type=parse_time)
|
|||
|
|
a.add_argument("--end", type=parse_time)
|
|||
|
|
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
|
|||
|
|
o = a.parse_args()
|
|||
|
|
|
|||
|
|
by_sensor = load(o.folder)
|
|||
|
|
if not by_sensor:
|
|||
|
|
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
|
|||
|
|
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
|
|||
|
|
columns = o.columns.split(",")
|
|||
|
|
residuals = [c for c in o.residuals.split(",") if c and c != "no"] # '' or "no": none, as for --table
|
|||
|
|
outside = tuple(parse_time(x) for x in o.outside.split("/")) if o.outside else None
|
|||
|
|
if outside and len(outside) != 2:
|
|||
|
|
sys.exit("--outside expects T1/T2.")
|
|||
|
|
if o.table in ("", "no"):
|
|||
|
|
o.in_table = lambda t: o.table == ""
|
|||
|
|
else:
|
|||
|
|
limits = [parse_time(x) for x in o.table.split("/")]
|
|||
|
|
if len(limits) != 2:
|
|||
|
|
sys.exit("--table expects T1/T2 or no.")
|
|||
|
|
o.in_table = lambda t: limits[0] <= t <= limits[1]
|
|||
|
|
for requested, known, what in ((names, by_sensor, "sensors"), ([s for s in o.follow.split(",") if s], names, "sensors of --follow (taken from --sensors)"), (columns + residuals + [o.column], COLUMNS, "columns")):
|
|||
|
|
missing = [x for x in requested if x not in known]
|
|||
|
|
if missing:
|
|||
|
|
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
|
|||
|
|
series = {s: filter_window(by_sensor[s], o.start, o.end) for s in names}
|
|||
|
|
if not any(series.values()):
|
|||
|
|
sys.exit("No measurement in the requested window.")
|
|||
|
|
times = sorted({m["t"] for s in names for m in series[s]})
|
|||
|
|
poses = filter_window(load_camera(o.folder), times[0].replace(second=0, microsecond=0), times[-1]) # poses to the minute, measurements to the second
|
|||
|
|
pose_cols = POSE if poses else ()
|
|||
|
|
station = os.path.basename(os.path.normpath(o.folder))
|
|||
|
|
os.makedirs(o.out, exist_ok=True)
|
|||
|
|
base = os.path.join(o.out, f"{station}_{{}}_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png")
|
|||
|
|
|
|||
|
|
def deviations(col):
|
|||
|
|
"""{sensor: {t: value - median of the window}}, and the MADs."""
|
|||
|
|
e, mad = {}, {}
|
|||
|
|
for s in names:
|
|||
|
|
n = level(series[s], col)
|
|||
|
|
if n:
|
|||
|
|
e[s] = {m["t"]: m[col] - n[0] for m in series[s] if m[col] is not None}
|
|||
|
|
mad[s] = n[1]
|
|||
|
|
return e, mad
|
|||
|
|
|
|||
|
|
# 1. common
|
|||
|
|
n_ax = len(pose_cols) + len(columns) + 1
|
|||
|
|
fig, axes = plt.subplots(n_ax, 1, sharex=True, figsize=(14, 3 * n_ax), squeeze=False)
|
|||
|
|
for ax, col in zip(axes[:, 0], pose_cols):
|
|||
|
|
pose_t = {p["t"].replace(second=0, microsecond=0): p[col] for p in poses if p[col] is not None}
|
|||
|
|
cam_t = sorted(set(pose_t) | {t.replace(second=0, microsecond=0) for t in times})
|
|||
|
|
# line broken at measurement times without registration: it does not join two poses across a gap
|
|||
|
|
ax.plot(cam_t, [pose_t.get(t, math.nan) for t in cam_t], ".-", ms=3, lw=0.5, color="C2", label="camera pose (one point per registration)")
|
|||
|
|
ax.set_ylabel(f"camera: {col}" + (" [deg]" if col in ("pan", "tilt", "rotation") else ""))
|
|||
|
|
ax.legend(loc="upper left", fontsize=8)
|
|||
|
|
commons, refs_per_col, ref_devs = {}, {}, {}
|
|||
|
|
for ax, col in zip(axes[len(pose_cols):, 0], columns):
|
|||
|
|
e, _ = deviations(col)
|
|||
|
|
med, low, high, tt = [], [], [], []
|
|||
|
|
for t in times:
|
|||
|
|
v = sorted(e[s][t] for s in e if t in e[s])
|
|||
|
|
if len(v) >= 2:
|
|||
|
|
q = statistics.quantiles(v, n=10, method="inclusive")
|
|||
|
|
tt.append(t), med.append(statistics.median(v)), low.append(q[0]), high.append(q[-1])
|
|||
|
|
commons[col] = dict(zip(tt, med))
|
|||
|
|
ax.fill_between(tt, low, high, color="0.85", label="10th-90th percentile between sensors")
|
|||
|
|
ax.plot(tt, med, ".-", ms=2, lw=0.8, label="median of the sensors")
|
|||
|
|
refs = [s for s in e if is_reference(series[s])]
|
|||
|
|
if refs and col != "sigma0": # sigma0 is the same for all: nothing to separate
|
|||
|
|
# global references one by one, and median of the targets alone: what moves in the image, the references or everything
|
|||
|
|
tgt = {}
|
|||
|
|
for t in times:
|
|||
|
|
v = [e[s][t] for s in e if s not in refs and t in e[s]]
|
|||
|
|
if v:
|
|||
|
|
tgt[t] = statistics.median(v)
|
|||
|
|
commons[col + "_targets"] = tgt
|
|||
|
|
ax.plot(list(tgt), list(tgt.values()), "-", lw=1.2, color="k", label="median of the targets alone")
|
|||
|
|
for i, s in enumerate(refs):
|
|||
|
|
ax.plot(list(e[s]), list(e[s].values()), "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (ref.)")
|
|||
|
|
refs_per_col[col] = refs
|
|||
|
|
ref_devs[col] = {s: e[s] for s in refs}
|
|||
|
|
ax.set_ylabel(f"{col}\ndeviation from its median")
|
|||
|
|
ax.grid(alpha=0.3)
|
|||
|
|
ax.legend(loc="upper left", fontsize=8, ncol=1 + len(refs_per_col.get(col, [])) // 4)
|
|||
|
|
e, mad = deviations(o.column)
|
|||
|
|
count, who = [], {}
|
|||
|
|
for t in times:
|
|||
|
|
highs = [s for s in e if t in e[s] and mad[s] > 0 and e[s][t] > o.k * mad[s]]
|
|||
|
|
count.append(len(highs))
|
|||
|
|
if len(highs) >= o.min:
|
|||
|
|
who[t] = highs
|
|||
|
|
axes[-1, 0].step(times, count, where="mid", lw=0.8)
|
|||
|
|
axes[-1, 0].set_ylabel(f"sensors above\nmedian + {o.k:g} MAD\n({o.column})")
|
|||
|
|
axes[-1, 0].grid(alpha=0.3)
|
|||
|
|
h = next((h for h in (1, 2, 3, 6, 12) if (times[-1] - times[0]).total_seconds() / 3600 / h <= 20), 24)
|
|||
|
|
days = (times[-1] - times[0]).total_seconds() / 86400
|
|||
|
|
# beyond 20 days, one tick per day overlaps: every n days, at most 20 ticks
|
|||
|
|
axes[-1, 0].xaxis.set_major_locator(mdates.HourLocator(byhour=range(0, 24, h)) if h < 24 else mdates.DayLocator(interval=math.ceil(days / 20)))
|
|||
|
|
axes[-1, 0].set_xlim(times[0], times[-1])
|
|||
|
|
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
|
|||
|
|
for ax in axes[:, 0]:
|
|||
|
|
ax.grid(alpha=0.5, which="major")
|
|||
|
|
fig.suptitle(f"{station}: {len(e)} sensors together, each against its level over the window (UTC times)")
|
|||
|
|
fig.tight_layout(rect=(0, 0, 1, 1 - 0.4 / fig.get_figheight())) # room for the title
|
|||
|
|
common = base.format("common")
|
|||
|
|
fig.savefig(common, dpi=100, bbox_inches="tight")
|
|||
|
|
plt.close(fig)
|
|||
|
|
print(common)
|
|||
|
|
|
|||
|
|
# table of the common median, per step
|
|||
|
|
midnight = times[0].replace(hour=0, minute=0, second=0, microsecond=0)
|
|||
|
|
t0 = midnight + ((times[0] - midnight) // o.step) * o.step # steps aligned on the clock (hh:00, hh:15…), not on the 1st measurement
|
|||
|
|
rows = {}
|
|||
|
|
for col in columns:
|
|||
|
|
for t, v in commons[col].items():
|
|||
|
|
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col, []).append(v)
|
|||
|
|
for col in pose_cols:
|
|||
|
|
commons["cam_" + col] = {p["t"]: p[col] for p in poses if p[col] is not None}
|
|||
|
|
for t, v in commons["cam_" + col].items():
|
|||
|
|
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault("cam_" + col, []).append(v)
|
|||
|
|
for col in refs_per_col:
|
|||
|
|
for t, v in commons[col + "_targets"].items():
|
|||
|
|
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col + "_targets", []).append(v)
|
|||
|
|
table_cols = columns + [c + "_targets" for c in refs_per_col] + ["cam_" + c for c in pose_cols]
|
|||
|
|
print(f"\nCommon median (deviation from the median of the window){' and raw camera pose (cam_, empty without registration)' if pose_cols else ''}, "
|
|||
|
|
f"median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
|
|||
|
|
header = f"{'time':17}" + "".join(f"{c:>13}" for c in table_cols)
|
|||
|
|
for i, t in enumerate(t for t in sorted(rows) if o.in_table(t)):
|
|||
|
|
if i % 24 == 0:
|
|||
|
|
print(header)
|
|||
|
|
print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):13.{6 if c.startswith('cam_') else 4}g}" if c in rows[t] else " " * 13 for c in table_cols))
|
|||
|
|
for col in table_cols:
|
|||
|
|
if commons[col]:
|
|||
|
|
lo, hi = min(commons[col].items(), key=lambda x: x[1]), max(commons[col].items(), key=lambda x: x[1])
|
|||
|
|
g = 6 if col.startswith("cam_") else 4
|
|||
|
|
print(f"{col} (measurements): min {lo[1]:.{g}g} at {iso(lo[0])}, max {hi[1]:.{g}g} at {iso(hi[0])}")
|
|||
|
|
|
|||
|
|
for col, refs in refs_per_col.items():
|
|||
|
|
print(f"\n{col} raw of the global references, deviation from their median of the window (measurements); {col}_targets: median of the targets alone:")
|
|||
|
|
for s in refs:
|
|||
|
|
d = ref_devs[col][s]
|
|||
|
|
lo, hi = min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
|
|||
|
|
rej = {m["t"] for m in series[s] if rejected(m)}
|
|||
|
|
r = lambda t: " (rejected)" if t in rej else ""
|
|||
|
|
print(f"{s:9} min {lo[1]:+.3g} at {iso(lo[0])}{r(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}{r(hi[0])}"
|
|||
|
|
+ (f"; {len(rej)} rejected measurements in the window" if rej else ""))
|
|||
|
|
if not pose_cols:
|
|||
|
|
print("Camera pose: none in the window (no camera/ in the folder, or no registration).")
|
|||
|
|
else:
|
|||
|
|
posed = {p["t"].replace(second=0, microsecond=0) for p in poses}
|
|||
|
|
runs = [] # runs of consecutive measurement times without a pose: [start, end, number]
|
|||
|
|
for i, t in enumerate(times):
|
|||
|
|
if t.replace(second=0, microsecond=0) in posed:
|
|||
|
|
continue
|
|||
|
|
if runs and runs[-1][3] == i - 1:
|
|||
|
|
runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i
|
|||
|
|
else:
|
|||
|
|
runs.append([t, t, 1, i])
|
|||
|
|
print(f"Measurement times without a camera pose (no registration): {sum(x[2] for x in runs)} of {len(times)}"
|
|||
|
|
+ (": " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n, _ in runs) if runs else ""))
|
|||
|
|
|
|||
|
|
# 2. small multiples
|
|||
|
|
col = o.column
|
|||
|
|
n = len(names)
|
|||
|
|
nrows = math.ceil(n / 5)
|
|||
|
|
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
|
|||
|
|
for ax, s in zip(axes.flat, names):
|
|||
|
|
ms = [m for m in series[s] if m[col] is not None]
|
|||
|
|
ax.plot([m["t"] for m in ms], [m[col] for m in ms], "-", lw=0.4, color="0.7")
|
|||
|
|
steps = {}
|
|||
|
|
for m in ms:
|
|||
|
|
steps.setdefault(t0 + ((m["t"] - t0) // o.step) * o.step, []).append(m[col])
|
|||
|
|
ax.plot(list(steps), [statistics.median(v) for v in steps.values()], "-", lw=1.1, color="C0")
|
|||
|
|
lv = level(series[s], col)
|
|||
|
|
if lv:
|
|||
|
|
ax.axhline(lv[0], color="C3", ls=":", lw=0.8)
|
|||
|
|
ax.set_title(s + (" (ref.)" if series[s] and is_reference(series[s]) else ""), fontsize=9)
|
|||
|
|
ax.grid(alpha=0.3)
|
|||
|
|
ax.tick_params(labelsize=7)
|
|||
|
|
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
|
|||
|
|
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
|
|||
|
|
for ax in list(axes.flat)[n:]:
|
|||
|
|
ax.axis("off")
|
|||
|
|
fig.suptitle(f"{station}: {col}, median per step of {int(o.step.total_seconds() // 60)} min (blue), measurements (grey), "
|
|||
|
|
f"median of the window (dotted); one scale per sensor (UTC times)")
|
|||
|
|
fig.tight_layout()
|
|||
|
|
multiples = base.format(f"multiples-{col}")
|
|||
|
|
fig.savefig(multiples, dpi=100, bbox_inches="tight")
|
|||
|
|
print(multiples)
|
|||
|
|
|
|||
|
|
# 3. residuals: each sensor minus its own level
|
|||
|
|
for col, others in [(c, False) for c in residuals] + [(c, True) for c in residuals if o.others]:
|
|||
|
|
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
|
|||
|
|
dev = {}
|
|||
|
|
if others: # {sensor: {t: value - own level}} for the median of the others
|
|||
|
|
for q in names:
|
|||
|
|
mq = [m for m in series[q] if m[col] is not None]
|
|||
|
|
rq = [m for m in mq if not outside or not outside[0] <= m["t"] <= outside[1]] or mq
|
|||
|
|
nq = level(rq, col)
|
|||
|
|
if nq:
|
|||
|
|
dev[q] = {m["t"]: m[col] - nq[0] for m in mq}
|
|||
|
|
print(f"\n{col} minus the own level{' and minus the median of the other sensors at the same time' if others else ''} ({'outside ' + o.outside if outside else 'median of the window'}), median per step of "
|
|||
|
|
f"{int(o.step.total_seconds() // 60)} min:")
|
|||
|
|
slopes = []
|
|||
|
|
print(f"{'sensor':9}{'level':>10}{'noise':>10}{'min dev.':>11} {'at':<17}{'(noises)':>9}{'max dev.':>11} {'at':<17}{'(noises)':>9}")
|
|||
|
|
for ax, s in zip(axes.flat, names):
|
|||
|
|
ms = [m for m in series[s] if m[col] is not None]
|
|||
|
|
ref = [m for m in ms if not outside or not outside[0] <= m["t"] <= outside[1]] or ms
|
|||
|
|
lv = level(ref, col)
|
|||
|
|
if not lv:
|
|||
|
|
ax.axis("off")
|
|||
|
|
continue
|
|||
|
|
noise = own_noise([m[col] for m in ms])
|
|||
|
|
if others:
|
|||
|
|
res = {}
|
|||
|
|
for m in ms:
|
|||
|
|
v = [dev[q][m["t"]] for q in dev if q != s and m["t"] in dev[q]]
|
|||
|
|
if len(v) >= 2:
|
|||
|
|
res[m["t"]] = m[col] - lv[0] - statistics.median(v)
|
|||
|
|
if len(res) < 2:
|
|||
|
|
ax.axis("off")
|
|||
|
|
continue
|
|||
|
|
else:
|
|||
|
|
res = {m["t"]: m[col] - lv[0] for m in ms}
|
|||
|
|
steps = {}
|
|||
|
|
for t, v in res.items():
|
|||
|
|
steps.setdefault(t0 + ((t - t0) // o.step) * o.step, []).append(v)
|
|||
|
|
pm = {t: statistics.median(v) for t, v in steps.items()}
|
|||
|
|
ax.plot(list(res), list(res.values()), "-", lw=0.4, color="0.7")
|
|||
|
|
ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0")
|
|||
|
|
ax.axhline(0, color="C3", ls=":", lw=0.8)
|
|||
|
|
ax.set_title(s + (" (ref.)" if is_reference(ms) else "") + f" noise {noise:.2g}", fontsize=9)
|
|||
|
|
ax.grid(alpha=0.3)
|
|||
|
|
ax.tick_params(labelsize=7)
|
|||
|
|
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
|
|||
|
|
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
|
|||
|
|
lo, hi = min(pm.items(), key=lambda x: x[1]), max(pm.items(), key=lambda x: x[1])
|
|||
|
|
f = lambda v: f"{v / noise:9.1f}" if noise else f"{'-':>9}"
|
|||
|
|
print(f"{s:9}{lv[0]:10.4g}{noise:10.3g}{lo[1]:11.3g} {iso(lo[0]):<17}{f(lo[1])}{hi[1]:11.3g} {iso(hi[0]):<17}{f(hi[1])}")
|
|||
|
|
if others:
|
|||
|
|
for c in columns:
|
|||
|
|
tt = [t for t in res if t in commons[c]]
|
|||
|
|
if len(tt) > 2 and len({commons[c][t] for t in tt}) > 1:
|
|||
|
|
xs, ys = [commons[c][t] for t in tt], [res[t] for t in tt]
|
|||
|
|
slope = statistics.linear_regression(xs, ys).slope
|
|||
|
|
slopes.append(f"{s:9}slope on common {c} {slope:+.4g} per unit, r = {statistics.correlation(xs, ys):+.2f}")
|
|||
|
|
for ax in list(axes.flat)[n:]:
|
|||
|
|
ax.axis("off")
|
|||
|
|
if slopes:
|
|||
|
|
print("\n".join(slopes))
|
|||
|
|
fig.suptitle(f"{station}: {col} minus the own level{' and the median of the others' if others else ''} of each sensor, median per step of "
|
|||
|
|
f"{int(o.step.total_seconds() // 60)} min (blue), measurements (grey); one scale per sensor (UTC times)")
|
|||
|
|
fig.tight_layout()
|
|||
|
|
path = base.format(f"residuals-{col}" + ("-others" if others else ""))
|
|||
|
|
fig.savefig(path, dpi=100, bbox_inches="tight")
|
|||
|
|
plt.close(fig)
|
|||
|
|
print(path)
|
|||
|
|
|
|||
|
|
if o.rigid:
|
|||
|
|
rigid(o, names, series, times, base, station)
|
|||
|
|
|
|||
|
|
print(f"\nTimes when at least {o.min} sensors have {o.column} > median of the window + {o.k:g} MAD:")
|
|||
|
|
for t, highs in who.items():
|
|||
|
|
print(f"{iso(t)} {len(highs):3} {','.join(highs)}")
|
|||
|
|
if not who:
|
|||
|
|
print("(none)")
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|