Scripts: shared helpers in common.py, no tool imports another; CLI and outputs unchanged
data.py stays the reader (METRIC_RE now also captures the file date, same matches); common.py holds columns, steps, noise, sensor list, runs, file names. Duplicated run grouping, pose gaps, step bucketing, --table/--outside parsing removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c1719cbd1d
commit
6ed4c420e1
13 files changed
+308
-359
No files matched your search
+75
-111
@@ -62,9 +62,8 @@ matplotlib.use("Agg")
|
||||
import matplotlib.dates as mdates # noqa: E402
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402
|
||||
from plot import COLUMNS, bound # noqa: E402
|
||||
from summary import duration, own_noise # noqa: E402
|
||||
from common import COLUMNS, bound, check, duration, extremes, outside, own_noise, per_step, sensors, window_filter, without_pose # noqa: E402
|
||||
from data import POSE, filter_window, is_reference, iso, load_camera, parse_time, rejected # noqa: E402
|
||||
|
||||
|
||||
def level(ms, col):
|
||||
@@ -166,11 +165,9 @@ def rigid(o, names, series, times, base, station):
|
||||
+ f".\nResidual = raw position minus the position at {iso(t_b)} carried by this movement [px]; rejected measurements left out of the fit, "
|
||||
f"of tgt_med, tgt_max and of the sensor columns (separately in the per-sensor summary); median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
|
||||
cols = ["rot_deg", "scale", "tx", "ty", "tgt_med", "tgt_max"] + [f"{s}_{c}" for s in refs + followed for c in ("dx", "dy")]
|
||||
midnight = tf[0].replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
t0 = midnight + ((tf[0] - midnight) // o.step) * o.step
|
||||
per_step = {}
|
||||
key, rows = per_step(tf[0], o.step), {}
|
||||
for t in tf:
|
||||
line = per_step.setdefault(t0 + ((t - t0) // o.step) * o.step, {})
|
||||
line = rows.setdefault(key(t), {})
|
||||
for c, v in zip(cols, fit[t]):
|
||||
line.setdefault(c, []).append(v)
|
||||
for s in refs + followed:
|
||||
@@ -178,10 +175,10 @@ def rigid(o, names, series, times, base, station):
|
||||
line.setdefault(f"{s}_dx", []).append(res[s][t][0]), line.setdefault(f"{s}_dy", []).append(res[s][t][1])
|
||||
w = max(10, max(len(c) for c in cols) + 1)
|
||||
header = f"{'time':17}" + "".join(f"{c:>{w}}" for c in cols)
|
||||
for i, t in enumerate(t for t in sorted(per_step) if o.in_table(t)):
|
||||
for i, t in enumerate(t for t in sorted(rows) if o.in_table(t)):
|
||||
if i % 24 == 0:
|
||||
print(header)
|
||||
print(f"{iso(t):17}" + "".join(f"{statistics.median(per_step[t][c]):{w}.{7 if c == 'scale' else 3}g}" if c in per_step[t] else " " * w for c in cols))
|
||||
print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):{w}.{7 if c == 'scale' else 3}g}" if c in rows[t] else " " * w for c in cols))
|
||||
print("\nPer sensor, accepted measurements: min and max of dx, dy, value at the last time; then the rejected ones separately:")
|
||||
for s in names:
|
||||
good = {t: v for t, v in res[s].items() if t in acc[s]}
|
||||
@@ -189,15 +186,15 @@ def rigid(o, names, series, times, base, station):
|
||||
for k, c in ((0, "dx"), (1, "dy")):
|
||||
line = f"{s:9}{' (ref.)' if s in refs else ' '} {c}"
|
||||
if good:
|
||||
lo, hi = min(good.items(), key=lambda x: x[1][k]), max(good.items(), key=lambda x: x[1][k])
|
||||
lo, hi = extremes({t: v[k] for t, v in good.items()})
|
||||
last = res[s].get(tf[-1])
|
||||
line += (f" min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}, at the last time "
|
||||
line += (f" min {lo[1]:+.3g} at {iso(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}, at the last time "
|
||||
f"{last[k] if last else math.nan:+.3g}{' (rejected)' if last and tf[-1] not in acc[s] else ''}")
|
||||
else:
|
||||
line += " no accepted measurement"
|
||||
if rej:
|
||||
lo, hi = min(rej.items(), key=lambda x: x[1][k]), max(rej.items(), key=lambda x: x[1][k])
|
||||
line += f"; {len(rej)} rejected: min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}"
|
||||
lo, hi = extremes({t: v[k] for t, v in rej.items()})
|
||||
line += f"; {len(rej)} rejected: min {lo[1]:+.3g} at {iso(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}"
|
||||
if good or rej:
|
||||
print(line)
|
||||
|
||||
@@ -224,26 +221,14 @@ def main():
|
||||
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
|
||||
o = a.parse_args()
|
||||
|
||||
by_sensor = load(o.folder)
|
||||
if not by_sensor:
|
||||
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
|
||||
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
|
||||
by_sensor, names = sensors(o.folder, o.sensors)
|
||||
columns = o.columns.split(",")
|
||||
residuals = [c for c in o.residuals.split(",") if c and c != "no"] # '' or "no": none, as for --table
|
||||
outside = tuple(parse_time(x) for x in o.outside.split("/")) if o.outside else None
|
||||
if outside and len(outside) != 2:
|
||||
sys.exit("--outside expects T1/T2.")
|
||||
if o.table in ("", "no"):
|
||||
o.in_table = lambda t: o.table == ""
|
||||
else:
|
||||
limits = [parse_time(x) for x in o.table.split("/")]
|
||||
if len(limits) != 2:
|
||||
sys.exit("--table expects T1/T2 or no.")
|
||||
o.in_table = lambda t: limits[0] <= t <= limits[1]
|
||||
for requested, known, what in ((names, by_sensor, "sensors"), ([s for s in o.follow.split(",") if s], names, "sensors of --follow (taken from --sensors)"), (columns + residuals + [o.column], COLUMNS, "columns")):
|
||||
missing = [x for x in requested if x not in known]
|
||||
if missing:
|
||||
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
|
||||
kept = outside(o.outside)
|
||||
o.in_table = window_filter(o.table)
|
||||
check("sensors", names, by_sensor)
|
||||
check("sensors of --follow (taken from --sensors)", [s for s in o.follow.split(",") if s], names)
|
||||
check("columns", columns + residuals + [o.column], COLUMNS)
|
||||
series = {s: filter_window(by_sensor[s], o.start, o.end) for s in names}
|
||||
if not any(series.values()):
|
||||
sys.exit("No measurement in the requested window.")
|
||||
@@ -274,7 +259,7 @@ def main():
|
||||
ax.plot(cam_t, [pose_t.get(t, math.nan) for t in cam_t], ".-", ms=3, lw=0.5, color="C2", label="camera pose (one point per registration)")
|
||||
ax.set_ylabel(f"camera: {col}" + (" [deg]" if col in ("pan", "tilt", "rotation") else ""))
|
||||
ax.legend(loc="upper left", fontsize=8)
|
||||
commons, refs_per_col, ref_devs = {}, {}, {}
|
||||
commons, refs_per_col, ref_devs = {}, {}, {} # median per measurement of each column; references per column, their deviations
|
||||
for ax, col in zip(axes[len(pose_cols):, 0], columns):
|
||||
e, _ = deviations(col)
|
||||
med, low, high, tt = [], [], [], []
|
||||
@@ -329,19 +314,13 @@ def main():
|
||||
print(common)
|
||||
|
||||
# table of the common median, per step
|
||||
midnight = times[0].replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
t0 = midnight + ((times[0] - midnight) // o.step) * o.step # steps aligned on the clock (hh:00, hh:15…), not on the 1st measurement
|
||||
rows = {}
|
||||
for col in columns:
|
||||
for t, v in commons[col].items():
|
||||
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col, []).append(v)
|
||||
key = per_step(times[0], o.step) # steps aligned on the clock (hh:00, hh:15…), not on the 1st measurement
|
||||
for col in pose_cols:
|
||||
commons["cam_" + col] = {p["t"]: p[col] for p in poses if p[col] is not None}
|
||||
for t, v in commons["cam_" + col].items():
|
||||
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault("cam_" + col, []).append(v)
|
||||
for col in refs_per_col:
|
||||
for t, v in commons[col + "_targets"].items():
|
||||
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col + "_targets", []).append(v)
|
||||
rows = {}
|
||||
for col in columns + ["cam_" + c for c in pose_cols] + [c + "_targets" for c in refs_per_col]:
|
||||
for t, v in commons[col].items():
|
||||
rows.setdefault(key(t), {}).setdefault(col, []).append(v)
|
||||
table_cols = columns + [c + "_targets" for c in refs_per_col] + ["cam_" + c for c in pose_cols]
|
||||
print(f"\nCommon median (deviation from the median of the window){' and raw camera pose (cam_, empty without registration)' if pose_cols else ''}, "
|
||||
f"median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
|
||||
@@ -352,15 +331,14 @@ def main():
|
||||
print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):13.{6 if c.startswith('cam_') else 4}g}" if c in rows[t] else " " * 13 for c in table_cols))
|
||||
for col in table_cols:
|
||||
if commons[col]:
|
||||
lo, hi = min(commons[col].items(), key=lambda x: x[1]), max(commons[col].items(), key=lambda x: x[1])
|
||||
lo, hi = extremes(commons[col])
|
||||
g = 6 if col.startswith("cam_") else 4
|
||||
print(f"{col} (measurements): min {lo[1]:.{g}g} at {iso(lo[0])}, max {hi[1]:.{g}g} at {iso(hi[0])}")
|
||||
|
||||
for col, refs in refs_per_col.items():
|
||||
print(f"\n{col} raw of the global references, deviation from their median of the window (measurements); {col}_targets: median of the targets alone:")
|
||||
for s in refs:
|
||||
d = ref_devs[col][s]
|
||||
lo, hi = min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
|
||||
lo, hi = extremes(ref_devs[col][s])
|
||||
rej = {m["t"] for m in series[s] if rejected(m)}
|
||||
r = lambda t: " (rejected)" if t in rej else ""
|
||||
print(f"{s:9} min {lo[1]:+.3g} at {iso(lo[0])}{r(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}{r(hi[0])}"
|
||||
@@ -368,66 +346,71 @@ def main():
|
||||
if not pose_cols:
|
||||
print("Camera pose: none in the window (no camera/ in the folder, or no registration).")
|
||||
else:
|
||||
posed = {p["t"].replace(second=0, microsecond=0) for p in poses}
|
||||
runs = [] # runs of consecutive measurement times without a pose: [start, end, number]
|
||||
for i, t in enumerate(times):
|
||||
if t.replace(second=0, microsecond=0) in posed:
|
||||
continue
|
||||
if runs and runs[-1][3] == i - 1:
|
||||
runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i
|
||||
else:
|
||||
runs.append([t, t, 1, i])
|
||||
runs = without_pose(times, poses)
|
||||
print(f"Measurement times without a camera pose (no registration): {sum(x[2] for x in runs)} of {len(times)}"
|
||||
+ (": " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n, _ in runs) if runs else ""))
|
||||
+ (": " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n in runs) if runs else ""))
|
||||
|
||||
# 2. small multiples
|
||||
col = o.column
|
||||
n = len(names)
|
||||
nrows = math.ceil(n / 5)
|
||||
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
|
||||
for ax, s in zip(axes.flat, names):
|
||||
ms = [m for m in series[s] if m[col] is not None]
|
||||
ax.plot([m["t"] for m in ms], [m[col] for m in ms], "-", lw=0.4, color="0.7")
|
||||
# 2. small multiples and 3. residuals: one small plot per sensor, one scale each
|
||||
mins = int(o.step.total_seconds() // 60)
|
||||
nrows = math.ceil(len(names) / 5)
|
||||
|
||||
def sheet():
|
||||
return plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
|
||||
|
||||
def small(ax, pts, line, title):
|
||||
"""Measurements {t: v} in grey, their median per step in blue (returned), a dotted line at `line`."""
|
||||
steps = {}
|
||||
for m in ms:
|
||||
steps.setdefault(t0 + ((m["t"] - t0) // o.step) * o.step, []).append(m[col])
|
||||
ax.plot(list(steps), [statistics.median(v) for v in steps.values()], "-", lw=1.1, color="C0")
|
||||
lv = level(series[s], col)
|
||||
if lv:
|
||||
ax.axhline(lv[0], color="C3", ls=":", lw=0.8)
|
||||
ax.set_title(s + (" (ref.)" if series[s] and is_reference(series[s]) else ""), fontsize=9)
|
||||
for t, v in pts.items():
|
||||
steps.setdefault(key(t), []).append(v)
|
||||
pm = {t: statistics.median(v) for t, v in steps.items()}
|
||||
ax.plot(list(pts), list(pts.values()), "-", lw=0.4, color="0.7")
|
||||
ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0")
|
||||
if line is not None:
|
||||
ax.axhline(line, color="C3", ls=":", lw=0.8)
|
||||
ax.set_title(title, fontsize=9)
|
||||
ax.grid(alpha=0.3)
|
||||
ax.tick_params(labelsize=7)
|
||||
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
|
||||
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
|
||||
for ax in list(axes.flat)[n:]:
|
||||
ax.axis("off")
|
||||
fig.suptitle(f"{station}: {col}, median per step of {int(o.step.total_seconds() // 60)} min (blue), measurements (grey), "
|
||||
f"median of the window (dotted); one scale per sensor (UTC times)")
|
||||
fig.tight_layout()
|
||||
multiples = base.format(f"multiples-{col}")
|
||||
fig.savefig(multiples, dpi=100, bbox_inches="tight")
|
||||
print(multiples)
|
||||
return pm
|
||||
|
||||
def save(fig, axes, title, name):
|
||||
for ax in list(axes.flat)[len(names):]:
|
||||
ax.axis("off")
|
||||
fig.suptitle(title)
|
||||
fig.tight_layout()
|
||||
path = base.format(name)
|
||||
fig.savefig(path, dpi=100, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(path)
|
||||
|
||||
col = o.column
|
||||
fig, axes = sheet()
|
||||
for ax, s in zip(axes.flat, names):
|
||||
lv = level(series[s], col)
|
||||
small(ax, {m["t"]: m[col] for m in series[s] if m[col] is not None}, lv and lv[0],
|
||||
s + (" (ref.)" if series[s] and is_reference(series[s]) else ""))
|
||||
save(fig, axes, f"{station}: {col}, median per step of {mins} min (blue), measurements (grey), "
|
||||
f"median of the window (dotted); one scale per sensor (UTC times)", f"multiples-{col}")
|
||||
|
||||
def own_level(ms, col): # (median, MAD) outside --outside, or over the window when nothing is left
|
||||
return level([m for m in ms if kept(m["t"])] or ms, col)
|
||||
|
||||
# 3. residuals: each sensor minus its own level
|
||||
for col, others in [(c, False) for c in residuals] + [(c, True) for c in residuals if o.others]:
|
||||
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
|
||||
fig, axes = sheet()
|
||||
dev = {}
|
||||
if others: # {sensor: {t: value - own level}} for the median of the others
|
||||
for q in names:
|
||||
mq = [m for m in series[q] if m[col] is not None]
|
||||
rq = [m for m in mq if not outside or not outside[0] <= m["t"] <= outside[1]] or mq
|
||||
nq = level(rq, col)
|
||||
if nq:
|
||||
if nq := own_level(mq, col):
|
||||
dev[q] = {m["t"]: m[col] - nq[0] for m in mq}
|
||||
print(f"\n{col} minus the own level{' and minus the median of the other sensors at the same time' if others else ''} ({'outside ' + o.outside if outside else 'median of the window'}), median per step of "
|
||||
f"{int(o.step.total_seconds() // 60)} min:")
|
||||
print(f"\n{col} minus the own level{' and minus the median of the other sensors at the same time' if others else ''} ({'outside ' + o.outside if o.outside else 'median of the window'}), median per step of "
|
||||
f"{mins} min:")
|
||||
slopes = []
|
||||
print(f"{'sensor':9}{'level':>10}{'noise':>10}{'min dev.':>11} {'at':<17}{'(noises)':>9}{'max dev.':>11} {'at':<17}{'(noises)':>9}")
|
||||
for ax, s in zip(axes.flat, names):
|
||||
ms = [m for m in series[s] if m[col] is not None]
|
||||
ref = [m for m in ms if not outside or not outside[0] <= m["t"] <= outside[1]] or ms
|
||||
lv = level(ref, col)
|
||||
lv = own_level(ms, col)
|
||||
if not lv:
|
||||
ax.axis("off")
|
||||
continue
|
||||
@@ -443,20 +426,8 @@ def main():
|
||||
continue
|
||||
else:
|
||||
res = {m["t"]: m[col] - lv[0] for m in ms}
|
||||
steps = {}
|
||||
for t, v in res.items():
|
||||
steps.setdefault(t0 + ((t - t0) // o.step) * o.step, []).append(v)
|
||||
pm = {t: statistics.median(v) for t, v in steps.items()}
|
||||
ax.plot(list(res), list(res.values()), "-", lw=0.4, color="0.7")
|
||||
ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0")
|
||||
ax.axhline(0, color="C3", ls=":", lw=0.8)
|
||||
ax.set_title(s + (" (ref.)" if is_reference(ms) else "") + f" noise {noise:.2g}", fontsize=9)
|
||||
ax.grid(alpha=0.3)
|
||||
ax.tick_params(labelsize=7)
|
||||
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
|
||||
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
|
||||
lo, hi = min(pm.items(), key=lambda x: x[1]), max(pm.items(), key=lambda x: x[1])
|
||||
f = lambda v: f"{v / noise:9.1f}" if noise else f"{'-':>9}"
|
||||
lo, hi = extremes(small(ax, res, 0, s + (" (ref.)" if is_reference(ms) else "") + f" noise {noise:.2g}"))
|
||||
f = lambda v: f"{v / noise:9.1f}" if noise else f"{'-':>9}" # noqa: E731
|
||||
print(f"{s:9}{lv[0]:10.4g}{noise:10.3g}{lo[1]:11.3g} {iso(lo[0]):<17}{f(lo[1])}{hi[1]:11.3g} {iso(hi[0]):<17}{f(hi[1])}")
|
||||
if others:
|
||||
for c in columns:
|
||||
@@ -465,17 +436,10 @@ def main():
|
||||
xs, ys = [commons[c][t] for t in tt], [res[t] for t in tt]
|
||||
slope = statistics.linear_regression(xs, ys).slope
|
||||
slopes.append(f"{s:9}slope on common {c} {slope:+.4g} per unit, r = {statistics.correlation(xs, ys):+.2f}")
|
||||
for ax in list(axes.flat)[n:]:
|
||||
ax.axis("off")
|
||||
if slopes:
|
||||
print("\n".join(slopes))
|
||||
fig.suptitle(f"{station}: {col} minus the own level{' and the median of the others' if others else ''} of each sensor, median per step of "
|
||||
f"{int(o.step.total_seconds() // 60)} min (blue), measurements (grey); one scale per sensor (UTC times)")
|
||||
fig.tight_layout()
|
||||
path = base.format(f"residuals-{col}" + ("-others" if others else ""))
|
||||
fig.savefig(path, dpi=100, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(path)
|
||||
save(fig, axes, f"{station}: {col} minus the own level{' and the median of the others' if others else ''} of each sensor, median per step of "
|
||||
f"{mins} min (blue), measurements (grey); one scale per sensor (UTC times)", f"residuals-{col}" + ("-others" if others else ""))
|
||||
|
||||
if o.rigid:
|
||||
rigid(o, names, series, times, base, station)
|
||||
|
||||
Reference in new issue
Block a user