"""Several stations on one timeline: enclosure temperature, camera pose, corr of the targets, rejections, gaps. Usage: .venv/bin/python tools/scripts/stations.py data/BREITHORN data/MICHIGAN data/NEBRASKA [--columns temperature,cam_tilt,cam_rotation,cam_scale,tgt_corr_x,tgt_corr_y] [--step 1h] [--stat median|min|max] [--table T1/T2|no] [--start ...] [--end ...] [--out FOLDER] For each station (one station folder each), over the window --start/--end: - possible columns: temperature (median of the sensor_*_deg_c components, as in metrics.py), cam_pan, cam_tilt, cam_rotation, cam_scale, cam_sigma0 (raw camera pose), tgt_corr_x, tgt_corr_y (median over the targets of corr minus each target's own level, accepted measurements, as in metrics.py), rej_targets, rej_refs (number of rejected targets, of rejected references: the largest over the measurement times of the step); - prints, per station, the period of the measurements, the cadence (median step) and the number of times; then the **gaps**: the runs of steps of --step without a measurement or without temperature, with the stations where they are missing (a gap shared by several stations is on a single line); then the table, one line per step (start of the step, UTC; --stat of the values of the step; "-": no value), one column per station and per column, tab-separated (--table T1/T2: only between T1 and T2, "no": not at all; the rest covers the whole window); then, for each column and each pair of stations, the correlation r of the values and that of the steps (difference from one value to the next, over the steps where both stations have a value: a common movement shows on the steps, a common trend on the values), with the number of steps; - one PNG "stations": one panel per column, one line per station (raw temperature; the other columns minus their median over the window, so that they overlap), broken over a gap; measurement gaps shaded below the axis, one band per station. Never an identifier from metrics/ (SIM_CCID, ip*): only the temperatures are read there. Gives values, not a verdict. """ import argparse import os import statistics import sys import matplotlib matplotlib.use("Agg") import matplotlib.dates as mdates # noqa: E402 import matplotlib.pyplot as plt # noqa: E402 from common import STATS, bound, cut, duration, group, targets, window_filter # noqa: E402 from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time, rejected # noqa: E402 COLUMNS = ("temperature",) + tuple("cam_" + c for c in POSE) + ("tgt_corr_x", "tgt_corr_y", "rej_targets", "rej_refs") def read(folder, start, end): """Station folder -> ({column: [{'t', 'v'}]}, [measurement times]).""" by_sensor = load(folder) series = {"temperature": filter_window([{"t": t, "v": v} for t, v in load_temperature(folder)], start, end)} for p in filter_window(load_camera(folder), start, end): for c in POSE: if p[c] is not None: series.setdefault("cam_" + c, []).append({"t": p["t"], "v": p[c]}) for c in ("corr_x", "corr_y"): series["tgt_" + c] = targets(by_sensor, c, start, end) rej = {} # t -> [rejected targets, rejected references] for s, ms in by_sensor.items(): ref = is_reference(ms) for m in filter_window(ms, start, end): n = rej.setdefault(m["t"], [0, 0]) n[ref] += rejected(m) series["rej_targets"] = [{"t": t, "v": float(n[0])} for t, n in sorted(rej.items())] series["rej_refs"] = [{"t": t, "v": float(n[1])} for t, n in sorted(rej.items())] return series, sorted(rej) def correlation(a, b): try: return f"{statistics.correlation(a, b):+.2f}" except (statistics.StatisticsError, ValueError): return "-" def main(): a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a.add_argument("folders", nargs="+", help="station folders, at least two") a.add_argument("--columns", default="temperature,cam_tilt,cam_rotation,cam_scale,tgt_corr_x,tgt_corr_y") a.add_argument("--step", default="1h", type=duration) a.add_argument("--stat", default="median", choices=STATS) a.add_argument("--table", default="", metavar="T1/T2|no", help="table printed only between T1 and T2 (\"no\": not at all)") a.add_argument("--start", type=parse_time) a.add_argument("--end", type=parse_time) a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG") o = a.parse_args() columns = o.columns.split(",") if missing := [c for c in columns if c not in COLUMNS]: sys.exit(f"Unknown columns: {', '.join(missing)}. Known: {', '.join(COLUMNS)}.") in_table = window_filter(o.table) names = [os.path.basename(os.path.normpath(d)) for d in o.folders] by_station, measured = {}, {} for n, d in zip(names, o.folders): by_station[n], measured[n] = read(d, o.start, o.end) times = [p["t"] for s in by_station.values() for c in s.values() for p in c] if not times: sys.exit("No value in the requested window.") t0 = (o.start or min(times)).replace(minute=0, second=0, microsecond=0) t1 = o.end or max(times) grid = [t0 + i * o.step for i in range(int((t1 - t0) / o.step) + 1)] cell = lambda t: t0 + ((t - t0) // o.step) * o.step # noqa: E731 print("Stations:") for n in names: ts = measured[n] if not ts: print(f" {n}: no measurement in the window") continue cad = statistics.median(b - a_ for a_, b in zip(ts, ts[1:])) if len(ts) > 1 else None print(f" {n}: measurements {iso(ts[0])} → {iso(ts[-1])}, {len(ts)} times, cadence {round(cad.total_seconds() / 60) if cad else '-'} min; " + ", ".join(f"{c} {len(by_station[n].get(c, []))}" for c in columns)) # gaps: steps without a measurement, without temperature, and the stations concerned with_meas = {n: {cell(t) for t in measured[n]} for n in names} with_temp = {n: {cell(p["t"]) for p in by_station[n]["temperature"]} for n in names} print(f"\nGaps (steps of {int(o.step.total_seconds() // 60)} min without a measurement or without temperature), with the stations where they are missing:") keys = {} for k in grid: m = tuple(n for n in names if k not in with_meas[n]) t = tuple(n for n in names if k not in with_temp[n]) if m or t: keys.setdefault((m, t), []).append(k) lines = [(g[0], g[-1], len(g), m, t) for (m, t), ks in keys.items() for g in group(ks, lambda a, b: b - a == o.step)] for d, f, nb, m, t in sorted(lines): print(f" {iso(d)} → {iso(f)} ({nb} steps): no measurement {','.join(m) or '-'}; no temperature {','.join(t) or '-'}") if not lines: print(" (none)") stat = STATS[o.stat] table = {} # (station, column) -> {step: value} for n in names: for c in columns: cells = {} for p in by_station[n].get(c, []): cells.setdefault(cell(p["t"]), []).append(p["v"]) table[n, c] = {k: stat(v) for k, v in cells.items()} header = [f"{n}:{c}" for n in names for c in columns] print(f"\n{o.stat} per step of {int(o.step.total_seconds() // 60)} min (start of the step, UTC); \"-\": no value" + (f" (table: {o.table})" if o.table else "") + ":") if o.table != "no": print("\t".join(["time"] + header)) for k in grid: if in_table(k): v = [table[n, c].get(k) for n in names for c in columns] if any(x is not None for x in v): print("\t".join([iso(k)] + ["-" if x is None else f"{x:.6g}" for x in v])) print("\nCorrelation between stations, r of the values, then r of the steps from one value to the next (n: steps where both have a value):") for c in columns: for i, s1 in enumerate(names): for s2 in names[i + 1:]: ks = sorted(set(table[s1, c]) & set(table[s2, c])) v1, v2 = [table[s1, c][k] for k in ks], [table[s2, c][k] for k in ks] d1, d2 = [b - a_ for a_, b in zip(v1, v1[1:])], [b - a_ for a_, b in zip(v2, v2[1:])] print(f" {c:12} {s1}-{s2}: r values {correlation(v1, v2)}, r steps {correlation(d1, d2)} (n = {len(ks)})") fig, axes = plt.subplots(len(columns) + 1, 1, sharex=True, figsize=(14, 2.4 * (len(columns) + 1)), squeeze=False, layout="constrained", gridspec_kw={"height_ratios": [3] * len(columns) + [0.3 * len(names)]}) for ax, c in zip(axes[:-1, 0], columns): for i, n in enumerate(names): s = by_station[n].get(c, []) if not s: continue med = 0 if c == "temperature" else statistics.median(p["v"] for p in s) ts, vs = cut([{"t": p["t"], "v": p["v"] - med} for p in s]) ax.plot(ts, vs, ".-", ms=2, lw=0.8, color=f"C{i % 10}", label=n) ax.set_ylabel(c if c == "temperature" else f"{c}\nminus its median", fontsize=8) ax.grid(alpha=0.3) ax.legend(loc="upper left", bbox_to_anchor=(1.0, 1.0), fontsize=8) ax = axes[-1, 0] for i, n in enumerate(names): for k in grid: if k not in with_meas[n]: ax.axvspan(k, k + o.step, ymin=i / len(names), ymax=(i + 1) / len(names), color=f"C{i % 10}", lw=0) ax.set_yticks([(i + 0.5) / len(names) for i in range(len(names))], names, fontsize=7) ax.set_ylim(0, 1) ax.set_ylabel("no measurement", fontsize=8) ax.set_xlim(grid[0], grid[-1] + o.step) ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ")) fig.suptitle(f"{', '.join(names)}: same timeline (UTC times)") os.makedirs(o.out, exist_ok=True) path = os.path.join(o.out, f"{'-'.join(names)}_stations_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png") fig.savefig(path, dpi=100, bbox_inches="tight") print(f"\n{path}") if __name__ == "__main__": main()