"""Overview of all the sensors of an "advanced" export: what the sensors do together. Usage: .venv/bin/python tools/scripts/overview.py data/PS_DES_COUDRES/ [--columns x,y,sigma0] [--column error_value] [--step 15min] [--k 5] [--min 5] [--residuals corr_x,corr_y,error_value] [--outside T1/T2] [--others] [--rigid BASE [--without A,B] [--follow A,B]] [--table T1/T2|no] [--sensors A,B] [--start ...] [--end ...] [--out FOLDER] Each sensor is compared with its own level over the window --start/--end (median and MAD of the window). Two PNGs, whose paths are printed: 1. "common": first, if the folder has camera movements (camera/), one panel per pose quantity (pan, tilt, rotation [deg], scale, sigma0 of the registration), raw values, one point per registration; then one panel per column of --columns (default x,y,sigma0: raw position and sigma0 of the camera): median, over the sensors, of each sensor's deviation from its median; grey band = 10th to 90th percentile between sensors (the spread); on each panel except sigma0, in addition, each global reference separately (its deviation from its median, one line per reference) and the median of the targets alone (black line): one sees whether the references move in the image with the targets or without them; last panel: number of sensors whose --column (default error_value) exceeds its median by more than --k MAD at each step. 2. "multiples": one small plot per sensor (references first) of --column, median per step of --step, dotted line = median of the window; each plot has its own scale, a plateau shows. 3. "residuals-" (one PNG per column of --residuals, default corr_x,corr_y,error_value): one small plot per sensor of the column minus the sensor's own level (its median over the window, or outside --outside T1/T2 if given: the level is then that of the rest of the window, not that of the moment being read), one scale per sensor, zero dotted; a deviation of a few hundredths on a `corr` column that the parallel bands of `plot.py` hide shows there. Prints, per sensor, the level, the own noise (median of the differences between two consecutive measurements) and the minimum and maximum deviation of the median per step of --step, with their time, in units of the column and in noises (noise 0 and "-" in noises: values constant up to rounding, 1e-9; --residuals no: no residuals PNG). With --others, each sensor is in addition brought back to the deviation from the median of the OTHER sensors at the same time (one subtracts, at each measurement, the median of the other sensors of the same column minus their own level): what is shared by all (the remainder of the compensation of a common movement) cancels out, what is specific to the sensor remains. PNG "residuals--others"; the table also prints, per sensor, the slope of this residual on each column of --columns (common median, e.g. y) and the correlation r: a residual proportional to the common movement (even if it is not shared by all) has a non-zero slope and a high r. 4. "rigid" (with --rigid BASE): at each measurement, the overall movement of the targets since time BASE (similarity: translation, rotation, scale, estimated by medians over the targets, except --without); for each sensor, the residual dx, dy = raw position minus its position at BASE carried by this movement [px]. Panels: dx and dy of each global reference (one line each) against the median and the 10th-90th percentile of the targets; median and 90th percentile of |residual| of the targets; fitted rotation and scale; reproj_err of the references. Prints the table per step of --step (rot_deg, scale, tx, ty, tgt_med, tgt_max, then dx, dy of each reference and of each sensor of --follow A,B), then per sensor the min and the max of dx, dy with their time and the value at the last time. Rejected measurements enter neither the fit, nor the band and the median of the targets, tgt_med, tgt_max, nor the columns of the table; on the PNG, those of the references are crosses, which do not set the scale; in the per-sensor summary, they come separately (number, min and max). Also prints the table of the common median (one line per step of --step: time, deviation from the median of the window for each column of --columns, then _targets, the median of the targets alone, then the raw camera pose, empty without registration) then the minimum and the maximum of each column with their time, and those of each global reference; to date a start or an end, the table is at the step of --step (default 15min, the measurement cadence; a wider step, 1h for example, flattens a peak of one to four steps and dates to the hour). Also prints, for each time when at least --min sensors exceed their median by more than --k MAD, the number and the sensors (--min 1: also the times when a single sensor leaves its level), and the measurement times without a camera pose (no registration). The per-step tables (common median, --rigid) repeat their header every 24 lines. With --table T1/T2, they are printed only between T1 and T2 ("no": not at all); levels, minimum and maximum, PNGs and the rest always cover the whole window --start/--end: over a month, one reads all the rest, then the table of one day at the same levels. Gives values, not a verdict. """ import argparse import math import os import statistics import sys import matplotlib matplotlib.use("Agg") import matplotlib.dates as mdates # noqa: E402 import matplotlib.pyplot as plt # noqa: E402 from common import COLUMNS, bound, check, duration, extremes, outside, own_noise, per_step, sensors, window_filter, without_pose # noqa: E402 from data import POSE, filter_window, is_reference, iso, load_camera, parse_time, rejected # noqa: E402 def level(ms, col): """(median, MAD) of the non-empty values; None if there are none.""" v = [m[col] for m in ms if m[col] is not None] if not v: return None med = statistics.median(v) return med, statistics.median(abs(x - med) for x in v) def similarity(p, q): """Similarity (a, b, tx, ty) that sends the points p onto q: q ≈ (a·x − b·y + tx, b·x + a·y + ty). Estimation by medians (a target that jumps alone does not pull the fit): rotation and scale, median over all pairs of targets of the complex ratio (qi − qj) / (pi − pj); translation, median of q − (rotation, scale)·p. """ p, q = [complex(*u) for u in p], [complex(*u) for u in q] z = [(q[i] - q[j]) / (p[i] - p[j]) for i in range(len(p)) for j in range(i) if p[i] != p[j]] r = complex(statistics.median(v.real for v in z), statistics.median(v.imag for v in z)) d = [w - r * u for u, w in zip(p, q)] return r.real, r.imag, statistics.median(v.real for v in d), statistics.median(v.imag for v in d) def rigid(o, names, series, times, base, station): """4. Each sensor against the overall movement of the targets since time --rigid: PNG "rigid" and table.""" t_b = min(times, key=lambda t: abs(t - o.rigid)) pos = {s: {m["t"]: (m["x"], m["y"]) for m in series[s] if m["x"] is not None and m["y"] is not None} for s in names} without = set(o.without.split(",")) - {""} acc = {s: {m["t"] for m in series[s] if not rejected(m)} for s in names} # accepted measurements: the only ones to enter the fit and the target statistics refs = [s for s in names if series[s] and is_reference(series[s])] tgt = [s for s in names if s not in refs and t_b in pos[s]] fitted = [s for s in tgt if s not in without and t_b in acc[s]] followed = [s for s in o.follow.split(",") if s and s not in refs] res, fit = {s: {} for s in names}, {} # res: {sensor: {t: (dx, dy)}}; fit: {t: (rotation deg, scale, tx, ty, median |r|, max of the targets)} for t in times: ok = [s for s in fitted if t in pos[s] and t in acc[s]] if len(ok) < 2: continue a, b, tx, ty = similarity([pos[s][t_b] for s in ok], [pos[s][t] for s in ok]) for s in names: if t in pos[s] and t_b in pos[s]: (x0, y0), (x, y) = pos[s][t_b], pos[s][t] res[s][t] = (x - (a * x0 - b * y0 + tx), y - (b * x0 + a * y0 + ty)) r = [math.hypot(*res[s][t]) for s in tgt if t in res[s] and t in acc[s]] fit[t] = (math.degrees(math.atan2(b, a)), math.hypot(a, b), tx, ty, statistics.median(r), max(r)) if not fit: print(f"\n--rigid: fewer than two targets with x, y at {iso(t_b)} and after.") return tf = sorted(fit) reproj = {s: {m["t"]: m["reproj_err"] for m in series[s] if m["reproj_err"] is not None} for s in refs} panels = [("dx", 0), ("dy", 1)] fig, axes = plt.subplots(6, 1, sharex=True, figsize=(14, 18), squeeze=False) for ax, (name, k) in zip(axes[:2, 0], panels): band = [sorted(res[s][t][k] for s in tgt if t in res[s] and t in acc[s]) for t in tf] q = [statistics.quantiles(v, n=10, method="inclusive") if len(v) >= 2 else [v[0]] * 9 for v in band] ax.fill_between(tf, [x[0] for x in q], [x[-1] for x in q], color="0.85", label="targets: 10th-90th percentile") ax.plot(tf, [statistics.median(v) for v in band], "-", lw=1.2, color="k", label="targets: median") for i, s in enumerate(refs): ta = [t for t in res[s] if t in acc[s]] ax.plot(ta, [res[s][t][k] for t in ta], "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (ref.)") frame = ax.get_ylim() # the scale of the accepted measurements: a reference rejected far away is out of frame for i, s in enumerate(refs): tr = [t for t in res[s] if t not in acc[s]] if tr: # rejected measurements: separately, one cross each, off the line ax.plot(tr, [res[s][t][k] for t in tr], "x", ms=3, color=f"C{(i + 1) % 10}", label=f"{s} rejected") ax.set_ylim(frame) ax.set_ylabel(f"{name} [px]\nraw position minus\nthe targets' movement") ax.legend(loc="upper left", fontsize=8, ncol=1 + len(refs) // 4) axes[2, 0].plot(tf, [fit[t][4] for t in tf], "-", lw=1, color="k", label="targets: median of |residual|") p90 = {t: statistics.quantiles([math.hypot(*res[s][t]) for s in tgt if t in res[s] and t in acc[s]], n=10, method="inclusive")[-1] for t in tf} axes[2, 0].plot(tf, [p90[t] for t in tf], "-", lw=0.8, color="C3", label="targets: 90th percentile of |residual| (the max is in the table)") axes[2, 0].set_ylabel("|residual| of the targets [px]") axes[2, 0].legend(loc="upper left", fontsize=8) axes[3, 0].plot(tf, [fit[t][0] for t in tf], "-", lw=1, color="C2") axes[3, 0].set_ylabel("fitted rotation [deg]") axes[4, 0].plot(tf, [fit[t][1] for t in tf], "-", lw=1, color="C2") axes[4, 0].set_ylabel("fitted scale") for i, s in enumerate(refs): axes[5, 0].plot(list(reproj[s]), list(reproj[s].values()), "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (ref.)") axes[5, 0].set_ylabel("reproj_err\nof the references") if refs: axes[5, 0].legend(loc="upper left", fontsize=8, ncol=1 + len(refs) // 4) for ax in axes[:, 0]: ax.grid(alpha=0.4) ax.axvline(t_b, color="C3", ls=":", lw=0.8) axes[-1, 0].set_xlim(tf[0], tf[-1]) axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ")) fig.suptitle(f"{station}: overall movement (translation, rotation, scale) fitted on {len(fitted)} targets since {iso(t_b)} (dotted), " "residual of each sensor, accepted measurements (cross: rejected reference) (UTC times)") fig.tight_layout(rect=(0, 0, 1, 1 - 0.4 / fig.get_figheight())) path = base.format("rigid") fig.savefig(path, dpi=100, bbox_inches="tight") plt.close(fig) print(path) print(f"\nOverall movement fitted on the targets since {iso(t_b)}: {', '.join(fitted)}" + (f" (outside the fit: {', '.join(sorted(without & set(tgt)))})" if without & set(tgt) else "") + f".\nResidual = raw position minus the position at {iso(t_b)} carried by this movement [px]; rejected measurements left out of the fit, " f"of tgt_med, tgt_max and of the sensor columns (separately in the per-sensor summary); median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:") cols = ["rot_deg", "scale", "tx", "ty", "tgt_med", "tgt_max"] + [f"{s}_{c}" for s in refs + followed for c in ("dx", "dy")] key, rows = per_step(tf[0], o.step), {} for t in tf: line = rows.setdefault(key(t), {}) for c, v in zip(cols, fit[t]): line.setdefault(c, []).append(v) for s in refs + followed: if t in res[s] and t in acc[s]: line.setdefault(f"{s}_dx", []).append(res[s][t][0]), line.setdefault(f"{s}_dy", []).append(res[s][t][1]) w = max(10, max(len(c) for c in cols) + 1) header = f"{'time':17}" + "".join(f"{c:>{w}}" for c in cols) for i, t in enumerate(t for t in sorted(rows) if o.in_table(t)): if i % 24 == 0: print(header) print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):{w}.{7 if c == 'scale' else 3}g}" if c in rows[t] else " " * w for c in cols)) print("\nPer sensor, accepted measurements: min and max of dx, dy, value at the last time; then the rejected ones separately:") for s in names: good = {t: v for t, v in res[s].items() if t in acc[s]} rej = {t: v for t, v in res[s].items() if t not in acc[s]} for k, c in ((0, "dx"), (1, "dy")): line = f"{s:9}{' (ref.)' if s in refs else ' '} {c}" if good: lo, hi = extremes({t: v[k] for t, v in good.items()}) last = res[s].get(tf[-1]) line += (f" min {lo[1]:+.3g} at {iso(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}, at the last time " f"{last[k] if last else math.nan:+.3g}{' (rejected)' if last and tf[-1] not in acc[s] else ''}") else: line += " no accepted measurement" if rej: lo, hi = extremes({t: v[k] for t, v in rej.items()}) line += f"; {len(rej)} rejected: min {lo[1]:+.3g} at {iso(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}" if good or rej: print(line) def main(): a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a.add_argument("folder") a.add_argument("--columns", default="x,y,sigma0") a.add_argument("--column", default="error_value", choices=COLUMNS) a.add_argument("--step", default="15min", type=duration, help="step of the tables and of the blue curves (default 15min, the measurement cadence: a wider step flattens and shifts short events)") a.add_argument("--k", type=float, default=5, help="deviation counted, in MAD of the window") a.add_argument("--min", type=int, default=5, help="number of sensors from which a time is printed") a.add_argument("--residuals", default="corr_x,corr_y,error_value", help="columns plotted minus each sensor's own level (\"no\" or '': none)") a.add_argument("--outside", help="T1/T2: the own level of the residuals is computed outside this moment") a.add_argument("--others", action="store_true", help="residuals also brought back to the median of the other sensors at the same time, with slope and correlation on the common movement") a.add_argument("--rigid", type=parse_time, metavar="BASE", help="residuals against the overall movement of the targets (translation, rotation, scale) since time BASE") a.add_argument("--without", default="", help="A,B: targets left out of the fit of --rigid (their residual is still plotted)") a.add_argument("--follow", default="", help="A,B: with --rigid, dx, dy of these sensors added to the per-step table (the references are already there)") a.add_argument("--table", default="", metavar="T1/T2|no", help="per-step tables printed only between T1 and T2 (\"no\": not at all); the rest over the whole window") a.add_argument("--sensors", help="comma-separated list (default: all)") a.add_argument("--start", type=parse_time) a.add_argument("--end", type=parse_time) a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs") o = a.parse_args() by_sensor, names = sensors(o.folder, o.sensors) columns = o.columns.split(",") residuals = [c for c in o.residuals.split(",") if c and c != "no"] # '' or "no": none, as for --table kept = outside(o.outside) o.in_table = window_filter(o.table) check("sensors", names, by_sensor) check("sensors of --follow (taken from --sensors)", [s for s in o.follow.split(",") if s], names) check("columns", columns + residuals + [o.column], COLUMNS) series = {s: filter_window(by_sensor[s], o.start, o.end) for s in names} if not any(series.values()): sys.exit("No measurement in the requested window.") times = sorted({m["t"] for s in names for m in series[s]}) poses = filter_window(load_camera(o.folder), times[0].replace(second=0, microsecond=0), times[-1]) # poses to the minute, measurements to the second pose_cols = POSE if poses else () station = os.path.basename(os.path.normpath(o.folder)) os.makedirs(o.out, exist_ok=True) base = os.path.join(o.out, f"{station}_{{}}_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png") def deviations(col): """{sensor: {t: value - median of the window}}, and the MADs.""" e, mad = {}, {} for s in names: n = level(series[s], col) if n: e[s] = {m["t"]: m[col] - n[0] for m in series[s] if m[col] is not None} mad[s] = n[1] return e, mad # 1. common n_ax = len(pose_cols) + len(columns) + 1 fig, axes = plt.subplots(n_ax, 1, sharex=True, figsize=(14, 3 * n_ax), squeeze=False) for ax, col in zip(axes[:, 0], pose_cols): pose_t = {p["t"].replace(second=0, microsecond=0): p[col] for p in poses if p[col] is not None} cam_t = sorted(set(pose_t) | {t.replace(second=0, microsecond=0) for t in times}) # line broken at measurement times without registration: it does not join two poses across a gap ax.plot(cam_t, [pose_t.get(t, math.nan) for t in cam_t], ".-", ms=3, lw=0.5, color="C2", label="camera pose (one point per registration)") ax.set_ylabel(f"camera: {col}" + (" [deg]" if col in ("pan", "tilt", "rotation") else "")) ax.legend(loc="upper left", fontsize=8) commons, refs_per_col, ref_devs = {}, {}, {} # median per measurement of each column; references per column, their deviations for ax, col in zip(axes[len(pose_cols):, 0], columns): e, _ = deviations(col) med, low, high, tt = [], [], [], [] for t in times: v = sorted(e[s][t] for s in e if t in e[s]) if len(v) >= 2: q = statistics.quantiles(v, n=10, method="inclusive") tt.append(t), med.append(statistics.median(v)), low.append(q[0]), high.append(q[-1]) commons[col] = dict(zip(tt, med)) ax.fill_between(tt, low, high, color="0.85", label="10th-90th percentile between sensors") ax.plot(tt, med, ".-", ms=2, lw=0.8, label="median of the sensors") refs = [s for s in e if is_reference(series[s])] if refs and col != "sigma0": # sigma0 is the same for all: nothing to separate # global references one by one, and median of the targets alone: what moves in the image, the references or everything tgt = {} for t in times: v = [e[s][t] for s in e if s not in refs and t in e[s]] if v: tgt[t] = statistics.median(v) commons[col + "_targets"] = tgt ax.plot(list(tgt), list(tgt.values()), "-", lw=1.2, color="k", label="median of the targets alone") for i, s in enumerate(refs): ax.plot(list(e[s]), list(e[s].values()), "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (ref.)") refs_per_col[col] = refs ref_devs[col] = {s: e[s] for s in refs} ax.set_ylabel(f"{col}\ndeviation from its median") ax.grid(alpha=0.3) ax.legend(loc="upper left", fontsize=8, ncol=1 + len(refs_per_col.get(col, [])) // 4) e, mad = deviations(o.column) count, who = [], {} for t in times: highs = [s for s in e if t in e[s] and mad[s] > 0 and e[s][t] > o.k * mad[s]] count.append(len(highs)) if len(highs) >= o.min: who[t] = highs axes[-1, 0].step(times, count, where="mid", lw=0.8) axes[-1, 0].set_ylabel(f"sensors above\nmedian + {o.k:g} MAD\n({o.column})") axes[-1, 0].grid(alpha=0.3) h = next((h for h in (1, 2, 3, 6, 12) if (times[-1] - times[0]).total_seconds() / 3600 / h <= 20), 24) days = (times[-1] - times[0]).total_seconds() / 86400 # beyond 20 days, one tick per day overlaps: every n days, at most 20 ticks axes[-1, 0].xaxis.set_major_locator(mdates.HourLocator(byhour=range(0, 24, h)) if h < 24 else mdates.DayLocator(interval=math.ceil(days / 20))) axes[-1, 0].set_xlim(times[0], times[-1]) axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ")) for ax in axes[:, 0]: ax.grid(alpha=0.5, which="major") fig.suptitle(f"{station}: {len(e)} sensors together, each against its level over the window (UTC times)") fig.tight_layout(rect=(0, 0, 1, 1 - 0.4 / fig.get_figheight())) # room for the title common = base.format("common") fig.savefig(common, dpi=100, bbox_inches="tight") plt.close(fig) print(common) # table of the common median, per step key = per_step(times[0], o.step) # steps aligned on the clock (hh:00, hh:15…), not on the 1st measurement for col in pose_cols: commons["cam_" + col] = {p["t"]: p[col] for p in poses if p[col] is not None} rows = {} for col in columns + ["cam_" + c for c in pose_cols] + [c + "_targets" for c in refs_per_col]: for t, v in commons[col].items(): rows.setdefault(key(t), {}).setdefault(col, []).append(v) table_cols = columns + [c + "_targets" for c in refs_per_col] + ["cam_" + c for c in pose_cols] print(f"\nCommon median (deviation from the median of the window){' and raw camera pose (cam_, empty without registration)' if pose_cols else ''}, " f"median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:") header = f"{'time':17}" + "".join(f"{c:>13}" for c in table_cols) for i, t in enumerate(t for t in sorted(rows) if o.in_table(t)): if i % 24 == 0: print(header) print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):13.{6 if c.startswith('cam_') else 4}g}" if c in rows[t] else " " * 13 for c in table_cols)) for col in table_cols: if commons[col]: lo, hi = extremes(commons[col]) g = 6 if col.startswith("cam_") else 4 print(f"{col} (measurements): min {lo[1]:.{g}g} at {iso(lo[0])}, max {hi[1]:.{g}g} at {iso(hi[0])}") for col, refs in refs_per_col.items(): print(f"\n{col} raw of the global references, deviation from their median of the window (measurements); {col}_targets: median of the targets alone:") for s in refs: lo, hi = extremes(ref_devs[col][s]) rej = {m["t"] for m in series[s] if rejected(m)} r = lambda t: " (rejected)" if t in rej else "" print(f"{s:9} min {lo[1]:+.3g} at {iso(lo[0])}{r(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}{r(hi[0])}" + (f"; {len(rej)} rejected measurements in the window" if rej else "")) if not pose_cols: print("Camera pose: none in the window (no camera/ in the folder, or no registration).") else: runs = without_pose(times, poses) print(f"Measurement times without a camera pose (no registration): {sum(x[2] for x in runs)} of {len(times)}" + (": " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n in runs) if runs else "")) # 2. small multiples and 3. residuals: one small plot per sensor, one scale each mins = int(o.step.total_seconds() // 60) nrows = math.ceil(len(names) / 5) def sheet(): return plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False) def small(ax, pts, line, title): """Measurements {t: v} in grey, their median per step in blue (returned), a dotted line at `line`.""" steps = {} for t, v in pts.items(): steps.setdefault(key(t), []).append(v) pm = {t: statistics.median(v) for t, v in steps.items()} ax.plot(list(pts), list(pts.values()), "-", lw=0.4, color="0.7") ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0") if line is not None: ax.axhline(line, color="C3", ls=":", lw=0.8) ax.set_title(title, fontsize=9) ax.grid(alpha=0.3) ax.tick_params(labelsize=7) ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5)) ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh")) return pm def save(fig, axes, title, name): for ax in list(axes.flat)[len(names):]: ax.axis("off") fig.suptitle(title) fig.tight_layout() path = base.format(name) fig.savefig(path, dpi=100, bbox_inches="tight") plt.close(fig) print(path) col = o.column fig, axes = sheet() for ax, s in zip(axes.flat, names): lv = level(series[s], col) small(ax, {m["t"]: m[col] for m in series[s] if m[col] is not None}, lv and lv[0], s + (" (ref.)" if series[s] and is_reference(series[s]) else "")) save(fig, axes, f"{station}: {col}, median per step of {mins} min (blue), measurements (grey), " f"median of the window (dotted); one scale per sensor (UTC times)", f"multiples-{col}") def own_level(ms, col): # (median, MAD) outside --outside, or over the window when nothing is left return level([m for m in ms if kept(m["t"])] or ms, col) for col, others in [(c, False) for c in residuals] + [(c, True) for c in residuals if o.others]: fig, axes = sheet() dev = {} if others: # {sensor: {t: value - own level}} for the median of the others for q in names: mq = [m for m in series[q] if m[col] is not None] if nq := own_level(mq, col): dev[q] = {m["t"]: m[col] - nq[0] for m in mq} print(f"\n{col} minus the own level{' and minus the median of the other sensors at the same time' if others else ''} ({'outside ' + o.outside if o.outside else 'median of the window'}), median per step of " f"{mins} min:") slopes = [] print(f"{'sensor':9}{'level':>10}{'noise':>10}{'min dev.':>11} {'at':<17}{'(noises)':>9}{'max dev.':>11} {'at':<17}{'(noises)':>9}") for ax, s in zip(axes.flat, names): ms = [m for m in series[s] if m[col] is not None] lv = own_level(ms, col) if not lv: ax.axis("off") continue noise = own_noise([m[col] for m in ms]) if others: res = {} for m in ms: v = [dev[q][m["t"]] for q in dev if q != s and m["t"] in dev[q]] if len(v) >= 2: res[m["t"]] = m[col] - lv[0] - statistics.median(v) if len(res) < 2: ax.axis("off") continue else: res = {m["t"]: m[col] - lv[0] for m in ms} lo, hi = extremes(small(ax, res, 0, s + (" (ref.)" if is_reference(ms) else "") + f" noise {noise:.2g}")) f = lambda v: f"{v / noise:9.1f}" if noise else f"{'-':>9}" # noqa: E731 print(f"{s:9}{lv[0]:10.4g}{noise:10.3g}{lo[1]:11.3g} {iso(lo[0]):<17}{f(lo[1])}{hi[1]:11.3g} {iso(hi[0]):<17}{f(hi[1])}") if others: for c in columns: tt = [t for t in res if t in commons[c]] if len(tt) > 2 and len({commons[c][t] for t in tt}) > 1: xs, ys = [commons[c][t] for t in tt], [res[t] for t in tt] slope = statistics.linear_regression(xs, ys).slope slopes.append(f"{s:9}slope on common {c} {slope:+.4g} per unit, r = {statistics.correlation(xs, ys):+.2f}") if slopes: print("\n".join(slopes)) save(fig, axes, f"{station}: {col} minus the own level{' and the median of the others' if others else ''} of each sensor, median per step of " f"{mins} min (blue), measurements (grey); one scale per sensor (UTC times)", f"residuals-{col}" + ("-others" if others else "")) if o.rigid: rigid(o, names, series, times, base, station) print(f"\nTimes when at least {o.min} sensors have {o.column} > median of the window + {o.k:g} MAD:") for t, highs in who.items(): print(f"{iso(t)} {len(highs):3} {','.join(highs)}") if not who: print("(none)") if __name__ == "__main__": main()