"""What comes back from one day to the next: level of each night, and days overlaid by hour of the day, for the diagnostic agent. Usage: .venv/bin/python tools/scripts/days.py data/PS_DES_COUDRES/ [--columns corr_x,corr_y,error_value] [--night 20-3] [--k 5] [--days 2] [--step 15min] [--sensors A,B] [--bounds [--smoothing N] [--fraction F]] [--start ...] [--end ...] [--out FOLDER] Accepted measurements only: a rejected measurement (error_value above the threshold) enters nowhere. A "night": the hours --night H1-H2 (UTC, H1 included, H2 excluded, straddling midnight if H1 > H2), dated by the day it starts; the night in progress at the start of the window also counts (its hours after midnight). Own noise of the sensor: median of the differences between two consecutive measurements over the window. For each column of --columns: 1. Level of each night, per sensor: median, MAD and number of measurements; then, from one night to the next, the step (difference of the medians, in units and in noises) and where it falls: the run of night measurements of the two nights is cut where the sum of the absolute deviations from the level of the first night (before) and of the second (after) is smallest; "between A and B": last night measurement before the cut and first after (B at the start of a night: the cut falls between two nights, in daytime). 2. Overlaid days: each measurement minus the level of the last night started at its time (the previous one, or the one in progress; without it, the nearest night). PNG "byhour-": one small plot per sensor, on the x axis the hour of the day (UTC), one curve per day, night hours shaded, one scale per sensor, framed from the 1st to the 99th percentile widened by half (the title says how many measurements fall outside the frame: an isolated peak does not crush the scale; its value is in the tables). Table: per slot of --step of the hour of the day, the sensors whose deviation exceeds --k noises in that slot, on the same side, on at least --days different days, with the number of these days over the number of measured days and the median of these deviations (signed, in units and in noises); a sensor above on some days and below on others is cited twice. 3. With --bounds: each day against the night that precedes it, for the target median of each column (with --sensors: the median of these sensors, references included; a single sensor: its own) and for the pose of the camera (pan, tilt, rotation, scale, sigma0) if camera/ exists. A "day": from the end of one night to the end of the next (24 h), dated by the day it starts; its reference level is that of the night that has just ended (without it, the last night measured before, or at the start of the batch the first one: named, "not the previous one"). Deviation of each measurement from this level (target median: at each time, median of the deviations of each target from its own level of that night); noise: median of the differences between two consecutive values of this series. Per day: number of measurements, first and last; first and last time when |deviation| exceeds --k noises, with the measurement just before and just after (at the native step); the signed maximum and minimum and their time; around the larger of the two in absolute value, the run of consecutive measurements beyond --k noises on the same side that contains it, with the measurement just before and just after; the next night minus the reference night. A day without a measurement says so. Each day line starts with the short name of the series (the column, or pose_tilt…): `| grep '^pose_tilt '` keeps one, `| grep ' 12/10 '` one day. With --smoothing N, the deviation of each measurement is the median of the N measurements of the day centred on it (the noise stays that of the measurements, the figure also plots the raw deviation in grey): an isolated point no longer cuts the run around the extreme and no longer makes a first or a last exceedance. With --fraction F, the threshold of each day is F times the largest |deviation| of that day (instead of --k noises; it is printed on the day line): on a very noisy series, or one whose noise at the native step bears no relation to the amplitude of the cycle. PNG "bounds": one panel per series, the deviation over the batch, ±k noises dotted, each day's [first, last] shaded light and the run around the extreme shaded dark, the maximum and the minimum marked; framed from the 1st to the 99th percentile widened by half (the title says how many measurements fall outside the frame). Gives values, not a verdict. """ import argparse import math import os import statistics import sys from datetime import datetime, timedelta, timezone import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: E402 from common import COLUMNS, bound, check, duration, own_noise, sensors # noqa: E402 from data import POSE, filter_window, is_reference, iso, load_camera, parse_time, rejected # noqa: E402 def hours(text): """'20-3' -> (20, 3).""" try: h1, h2 = (int(x) for x in text.split("-")) except ValueError: raise argparse.ArgumentTypeError(f"invalid hours: {text!r} (e.g. 20-3)") return h1, h2 def at_night(t, h1, h2): return (h1 <= t.hour or t.hour < h2) if h1 > h2 else h1 <= t.hour < h2 def nights_of(pts, h1, h2): """[(t, v)] -> {date of the night: [(t, v)]}: the night hours, dated by the day the night starts.""" nights = {} for t, v in pts: if at_night(t, h1, h2): nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v)) return nights def frame(ax, values): """Frame from the 1st to the 99th percentile widened by half: an isolated peak does not crush the scale; -> the number of values out of frame.""" v = sorted(values) if not v: return 0 q1, q2 = v[len(v) // 100], v[-1 - len(v) // 100] margin = (q2 - q1) * 0.5 or 1 ax.set_ylim(q1 - margin, q2 + margin) return sum(not q1 - margin <= e <= q2 + margin for e in v) def cut_index(pts, la, lb): """Index j (1..n-1) where sum|v-la| before j + sum|v-lb| from j on is smallest.""" before, after = 0.0, sum(abs(v - lb) for _, v in pts) best = None for j in range(1, len(pts)): v = pts[j - 1][1] before += abs(v - la) after -= abs(v - lb) if best is None or before + after < best[0]: best = (before + after, j) return best[1] def main(): a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) a.add_argument("folder") a.add_argument("--columns", default="corr_x,corr_y,error_value") a.add_argument("--night", default="20-3", type=hours, help="night hours H1-H2, UTC (default 20-3)") a.add_argument("--k", type=float, default=5, help="deviation from the night level counted, in own noises of the sensor") a.add_argument("--days", type=int, default=2, help="number of days from which a sensor is cited in a slot") a.add_argument("--step", default="15min", type=duration, help="width of the slots of the hour of the day (default 15min)") a.add_argument("--sensors", help="comma-separated list (default: all)") a.add_argument("--bounds", action="store_true", help="adds, day by day, the bounds of the deviation from the level of the previous night") a.add_argument("--smoothing", type=int, default=1, metavar="N", help="--bounds: deviation smoothed by the median of N centred measurements (default 1: none)") a.add_argument("--fraction", type=float, metavar="F", help="--bounds: threshold of each day = F times its largest |deviation| (instead of --k noises)") a.add_argument("--start", type=parse_time) a.add_argument("--end", type=parse_time) a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs") o = a.parse_args() by_sensor, names = sensors(o.folder, o.sensors) columns = o.columns.split(",") check("sensors", names, by_sensor) check("columns", columns, COLUMNS) series = {s: [m for m in filter_window(by_sensor[s], o.start, o.end) if not rejected(m)] for s in names} if not any(series.values()): sys.exit("No accepted measurement in the requested window.") h1, h2 = o.night step = int(o.step.total_seconds() // 60) station = os.path.basename(os.path.normpath(o.folder)) os.makedirs(o.out, exist_ok=True) nrows = math.ceil(len(names) / 5) for col in columns: print(f"\n== {col}: level of each night ({h1} h → {h2} h UTC), median ±MAD (measurements); steps from one night to the next ==") slots = {} # minute of the day -> {(sensor, above): {day: deviation}} noises = {} fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False) for ax, s in zip(axes.flat, names): ms = [(m["t"], m[col]) for m in series[s] if m[col] is not None] if len(ms) < 2: print(f"{s:9}(fewer than 2 accepted values)") ax.axis("off") continue noise = noises[s] = own_noise([v for _, v in ms]) nights = nights_of(ms, h1, h2) lvl = {d: statistics.median(v for _, v in p) for d, p in nights.items()} cells = [] for d in sorted(nights): vs = [v for _, v in nights[d]] cells.append(f"{d:%d/%m} {lvl[d]:+.6g} ±{statistics.median(abs(v - lvl[d]) for v in vs):.2g} ({len(vs)})") print(f"{s:9}noise {noise:.3g} | " + (" | ".join(cells) if cells else "no night measurement")) night_days = sorted(nights) for da, db in zip(night_days, night_days[1:]): pts = nights[da] + nights[db] j = cut_index(pts, lvl[da], lvl[db]) d = lvl[db] - lvl[da] in_noises = f"{d / noise:+.1f}" if noise else "-" print(f"{'':9}{da:%d/%m} → {db:%d/%m}: {d:+.4g} ({in_noises} noises), between {iso(pts[j - 1][0])} and {iso(pts[j][0])}") def level_at(t): d = (t - timedelta(hours=h1)).date() if d in lvl: return lvl[d] return lvl[min(lvl, key=lambda x: abs((x - d).days))] if lvl else statistics.median(v for _, v in ms) per_day = {} for t, v in ms: e = v - level_at(t) per_day.setdefault(t.date(), []).append((t.hour + t.minute / 60, e)) if noise and abs(e) > o.k * noise: c = (t.hour * 60 + t.minute) // step * step slots.setdefault(c, {}).setdefault((s, e > 0), {})[t.date()] = e for i, (d, p) in enumerate(sorted(per_day.items())): ax.plot([x for x, _ in p], [e for _, e in p], "-", lw=0.8, color=f"C{i % 10}", label=f"{d:%d/%m}") ax.axhline(0, color="0.3", ls=":", lw=0.8) out_of_frame = frame(ax, (e for p in per_day.values() for _, e in p)) for x0, x1 in ((h1, 24), (0, h2)) if h1 > h2 else ((h1, h2),): ax.axvspan(x0, x1, color="0.9", lw=0) ax.set_title(s + (" (ref.)" if is_reference(series[s]) else "") + f" noise {noise:.2g}" + (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9) ax.set_xlim(0, 24) ax.set_xticks(range(0, 25, 3)) ax.grid(alpha=0.3) ax.tick_params(labelsize=7) if ax is axes.flat[0]: ax.legend(fontsize=6, ncol=2) for ax in list(axes.flat)[len(names):]: ax.axis("off") fig.suptitle(f"{station}: {col} minus the level of the last night ({h1} h → {h2} h, shaded), " "one curve per day, on the x axis the hour of the day (UTC); one scale per sensor") fig.tight_layout() path = os.path.join(o.out, f"{station}_byhour-{col}_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png") fig.savefig(path, dpi=100, bbox_inches="tight") plt.close(fig) print(path) n_days = {s: {} for s in names} # sensor -> slot -> measured days for s in names: for m in series[s]: if m[col] is not None: n_days[s].setdefault((m["t"].hour * 60 + m["t"].minute) // step * step, set()).add(m["t"].date()) print(f"\n{col}: per slot of {step} min of the hour of the day, sensors more than {o.k:g} noises from the night level " f"on at least {o.days} days (days / measured days, median of these deviations, in noises)") empty = True for c in sorted(slots): cited = [] for s, high in ((s, h) for s in names for h in (True, False)): ec = slots[c].get((s, high), {}) if len(ec) >= o.days: med = statistics.median(ec.values()) cited.append(f"{s} {len(ec)}/{len(n_days[s].get(c, ()))} {med:+.3g} ({med / noises[s]:+.1f})") if cited: empty = False print(f"{c // 60:02d}:{c % 60:02d} {len(cited):2} sensors: " + ", ".join(cited)) if empty: print("(none)") if o.bounds: day_bounds(o, names, series, columns, station) def day_bounds(o, names, series, columns, station): """Section 3: per day, the deviation from the level of the night that precedes it, for the target median and the pose.""" h1, h2 = o.night night_length = timedelta(hours=(h2 - h1) % 24) chosen = names if o.sensors else [s for s in names if series[s] and not is_reference(series[s])] or names seqs = [(f"{col} median of {len(chosen)} sensor(s)" if len(chosen) > 1 else f"{col} {chosen[0]}", col, {s: [(m["t"], m[col]) for m in series[s] if m[col] is not None] for s in chosen}) for col in columns] poses = filter_window(load_camera(o.folder), o.start, o.end) seqs += [(f"pose {c}", f"pose_{c}", {c: [(p["t"], p[c]) for p in poses if p[c] is not None]}) for c in POSE if any(p[c] is not None for p in poses)] every = [t for _, _, d in seqs for p in d.values() for t, _ in p] t0, t1 = min(every), max(every) night_of = lambda t: (t - night_length - timedelta(hours=h1)).date() # night that ends before t (the day of t starts at its end) days = [night_of(t0) + timedelta(days=i) for i in range((night_of(t1) - night_of(t0)).days + 1)] hm = lambda t: f"{t:%d/%m %H:%M}" # noqa: E731 span = lambda p, i, j: (f"from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) " # noqa: E731 f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})") exceed = "threshold" if o.fraction else f"{o.k:g} noises" threshold_txt = f"threshold {o.fraction:g} times the largest |deviation| of the day" if o.fraction else f"threshold {o.k:g} noises" print(f"\n== Bounds of each day: deviation from the level of the night that precedes it ({h1} h → {h2} h UTC), {threshold_txt}" + (f", deviation smoothed over {o.smoothing} measurements" if o.smoothing > 1 else "") + " ==") print("day: from the end of one night to the end of the next, dated by its start; \"before\", \"after\": measurement just before the first " "exceedance, just after the last; next night: its level minus that of the reference night") fig, axes = plt.subplots(len(seqs), 1, sharex=True, figsize=(18, 2.2 * len(seqs)), squeeze=False) for ax, (name, short, per_sensor) in zip(axes.flat, seqs): lvl = {s: {d: statistics.median(v for _, v in n) for d, n in nights_of(p, h1, h2).items()} for s, p in per_sensor.items()} nights = sorted({d for n in lvl.values() for d in n}) devs = [] # (day, ref. night, [(t, deviation)], deviation of the next night) for d in days: start = datetime.combine(d, datetime.min.time(), tzinfo=timezone.utc) + timedelta(hours=h1) + night_length end = start + timedelta(days=1) ref = max((n for n in nights if n <= d), default=None) or (min(nights) if nights else None) per_t = {} for s, p in per_sensor.items(): if ref in lvl[s]: for t, v in p: if start <= t < end: per_t.setdefault(t, []).append(v - lvl[s][ref]) next_n = [lvl[s][d + timedelta(days=1)] - lvl[s][ref] for s in lvl if ref in lvl[s] and d + timedelta(days=1) in lvl[s]] devs.append((start.date(), d, ref, [(t, statistics.median(e)) for t, e in sorted(per_t.items())], statistics.median(next_n) if next_n else None)) noise = own_noise(*[[e for _, e in p] for *_, p, _ in devs]) print(f"\n{name}: noise {noise:.3g}") in_noises = lambda e: f"{e:+.4g} ({e / noise:+.1f})" if noise else f"{e:+.4g}" h = o.smoothing // 2 for day, d, ref, raw, nxt in devs: if not raw: print(f"{short} {day:%d/%m} no measurement" + ("" if nights else " at night")) continue p = [(t, statistics.median(e for _, e in raw[max(0, i - h):i + h + 1])) for i, (t, _) in enumerate(raw)] if h else raw threshold = o.fraction * max(abs(e) for _, e in p) if o.fraction else o.k * noise line = (f"{short} {day:%d/%m} night {ref:%d/%m}" + (" (not the previous one)" if ref != d else "") + f" | {len(p)} measurements {hm(p[0][0])} → {hm(p[-1][0])} | " + (f"threshold {threshold:.4g} | " if o.fraction else "")) over = [i for i, (_, e) in enumerate(p) if threshold and abs(e) > threshold] if over: i, j = over[0], over[-1] line += f"> {exceed}: {len(over)} measurements, {span(p, i, j)}" ax.axvspan(p[i][0], p[j][0], color="0.85", lw=0) else: line += f"never > {exceed}" top, bottom = max(p, key=lambda x: x[1]), min(p, key=lambda x: x[1]) line += f" | max {in_noises(top[1])} at {hm(top[0])}, min {in_noises(bottom[1])} at {hm(bottom[0])}" x = max(range(len(p)), key=lambda i: abs(p[i][1])) sign = 1 if p[x][1] > 0 else -1 if threshold and sign * p[x][1] > threshold: i = j = x while i > 0 and sign * p[i - 1][1] > threshold: i -= 1 while j + 1 < len(p) and sign * p[j + 1][1] > threshold: j += 1 line += f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, {span(p, i, j)}" ax.axvspan(p[i][0], p[j][0], color="0.7", lw=0) line += f" | next night {in_noises(nxt)}" if nxt is not None else " | next night not measured" print(line) if h: ax.plot([t for t, _ in raw], [e for _, e in raw], "-", lw=0.5, color="0.6") ax.plot([t for t, _ in p], [e for _, e in p], "-", lw=0.8, color="C0") ax.plot([top[0], bottom[0]], [top[1], bottom[1]], "o", ms=3, color="C3") if noise and not o.fraction: for y in (o.k * noise, -o.k * noise): ax.axhline(y, color="0.4", ls=":", lw=0.8) ax.axhline(0, color="0.3", lw=0.5) out_of_frame = frame(ax, (e for *_, p, _ in devs for _, e in p)) ax.set_title(f"{name} noise {noise:.2g}" + (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9) ax.grid(alpha=0.3) ax.tick_params(labelsize=7) fig.suptitle(f"{station}: deviation from the level of the night that precedes each day ({h1} h → {h2} h); " f"shaded light: from the first to the last exceedance {'of the threshold (' + threshold_txt + ')' if o.fraction else f'of {o.k:g} noises'}, dark: the run around the extreme; points: max and min of the day" + (f"; deviation smoothed over {o.smoothing} measurements (blue), raw (grey)" if o.smoothing > 1 else "")) fig.tight_layout() path = os.path.join(o.out, f"{station}_bounds_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png") fig.savefig(path, dpi=100, bbox_inches="tight") plt.close(fig) print(path) if __name__ == "__main__": main()