Scripts: shared helpers in common.py, no tool imports another; CLI and outputs unchanged

data.py stays the reader (METRIC_RE now also captures the file date, same matches);
common.py holds columns, steps, noise, sensor list, runs, file names. Duplicated
run grouping, pose gaps, step bucketing, --table/--outside parsing removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 15:48:42 +01:00
1 parent c1719cbd1d
commit 6ed4c420e1
13 files changed
+308 -359

No files matched your search

+38 -46
View File
@@ -55,9 +55,8 @@ import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402
from plot import COLUMNS, bound # noqa: E402
from summary import ROUNDING, duration, own_noise # noqa: E402
from common import COLUMNS, bound, check, duration, own_noise, sensors # noqa: E402
from data import POSE, filter_window, is_reference, iso, load_camera, parse_time, rejected # noqa: E402
def hours(text):
@@ -73,6 +72,27 @@ def at_night(t, h1, h2):
return (h1 <= t.hour or t.hour < h2) if h1 > h2 else h1 <= t.hour < h2
def nights_of(pts, h1, h2):
"""[(t, v)] -> {date of the night: [(t, v)]}: the night hours, dated by the day the night starts."""
nights = {}
for t, v in pts:
if at_night(t, h1, h2):
nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
return nights
def frame(ax, values):
"""Frame from the 1st to the 99th percentile widened by half: an isolated peak does not crush the scale; -> the number
of values out of frame."""
v = sorted(values)
if not v:
return 0
q1, q2 = v[len(v) // 100], v[-1 - len(v) // 100]
margin = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - margin, q2 + margin)
return sum(not q1 - margin <= e <= q2 + margin for e in v)
def cut_index(pts, la, lb):
"""Index j (1..n-1) where sum|v-la| before j + sum|v-lb| from j on is smallest."""
before, after = 0.0, sum(abs(v - lb) for _, v in pts)
@@ -103,15 +123,10 @@ def main():
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
o = a.parse_args()
by_sensor = load(o.folder)
if not by_sensor:
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
by_sensor, names = sensors(o.folder, o.sensors)
columns = o.columns.split(",")
for requested, known, what in ((names, by_sensor, "sensors"), (columns, COLUMNS, "columns")):
missing = [x for x in requested if x not in known]
if missing:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
check("sensors", names, by_sensor)
check("columns", columns, COLUMNS)
series = {s: [m for m in filter_window(by_sensor[s], o.start, o.end) if not rejected(m)] for s in names}
if not any(series.values()):
sys.exit("No accepted measurement in the requested window.")
@@ -133,10 +148,7 @@ def main():
ax.axis("off")
continue
noise = noises[s] = own_noise([v for _, v in ms])
nights = {}
for t, v in ms:
if at_night(t, h1, h2):
nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
nights = nights_of(ms, h1, h2)
lvl = {d: statistics.median(v for _, v in p) for d, p in nights.items()}
cells = []
for d in sorted(nights):
@@ -167,11 +179,7 @@ def main():
for i, (d, p) in enumerate(sorted(per_day.items())):
ax.plot([x for x, _ in p], [e for _, e in p], "-", lw=0.8, color=f"C{i % 10}", label=f"{d:%d/%m}")
ax.axhline(0, color="0.3", ls=":", lw=0.8)
every = sorted(e for p in per_day.values() for _, e in p)
q1, q2 = every[len(every) // 100], every[-1 - len(every) // 100] # frame from the 1st to the 99th percentile: an isolated peak does not crush the scale
margin = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - margin, q2 + margin)
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in every)
out_of_frame = frame(ax, (e for p in per_day.values() for _, e in p))
for x0, x1 in ((h1, 24), (0, h2)) if h1 > h2 else ((h1, h2),):
ax.axvspan(x0, x1, color="0.9", lw=0)
ax.set_title(s + (" (ref.)" if is_reference(series[s]) else "") + f" noise {noise:.2g}"
@@ -217,15 +225,6 @@ def main():
day_bounds(o, names, series, columns, station)
def night_levels(pts, h1, h2):
"""[(t, v)] -> {date of the night: median}."""
nights = {}
for t, v in pts:
if at_night(t, h1, h2):
nights.setdefault((t - timedelta(hours=h1)).date(), []).append(v)
return {d: statistics.median(v) for d, v in nights.items()}
def day_bounds(o, names, series, columns, station):
"""Section 3: per day, the deviation from the level of the night that precedes it, for the target median and the pose."""
h1, h2 = o.night
@@ -240,7 +239,10 @@ def day_bounds(o, names, series, columns, station):
t0, t1 = min(every), max(every)
night_of = lambda t: (t - night_length - timedelta(hours=h1)).date() # night that ends before t (the day of t starts at its end)
days = [night_of(t0) + timedelta(days=i) for i in range((night_of(t1) - night_of(t0)).days + 1)]
hm = lambda t: f"{t:%d/%m %H:%M}"
hm = lambda t: f"{t:%d/%m %H:%M}" # noqa: E731
span = lambda p, i, j: (f"from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) " # noqa: E731
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
exceed = "threshold" if o.fraction else f"{o.k:g} noises"
threshold_txt = f"threshold {o.fraction:g} times the largest |deviation| of the day" if o.fraction else f"threshold {o.k:g} noises"
print(f"\n== Bounds of each day: deviation from the level of the night that precedes it ({h1} h → {h2} h UTC), {threshold_txt}"
+ (f", deviation smoothed over {o.smoothing} measurements" if o.smoothing > 1 else "") + " ==")
@@ -248,7 +250,7 @@ def day_bounds(o, names, series, columns, station):
"exceedance, just after the last; next night: its level minus that of the reference night")
fig, axes = plt.subplots(len(seqs), 1, sharex=True, figsize=(18, 2.2 * len(seqs)), squeeze=False)
for ax, (name, short, per_sensor) in zip(axes.flat, seqs):
lvl = {s: night_levels(p, h1, h2) for s, p in per_sensor.items()}
lvl = {s: {d: statistics.median(v for _, v in n) for d, n in nights_of(p, h1, h2).items()} for s, p in per_sensor.items()}
nights = sorted({d for n in lvl.values() for d in n})
devs = [] # (day, ref. night, [(t, deviation)], deviation of the next night)
for d in days:
@@ -264,9 +266,7 @@ def day_bounds(o, names, series, columns, station):
next_n = [lvl[s][d + timedelta(days=1)] - lvl[s][ref] for s in lvl if ref in lvl[s] and d + timedelta(days=1) in lvl[s]]
devs.append((start.date(), d, ref, [(t, statistics.median(e)) for t, e in sorted(per_t.items())],
statistics.median(next_n) if next_n else None))
diffs = [abs(b[1] - a_[1]) for *_, p, _ in devs for a_, b in zip(p, p[1:])]
noise = statistics.median(diffs) if diffs else 0
noise = noise if noise >= ROUNDING else 0
noise = own_noise(*[[e for _, e in p] for *_, p, _ in devs])
print(f"\n{name}: noise {noise:.3g}")
in_noises = lambda e: f"{e:+.4g} ({e / noise:+.1f})" if noise else f"{e:+.4g}"
h = o.smoothing // 2
@@ -281,11 +281,10 @@ def day_bounds(o, names, series, columns, station):
over = [i for i, (_, e) in enumerate(p) if threshold and abs(e) > threshold]
if over:
i, j = over[0], over[-1]
line += (f"> {'threshold' if o.fraction else f'{o.k:g} noises'}: {len(over)} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) "
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
line += f"> {exceed}: {len(over)} measurements, {span(p, i, j)}"
ax.axvspan(p[i][0], p[j][0], color="0.85", lw=0)
else:
line += f"never > {'threshold' if o.fraction else f'{o.k:g} noises'}"
line += f"never > {exceed}"
top, bottom = max(p, key=lambda x: x[1]), min(p, key=lambda x: x[1])
line += f" | max {in_noises(top[1])} at {hm(top[0])}, min {in_noises(bottom[1])} at {hm(bottom[0])}"
x = max(range(len(p)), key=lambda i: abs(p[i][1]))
@@ -296,8 +295,7 @@ def day_bounds(o, names, series, columns, station):
i -= 1
while j + 1 < len(p) and sign * p[j + 1][1] > threshold:
j += 1
line += (f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) "
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
line += f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, {span(p, i, j)}"
ax.axvspan(p[i][0], p[j][0], color="0.7", lw=0)
line += f" | next night {in_noises(nxt)}" if nxt is not None else " | next night not measured"
print(line)
@@ -309,13 +307,7 @@ def day_bounds(o, names, series, columns, station):
for y in (o.k * noise, -o.k * noise):
ax.axhline(y, color="0.4", ls=":", lw=0.8)
ax.axhline(0, color="0.3", lw=0.5)
all_e = sorted(e for *_, p, _ in devs for _, e in p)
out_of_frame = 0
if all_e: # frame from the 1st to the 99th percentile widened by half, as in byhour: an isolated peak does not crush the scale
q1, q2 = all_e[len(all_e) // 100], all_e[-1 - len(all_e) // 100]
margin = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - margin, q2 + margin)
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in all_e)
out_of_frame = frame(ax, (e for *_, p, _ in devs for _, e in p))
ax.set_title(f"{name} noise {noise:.2g}" + (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9)
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)