Scripts: shared helpers in common.py, no tool imports another; CLI and outputs unchanged
data.py stays the reader (METRIC_RE now also captures the file date, same matches); common.py holds columns, steps, noise, sensor list, runs, file names. Duplicated run grouping, pose gaps, step bucketing, --table/--outside parsing removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c1719cbd1d
commit
6ed4c420e1
13 files changed
+308
-359
No files matched your search
+38
-46
@@ -55,9 +55,8 @@ import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402
|
||||
from plot import COLUMNS, bound # noqa: E402
|
||||
from summary import ROUNDING, duration, own_noise # noqa: E402
|
||||
from common import COLUMNS, bound, check, duration, own_noise, sensors # noqa: E402
|
||||
from data import POSE, filter_window, is_reference, iso, load_camera, parse_time, rejected # noqa: E402
|
||||
|
||||
|
||||
def hours(text):
|
||||
@@ -73,6 +72,27 @@ def at_night(t, h1, h2):
|
||||
return (h1 <= t.hour or t.hour < h2) if h1 > h2 else h1 <= t.hour < h2
|
||||
|
||||
|
||||
def nights_of(pts, h1, h2):
|
||||
"""[(t, v)] -> {date of the night: [(t, v)]}: the night hours, dated by the day the night starts."""
|
||||
nights = {}
|
||||
for t, v in pts:
|
||||
if at_night(t, h1, h2):
|
||||
nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
|
||||
return nights
|
||||
|
||||
|
||||
def frame(ax, values):
|
||||
"""Frame from the 1st to the 99th percentile widened by half: an isolated peak does not crush the scale; -> the number
|
||||
of values out of frame."""
|
||||
v = sorted(values)
|
||||
if not v:
|
||||
return 0
|
||||
q1, q2 = v[len(v) // 100], v[-1 - len(v) // 100]
|
||||
margin = (q2 - q1) * 0.5 or 1
|
||||
ax.set_ylim(q1 - margin, q2 + margin)
|
||||
return sum(not q1 - margin <= e <= q2 + margin for e in v)
|
||||
|
||||
|
||||
def cut_index(pts, la, lb):
|
||||
"""Index j (1..n-1) where sum|v-la| before j + sum|v-lb| from j on is smallest."""
|
||||
before, after = 0.0, sum(abs(v - lb) for _, v in pts)
|
||||
@@ -103,15 +123,10 @@ def main():
|
||||
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
|
||||
o = a.parse_args()
|
||||
|
||||
by_sensor = load(o.folder)
|
||||
if not by_sensor:
|
||||
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
|
||||
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
|
||||
by_sensor, names = sensors(o.folder, o.sensors)
|
||||
columns = o.columns.split(",")
|
||||
for requested, known, what in ((names, by_sensor, "sensors"), (columns, COLUMNS, "columns")):
|
||||
missing = [x for x in requested if x not in known]
|
||||
if missing:
|
||||
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
|
||||
check("sensors", names, by_sensor)
|
||||
check("columns", columns, COLUMNS)
|
||||
series = {s: [m for m in filter_window(by_sensor[s], o.start, o.end) if not rejected(m)] for s in names}
|
||||
if not any(series.values()):
|
||||
sys.exit("No accepted measurement in the requested window.")
|
||||
@@ -133,10 +148,7 @@ def main():
|
||||
ax.axis("off")
|
||||
continue
|
||||
noise = noises[s] = own_noise([v for _, v in ms])
|
||||
nights = {}
|
||||
for t, v in ms:
|
||||
if at_night(t, h1, h2):
|
||||
nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
|
||||
nights = nights_of(ms, h1, h2)
|
||||
lvl = {d: statistics.median(v for _, v in p) for d, p in nights.items()}
|
||||
cells = []
|
||||
for d in sorted(nights):
|
||||
@@ -167,11 +179,7 @@ def main():
|
||||
for i, (d, p) in enumerate(sorted(per_day.items())):
|
||||
ax.plot([x for x, _ in p], [e for _, e in p], "-", lw=0.8, color=f"C{i % 10}", label=f"{d:%d/%m}")
|
||||
ax.axhline(0, color="0.3", ls=":", lw=0.8)
|
||||
every = sorted(e for p in per_day.values() for _, e in p)
|
||||
q1, q2 = every[len(every) // 100], every[-1 - len(every) // 100] # frame from the 1st to the 99th percentile: an isolated peak does not crush the scale
|
||||
margin = (q2 - q1) * 0.5 or 1
|
||||
ax.set_ylim(q1 - margin, q2 + margin)
|
||||
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in every)
|
||||
out_of_frame = frame(ax, (e for p in per_day.values() for _, e in p))
|
||||
for x0, x1 in ((h1, 24), (0, h2)) if h1 > h2 else ((h1, h2),):
|
||||
ax.axvspan(x0, x1, color="0.9", lw=0)
|
||||
ax.set_title(s + (" (ref.)" if is_reference(series[s]) else "") + f" noise {noise:.2g}"
|
||||
@@ -217,15 +225,6 @@ def main():
|
||||
day_bounds(o, names, series, columns, station)
|
||||
|
||||
|
||||
def night_levels(pts, h1, h2):
|
||||
"""[(t, v)] -> {date of the night: median}."""
|
||||
nights = {}
|
||||
for t, v in pts:
|
||||
if at_night(t, h1, h2):
|
||||
nights.setdefault((t - timedelta(hours=h1)).date(), []).append(v)
|
||||
return {d: statistics.median(v) for d, v in nights.items()}
|
||||
|
||||
|
||||
def day_bounds(o, names, series, columns, station):
|
||||
"""Section 3: per day, the deviation from the level of the night that precedes it, for the target median and the pose."""
|
||||
h1, h2 = o.night
|
||||
@@ -240,7 +239,10 @@ def day_bounds(o, names, series, columns, station):
|
||||
t0, t1 = min(every), max(every)
|
||||
night_of = lambda t: (t - night_length - timedelta(hours=h1)).date() # night that ends before t (the day of t starts at its end)
|
||||
days = [night_of(t0) + timedelta(days=i) for i in range((night_of(t1) - night_of(t0)).days + 1)]
|
||||
hm = lambda t: f"{t:%d/%m %H:%M}"
|
||||
hm = lambda t: f"{t:%d/%m %H:%M}" # noqa: E731
|
||||
span = lambda p, i, j: (f"from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) " # noqa: E731
|
||||
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
|
||||
exceed = "threshold" if o.fraction else f"{o.k:g} noises"
|
||||
threshold_txt = f"threshold {o.fraction:g} times the largest |deviation| of the day" if o.fraction else f"threshold {o.k:g} noises"
|
||||
print(f"\n== Bounds of each day: deviation from the level of the night that precedes it ({h1} h → {h2} h UTC), {threshold_txt}"
|
||||
+ (f", deviation smoothed over {o.smoothing} measurements" if o.smoothing > 1 else "") + " ==")
|
||||
@@ -248,7 +250,7 @@ def day_bounds(o, names, series, columns, station):
|
||||
"exceedance, just after the last; next night: its level minus that of the reference night")
|
||||
fig, axes = plt.subplots(len(seqs), 1, sharex=True, figsize=(18, 2.2 * len(seqs)), squeeze=False)
|
||||
for ax, (name, short, per_sensor) in zip(axes.flat, seqs):
|
||||
lvl = {s: night_levels(p, h1, h2) for s, p in per_sensor.items()}
|
||||
lvl = {s: {d: statistics.median(v for _, v in n) for d, n in nights_of(p, h1, h2).items()} for s, p in per_sensor.items()}
|
||||
nights = sorted({d for n in lvl.values() for d in n})
|
||||
devs = [] # (day, ref. night, [(t, deviation)], deviation of the next night)
|
||||
for d in days:
|
||||
@@ -264,9 +266,7 @@ def day_bounds(o, names, series, columns, station):
|
||||
next_n = [lvl[s][d + timedelta(days=1)] - lvl[s][ref] for s in lvl if ref in lvl[s] and d + timedelta(days=1) in lvl[s]]
|
||||
devs.append((start.date(), d, ref, [(t, statistics.median(e)) for t, e in sorted(per_t.items())],
|
||||
statistics.median(next_n) if next_n else None))
|
||||
diffs = [abs(b[1] - a_[1]) for *_, p, _ in devs for a_, b in zip(p, p[1:])]
|
||||
noise = statistics.median(diffs) if diffs else 0
|
||||
noise = noise if noise >= ROUNDING else 0
|
||||
noise = own_noise(*[[e for _, e in p] for *_, p, _ in devs])
|
||||
print(f"\n{name}: noise {noise:.3g}")
|
||||
in_noises = lambda e: f"{e:+.4g} ({e / noise:+.1f})" if noise else f"{e:+.4g}"
|
||||
h = o.smoothing // 2
|
||||
@@ -281,11 +281,10 @@ def day_bounds(o, names, series, columns, station):
|
||||
over = [i for i, (_, e) in enumerate(p) if threshold and abs(e) > threshold]
|
||||
if over:
|
||||
i, j = over[0], over[-1]
|
||||
line += (f"> {'threshold' if o.fraction else f'{o.k:g} noises'}: {len(over)} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) "
|
||||
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
|
||||
line += f"> {exceed}: {len(over)} measurements, {span(p, i, j)}"
|
||||
ax.axvspan(p[i][0], p[j][0], color="0.85", lw=0)
|
||||
else:
|
||||
line += f"never > {'threshold' if o.fraction else f'{o.k:g} noises'}"
|
||||
line += f"never > {exceed}"
|
||||
top, bottom = max(p, key=lambda x: x[1]), min(p, key=lambda x: x[1])
|
||||
line += f" | max {in_noises(top[1])} at {hm(top[0])}, min {in_noises(bottom[1])} at {hm(bottom[0])}"
|
||||
x = max(range(len(p)), key=lambda i: abs(p[i][1]))
|
||||
@@ -296,8 +295,7 @@ def day_bounds(o, names, series, columns, station):
|
||||
i -= 1
|
||||
while j + 1 < len(p) and sign * p[j + 1][1] > threshold:
|
||||
j += 1
|
||||
line += (f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) "
|
||||
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
|
||||
line += f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, {span(p, i, j)}"
|
||||
ax.axvspan(p[i][0], p[j][0], color="0.7", lw=0)
|
||||
line += f" | next night {in_noises(nxt)}" if nxt is not None else " | next night not measured"
|
||||
print(line)
|
||||
@@ -309,13 +307,7 @@ def day_bounds(o, names, series, columns, station):
|
||||
for y in (o.k * noise, -o.k * noise):
|
||||
ax.axhline(y, color="0.4", ls=":", lw=0.8)
|
||||
ax.axhline(0, color="0.3", lw=0.5)
|
||||
all_e = sorted(e for *_, p, _ in devs for _, e in p)
|
||||
out_of_frame = 0
|
||||
if all_e: # frame from the 1st to the 99th percentile widened by half, as in byhour: an isolated peak does not crush the scale
|
||||
q1, q2 = all_e[len(all_e) // 100], all_e[-1 - len(all_e) // 100]
|
||||
margin = (q2 - q1) * 0.5 or 1
|
||||
ax.set_ylim(q1 - margin, q2 + margin)
|
||||
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in all_e)
|
||||
out_of_frame = frame(ax, (e for *_, p, _ in devs for _, e in p))
|
||||
ax.set_title(f"{name} noise {noise:.2g}" + (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9)
|
||||
ax.grid(alpha=0.3)
|
||||
ax.tick_params(labelsize=7)
|
||||
|
||||
Reference in new issue
Block a user