English: tools renamed and translated (same numbers as before), options and printed names in English; the guide translated

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 12:40:03 +01:00
1 parent 67c88625c5
commit f28bd19e04
25 files changed
+2764 -2762

No files matched your search

+163
View File
@@ -0,0 +1,163 @@
"""Reading a station's data for the diagnostic agent (standard library).
Formats: labeling/knowledge/displacement-columns.md. Every time is UTC.
"""
import csv
import glob
import os
import re
import statistics
from datetime import datetime, timezone
# Numeric columns of the "advanced" export; an empty cell becomes None.
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
# Camera pose (camera/, pumping): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
IMAGE_RE = re.compile(r"^(\d{8}T\d{4})Z_")
def _header(path):
with open(path, newline="") as f:
return next(csv.reader(f), [])
def find_files(folder):
"""Station folder -> {'advanced', 'hourly', 'camera', 'images', 'others'}: sorted paths, subfolders included."""
out = {"advanced": [], "hourly": [], "camera": [], "images": [], "others": []}
for p in sorted(glob.glob(os.path.join(folder, "**", "*"), recursive=True)):
if os.path.isdir(p) or "images_diff" in p.split(os.sep): # differential images of the pumping (--diff): not camera views
continue
header = _header(p) if p.endswith(".csv") else []
if p.lower().endswith((".jpg", ".jpeg", ".png")):
out["images"].append(p)
elif "sensor_name" in header and "corr_x" in header: # quality/ (optical_emulator) has sensor_name, not corr_x
out["advanced"].append(p)
elif "timestamp" in header:
out["hourly"].append(p)
elif "median_pan" in header:
out["camera"].append(p)
else:
out["others"].append(p)
return out
def _number(v):
v = v.strip()
return float(v) if v else None
def load(folder):
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None).
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
for a target but leaves some filled (often absurd) for a reference; they are emptied here.
"""
by = {}
for path in find_files(folder)["advanced"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
m = {c: _number(r.get(c) or "") for c in NUM} # inlier is missing from old exports
m["t"] = parse_time(r["time_utc"])
m["pattern_type"] = r["pattern_type"]
if rejected(m):
m["corr_x"] = m["corr_y"] = None
by.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # some exports end the name with a newline
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
def load_temperature(folder):
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
hour; [] without these metrics. The components rise and fall together (the labeler reads "the temperatures of all
the components"); the median drops the spike of a single one, the computing GPU (D-124)."""
by = {}
for path in glob.glob(os.path.join(folder, "metrics", "*_deg_c_*.csv")):
with open(path, newline="") as f:
for r in csv.DictReader(f):
if (v := _number(r.get("metric") or "")) is not None:
by.setdefault(parse_time(r["time_utc"]), []).append(v)
return [(t, statistics.median(v)) for t, v in sorted(by.items())]
def load_camera(folder):
"""Camera movements of the folder -> [pose] sorted by t; pose: t and the POSE columns (float or None).
One row per camera registration; the medians are read (min = max in the files received: one reading per minute).
A measurement instant without a pose: the camera was not registered.
"""
by = {}
for path in find_files(folder)["camera"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
p = {c: _number(r.get("median_" + c, "")) for c in POSE}
p["t"] = parse_time(r["time_utc"])
by[p["t"]] = p
return [by[t] for t in sorted(by)]
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
METRIC_RE = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
def load_metrics(folder):
"""Station metrics (metrics/, one CSV per metric and per day) -> {name: [(t, value)]} sorted: numeric values only
(BusVoltage, sensor_CPU_deg_c, rsrp_dBm…); text metrics (link, mode…) and identifiers (SIM_CCID, ip*) are not
read. Name: the file's, without the station or the date."""
# ponytail: text metrics are skipped; read them (link up/down) when a detector needs them
station, by = os.path.basename(os.path.normpath(folder)) + "_", {}
for path in glob.glob(os.path.join(folder, "metrics", "*.csv")):
if not (m := METRIC_RE.match(os.path.basename(path))):
continue
name = m.group(1).removeprefix(station)
if IDENTIFIERS.match(name):
continue
with open(path, newline="") as f:
for r in csv.DictReader(f):
try:
v = float((r.get("metric") or "").strip("[]'\" ")) # some metrics come as a list: ['-95']
except ValueError:
continue
by.setdefault(name, {})[parse_time(r["time_utc"])] = v
return {n: sorted(d.items()) for n, d in sorted(by.items())}
def is_reference(measurements):
"""Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty
pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is
none, filled reproj_err or inlier (references only: labeling/knowledge/displacement-columns.md)."""
t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None)
if t is not None:
return t == "global_ref"
return any(m["reproj_err"] is not None or m["inlier"] is not None for m in measurements)
def rejected(m):
"""Measurement rejected by the station: error_value above the sensor's threshold (error_threshold); without these
two values, an empty corr_x."""
if m["error_value"] is not None and m["error_threshold"] is not None:
return m["error_value"] > m["error_threshold"]
return m["corr_x"] is None
def filter_window(measurements, start, end):
"""Measurements between start and end inclusive, to the minute (as iso() shows them); None = no bound."""
return [m for m in measurements if (start is None or m["t"] >= start)
and (end is None or m["t"].replace(second=0, microsecond=0) <= end)]
def image_time(path):
"""'20261013T1200Z_10167931.jpg' -> UTC datetime; None if the name does not follow this format (D-045)."""
m = IMAGE_RE.match(os.path.basename(path))
return datetime.strptime(m.group(1), "%Y%m%dT%H%M").replace(tzinfo=timezone.utc) if m else None
def parse_time(text):
"""'2026-01-01T00:00Z' or '2026-01-01 00:00' -> UTC datetime; a time without a zone is read as UTC."""
t = datetime.fromisoformat(text)
return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc)
def iso(t):
return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")
+333
View File
@@ -0,0 +1,333 @@
"""What comes back from one day to the next: level of each night, and days overlaid by hour of the day, for the diagnostic agent.
Usage: .venv/bin/python tools/scripts/days.py data/PS_DES_COUDRES/ [--columns corr_x,corr_y,error_value]
[--night 20-3] [--k 5] [--days 2] [--step 15min] [--sensors A,B] [--bounds [--smoothing N] [--fraction F]]
[--start ...] [--end ...] [--out FOLDER]
Accepted measurements only: a rejected measurement (error_value above the threshold) enters nowhere. A "night":
the hours --night H1-H2 (UTC, H1 included, H2 excluded, straddling midnight if H1 > H2), dated by the day it starts;
the night in progress at the start of the window also counts (its hours after midnight). Own noise of the sensor: median of the
differences between two consecutive measurements over the window. For each column of --columns:
1. Level of each night, per sensor: median, MAD and number of measurements; then, from one night to the next, the
step (difference of the medians, in units and in noises) and where it falls: the run of night measurements of the
two nights is cut where the sum of the absolute deviations from the level of the first night (before) and of the second
(after) is smallest; "between A and B": last night measurement before the cut and first after (B at the
start of a night: the cut falls between two nights, in daytime).
2. Overlaid days: each measurement minus the level of the last night started at its time (the previous one, or
the one in progress; without it, the nearest night). PNG "byhour-<column>": one small plot per sensor,
on the x axis the hour of the day (UTC), one curve per day, night hours shaded, one scale per sensor, framed from the 1st to the 99th percentile widened by half (the title says
how many measurements fall outside the frame: an isolated peak does not crush the scale; its value is in the tables).
Table: per slot of --step of the hour of the day, the sensors whose deviation exceeds --k noises in that slot, on the
same side, on at least --days different days, with the number of these days over the number of measured days and the
median of these deviations (signed, in units and in noises); a sensor above on some days and below on
others is cited twice.
3. With --bounds: each day against the night that precedes it, for the target median of each column (with
--sensors: the median of these sensors, references included; a single sensor: its own) and for the pose of the
camera (pan, tilt, rotation, scale, sigma0) if camera/ exists. A "day": from the end of one night to the end of the
next (24 h), dated by the day it starts; its reference level is that of the night that has just ended (without
it, the last night measured before, or at the start of the batch the first one: named, "not the previous one"). Deviation of each measurement from this level (target median: at each
time, median of the deviations of each target from its own level of that night); noise: median of the differences between
two consecutive values of this series. Per day: number of measurements, first and last; first and last
time when |deviation| exceeds --k noises, with the measurement just before and just after (at the native step); the signed maximum and
minimum and their time; around the larger of the two in absolute value, the run of consecutive measurements
beyond --k noises on the same side that contains it, with the measurement just before and just after; the next night
minus the reference night. A day without a measurement says so. Each day line starts with the short name of the series
(the column, or pose_tilt…): `| grep '^pose_tilt '` keeps one, `| grep ' 12/10 '` one day. With --smoothing N,
the deviation of each measurement is the median of the N measurements of the day centred on it (the noise stays that of the measurements,
the figure also plots the raw deviation in grey): an isolated point no longer cuts the run around the extreme and no longer makes
a first or a last exceedance. With --fraction F, the threshold of each day is F times the largest
|deviation| of that day (instead of --k noises; it is printed on the day line): on a very noisy series, or
one whose noise at the native step bears no relation to the amplitude of the cycle. PNG
"bounds": one panel per series, the deviation over the batch, ±k noises dotted, each day's [first, last]
shaded light and the run around the extreme shaded dark, the maximum and the minimum marked; framed from the 1st to the 99th
percentile widened by half (the title says how many measurements fall outside the frame).
Gives values, not a verdict.
"""
import argparse
import math
import os
import statistics
import sys
from datetime import datetime, timedelta, timezone
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402
from plot import COLUMNS, bound # noqa: E402
from summary import ROUNDING, duration, own_noise # noqa: E402
def hours(text):
"""'20-3' -> (20, 3)."""
try:
h1, h2 = (int(x) for x in text.split("-"))
except ValueError:
raise argparse.ArgumentTypeError(f"invalid hours: {text!r} (e.g. 20-3)")
return h1, h2
def at_night(t, h1, h2):
return (h1 <= t.hour or t.hour < h2) if h1 > h2 else h1 <= t.hour < h2
def cut_index(pts, la, lb):
"""Index j (1..n-1) where sum|v-la| before j + sum|v-lb| from j on is smallest."""
before, after = 0.0, sum(abs(v - lb) for _, v in pts)
best = None
for j in range(1, len(pts)):
v = pts[j - 1][1]
before += abs(v - la)
after -= abs(v - lb)
if best is None or before + after < best[0]:
best = (before + after, j)
return best[1]
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder")
a.add_argument("--columns", default="corr_x,corr_y,error_value")
a.add_argument("--night", default="20-3", type=hours, help="night hours H1-H2, UTC (default 20-3)")
a.add_argument("--k", type=float, default=5, help="deviation from the night level counted, in own noises of the sensor")
a.add_argument("--days", type=int, default=2, help="number of days from which a sensor is cited in a slot")
a.add_argument("--step", default="15min", type=duration, help="width of the slots of the hour of the day (default 15min)")
a.add_argument("--sensors", help="comma-separated list (default: all)")
a.add_argument("--bounds", action="store_true", help="adds, day by day, the bounds of the deviation from the level of the previous night")
a.add_argument("--smoothing", type=int, default=1, metavar="N", help="--bounds: deviation smoothed by the median of N centred measurements (default 1: none)")
a.add_argument("--fraction", type=float, metavar="F", help="--bounds: threshold of each day = F times its largest |deviation| (instead of --k noises)")
a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time)
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
o = a.parse_args()
by_sensor = load(o.folder)
if not by_sensor:
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
columns = o.columns.split(",")
for requested, known, what in ((names, by_sensor, "sensors"), (columns, COLUMNS, "columns")):
missing = [x for x in requested if x not in known]
if missing:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
series = {s: [m for m in filter_window(by_sensor[s], o.start, o.end) if not rejected(m)] for s in names}
if not any(series.values()):
sys.exit("No accepted measurement in the requested window.")
h1, h2 = o.night
step = int(o.step.total_seconds() // 60)
station = os.path.basename(os.path.normpath(o.folder))
os.makedirs(o.out, exist_ok=True)
nrows = math.ceil(len(names) / 5)
for col in columns:
print(f"\n== {col}: level of each night ({h1} h → {h2} h UTC), median ±MAD (measurements); steps from one night to the next ==")
slots = {} # minute of the day -> {(sensor, above): {day: deviation}}
noises = {}
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
for ax, s in zip(axes.flat, names):
ms = [(m["t"], m[col]) for m in series[s] if m[col] is not None]
if len(ms) < 2:
print(f"{s:9}(fewer than 2 accepted values)")
ax.axis("off")
continue
noise = noises[s] = own_noise([v for _, v in ms])
nights = {}
for t, v in ms:
if at_night(t, h1, h2):
nights.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
lvl = {d: statistics.median(v for _, v in p) for d, p in nights.items()}
cells = []
for d in sorted(nights):
vs = [v for _, v in nights[d]]
cells.append(f"{d:%d/%m} {lvl[d]:+.6g} ±{statistics.median(abs(v - lvl[d]) for v in vs):.2g} ({len(vs)})")
print(f"{s:9}noise {noise:.3g} | " + (" | ".join(cells) if cells else "no night measurement"))
night_days = sorted(nights)
for da, db in zip(night_days, night_days[1:]):
pts = nights[da] + nights[db]
j = cut_index(pts, lvl[da], lvl[db])
d = lvl[db] - lvl[da]
in_noises = f"{d / noise:+.1f}" if noise else "-"
print(f"{'':9}{da:%d/%m} → {db:%d/%m}: {d:+.4g} ({in_noises} noises), between {iso(pts[j - 1][0])} and {iso(pts[j][0])}")
def level_at(t):
d = (t - timedelta(hours=h1)).date()
if d in lvl:
return lvl[d]
return lvl[min(lvl, key=lambda x: abs((x - d).days))] if lvl else statistics.median(v for _, v in ms)
per_day = {}
for t, v in ms:
e = v - level_at(t)
per_day.setdefault(t.date(), []).append((t.hour + t.minute / 60, e))
if noise and abs(e) > o.k * noise:
c = (t.hour * 60 + t.minute) // step * step
slots.setdefault(c, {}).setdefault((s, e > 0), {})[t.date()] = e
for i, (d, p) in enumerate(sorted(per_day.items())):
ax.plot([x for x, _ in p], [e for _, e in p], "-", lw=0.8, color=f"C{i % 10}", label=f"{d:%d/%m}")
ax.axhline(0, color="0.3", ls=":", lw=0.8)
every = sorted(e for p in per_day.values() for _, e in p)
q1, q2 = every[len(every) // 100], every[-1 - len(every) // 100] # frame from the 1st to the 99th percentile: an isolated peak does not crush the scale
margin = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - margin, q2 + margin)
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in every)
for x0, x1 in ((h1, 24), (0, h2)) if h1 > h2 else ((h1, h2),):
ax.axvspan(x0, x1, color="0.9", lw=0)
ax.set_title(s + (" (ref.)" if is_reference(series[s]) else "") + f" noise {noise:.2g}"
+ (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9)
ax.set_xlim(0, 24)
ax.set_xticks(range(0, 25, 3))
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)
if ax is axes.flat[0]:
ax.legend(fontsize=6, ncol=2)
for ax in list(axes.flat)[len(names):]:
ax.axis("off")
fig.suptitle(f"{station}: {col} minus the level of the last night ({h1} h → {h2} h, shaded), "
"one curve per day, on the x axis the hour of the day (UTC); one scale per sensor")
fig.tight_layout()
path = os.path.join(o.out, f"{station}_byhour-{col}_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png")
fig.savefig(path, dpi=100, bbox_inches="tight")
plt.close(fig)
print(path)
n_days = {s: {} for s in names} # sensor -> slot -> measured days
for s in names:
for m in series[s]:
if m[col] is not None:
n_days[s].setdefault((m["t"].hour * 60 + m["t"].minute) // step * step, set()).add(m["t"].date())
print(f"\n{col}: per slot of {step} min of the hour of the day, sensors more than {o.k:g} noises from the night level "
f"on at least {o.days} days (days / measured days, median of these deviations, in noises)")
empty = True
for c in sorted(slots):
cited = []
for s, high in ((s, h) for s in names for h in (True, False)):
ec = slots[c].get((s, high), {})
if len(ec) >= o.days:
med = statistics.median(ec.values())
cited.append(f"{s} {len(ec)}/{len(n_days[s].get(c, ()))} {med:+.3g} ({med / noises[s]:+.1f})")
if cited:
empty = False
print(f"{c // 60:02d}:{c % 60:02d} {len(cited):2} sensors: " + ", ".join(cited))
if empty:
print("(none)")
if o.bounds:
day_bounds(o, names, series, columns, station)
def night_levels(pts, h1, h2):
"""[(t, v)] -> {date of the night: median}."""
nights = {}
for t, v in pts:
if at_night(t, h1, h2):
nights.setdefault((t - timedelta(hours=h1)).date(), []).append(v)
return {d: statistics.median(v) for d, v in nights.items()}
def day_bounds(o, names, series, columns, station):
"""Section 3: per day, the deviation from the level of the night that precedes it, for the target median and the pose."""
h1, h2 = o.night
night_length = timedelta(hours=(h2 - h1) % 24)
chosen = names if o.sensors else [s for s in names if series[s] and not is_reference(series[s])] or names
seqs = [(f"{col} median of {len(chosen)} sensor(s)" if len(chosen) > 1 else f"{col} {chosen[0]}", col,
{s: [(m["t"], m[col]) for m in series[s] if m[col] is not None] for s in chosen}) for col in columns]
poses = filter_window(load_camera(o.folder), o.start, o.end)
seqs += [(f"pose {c}", f"pose_{c}", {c: [(p["t"], p[c]) for p in poses if p[c] is not None]}) for c in POSE
if any(p[c] is not None for p in poses)]
every = [t for _, _, d in seqs for p in d.values() for t, _ in p]
t0, t1 = min(every), max(every)
night_of = lambda t: (t - night_length - timedelta(hours=h1)).date() # night that ends before t (the day of t starts at its end)
days = [night_of(t0) + timedelta(days=i) for i in range((night_of(t1) - night_of(t0)).days + 1)]
hm = lambda t: f"{t:%d/%m %H:%M}"
threshold_txt = f"threshold {o.fraction:g} times the largest |deviation| of the day" if o.fraction else f"threshold {o.k:g} noises"
print(f"\n== Bounds of each day: deviation from the level of the night that precedes it ({h1} h → {h2} h UTC), {threshold_txt}"
+ (f", deviation smoothed over {o.smoothing} measurements" if o.smoothing > 1 else "") + " ==")
print("day: from the end of one night to the end of the next, dated by its start; \"before\", \"after\": measurement just before the first "
"exceedance, just after the last; next night: its level minus that of the reference night")
fig, axes = plt.subplots(len(seqs), 1, sharex=True, figsize=(18, 2.2 * len(seqs)), squeeze=False)
for ax, (name, short, per_sensor) in zip(axes.flat, seqs):
lvl = {s: night_levels(p, h1, h2) for s, p in per_sensor.items()}
nights = sorted({d for n in lvl.values() for d in n})
devs = [] # (day, ref. night, [(t, deviation)], deviation of the next night)
for d in days:
start = datetime.combine(d, datetime.min.time(), tzinfo=timezone.utc) + timedelta(hours=h1) + night_length
end = start + timedelta(days=1)
ref = max((n for n in nights if n <= d), default=None) or (min(nights) if nights else None)
per_t = {}
for s, p in per_sensor.items():
if ref in lvl[s]:
for t, v in p:
if start <= t < end:
per_t.setdefault(t, []).append(v - lvl[s][ref])
next_n = [lvl[s][d + timedelta(days=1)] - lvl[s][ref] for s in lvl if ref in lvl[s] and d + timedelta(days=1) in lvl[s]]
devs.append((start.date(), d, ref, [(t, statistics.median(e)) for t, e in sorted(per_t.items())],
statistics.median(next_n) if next_n else None))
diffs = [abs(b[1] - a_[1]) for *_, p, _ in devs for a_, b in zip(p, p[1:])]
noise = statistics.median(diffs) if diffs else 0
noise = noise if noise >= ROUNDING else 0
print(f"\n{name}: noise {noise:.3g}")
in_noises = lambda e: f"{e:+.4g} ({e / noise:+.1f})" if noise else f"{e:+.4g}"
h = o.smoothing // 2
for day, d, ref, raw, nxt in devs:
if not raw:
print(f"{short} {day:%d/%m} no measurement" + ("" if nights else " at night"))
continue
p = [(t, statistics.median(e for _, e in raw[max(0, i - h):i + h + 1])) for i, (t, _) in enumerate(raw)] if h else raw
threshold = o.fraction * max(abs(e) for _, e in p) if o.fraction else o.k * noise
line = (f"{short} {day:%d/%m} night {ref:%d/%m}" + (" (not the previous one)" if ref != d else "") + f" | {len(p)} measurements {hm(p[0][0])} → {hm(p[-1][0])} | "
+ (f"threshold {threshold:.4g} | " if o.fraction else ""))
over = [i for i, (_, e) in enumerate(p) if threshold and abs(e) > threshold]
if over:
i, j = over[0], over[-1]
line += (f"> {'threshold' if o.fraction else f'{o.k:g} noises'}: {len(over)} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) "
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
ax.axvspan(p[i][0], p[j][0], color="0.85", lw=0)
else:
line += f"never > {'threshold' if o.fraction else f'{o.k:g} noises'}"
top, bottom = max(p, key=lambda x: x[1]), min(p, key=lambda x: x[1])
line += f" | max {in_noises(top[1])} at {hm(top[0])}, min {in_noises(bottom[1])} at {hm(bottom[0])}"
x = max(range(len(p)), key=lambda i: abs(p[i][1]))
sign = 1 if p[x][1] > 0 else -1
if threshold and sign * p[x][1] > threshold:
i = j = x
while i > 0 and sign * p[i - 1][1] > threshold:
i -= 1
while j + 1 < len(p) and sign * p[j + 1][1] > threshold:
j += 1
line += (f" | around the {'max' if sign > 0 else 'min'}: {j - i + 1} measurements, from {hm(p[i][0])} (before {hm(p[i - 1][0]) if i else '-'}) "
f"to {hm(p[j][0])} (after {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
ax.axvspan(p[i][0], p[j][0], color="0.7", lw=0)
line += f" | next night {in_noises(nxt)}" if nxt is not None else " | next night not measured"
print(line)
if h:
ax.plot([t for t, _ in raw], [e for _, e in raw], "-", lw=0.5, color="0.6")
ax.plot([t for t, _ in p], [e for _, e in p], "-", lw=0.8, color="C0")
ax.plot([top[0], bottom[0]], [top[1], bottom[1]], "o", ms=3, color="C3")
if noise and not o.fraction:
for y in (o.k * noise, -o.k * noise):
ax.axhline(y, color="0.4", ls=":", lw=0.8)
ax.axhline(0, color="0.3", lw=0.5)
all_e = sorted(e for *_, p, _ in devs for _, e in p)
out_of_frame = 0
if all_e: # frame from the 1st to the 99th percentile widened by half, as in byhour: an isolated peak does not crush the scale
q1, q2 = all_e[len(all_e) // 100], all_e[-1 - len(all_e) // 100]
margin = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - margin, q2 + margin)
out_of_frame = sum(not q1 - margin <= e <= q2 + margin for e in all_e)
ax.set_title(f"{name} noise {noise:.2g}" + (f" ({out_of_frame} out of frame)" if out_of_frame else ""), fontsize=9)
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)
fig.suptitle(f"{station}: deviation from the level of the night that precedes each day ({h1} h → {h2} h); "
f"shaded light: from the first to the last exceedance {'of the threshold (' + threshold_txt + ')' if o.fraction else f'of {o.k:g} noises'}, dark: the run around the extreme; points: max and min of the day"
+ (f"; deviation smoothed over {o.smoothing} measurements (blue), raw (grey)" if o.smoothing > 1 else ""))
fig.tight_layout()
path = os.path.join(o.out, f"{station}_bounds_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png")
fig.savefig(path, dpi=100, bbox_inches="tight")
plt.close(fig)
print(path)
if __name__ == "__main__":
main()
-162
View File
@@ -1,162 +0,0 @@
"""Lecture des données d'une station pour l'agent de diagnostic (bibliothèque standard).
Formats : labeling/connaissances/colonnes-des-deplacements.md. Toute heure est en UTC.
"""
import csv
import glob
import os
import re
import statistics
from datetime import datetime, timezone
# Colonnes numériques de l'export « advanced » ; une case vide devient None.
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # qualité de l'image : caméras récentes seulement (LENZSPITZE)
# Pose de la caméra (camera/, pompage) : pan, tilt, rotation [deg], scale, sigma0 (résidu du recalage).
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
IMAGE_RE = re.compile(r"^(\d{8}T\d{4})Z_")
def _entete(chemin):
with open(chemin, newline="") as f:
return next(csv.reader(f), [])
def trouver(dossier):
"""Dossier de station -> {'advanced', 'horaire', 'camera', 'images', 'autres'} : chemins triés, sous-dossiers compris."""
out = {"advanced": [], "horaire": [], "camera": [], "images": [], "autres": []}
for p in sorted(glob.glob(os.path.join(dossier, "**", "*"), recursive=True)):
if os.path.isdir(p) or "images_diff" in p.split(os.sep): # images différentielles du pompage (--diff) : pas des vues de la caméra
continue
entete = _entete(p) if p.endswith(".csv") else []
if p.lower().endswith((".jpg", ".jpeg", ".png")):
out["images"].append(p)
elif "sensor_name" in entete and "corr_x" in entete: # quality/ (optical_emulator) a sensor_name, pas corr_x
out["advanced"].append(p)
elif "timestamp" in entete:
out["horaire"].append(p)
elif "median_pan" in entete:
out["camera"].append(p)
else:
out["autres"].append(p)
return out
def _nombre(v):
v = v.strip()
return float(v) if v else None
def charger(dossier):
"""Exports « advanced » du dossier -> {capteur: [mesure]}, triées par t, un seul relevé par instant.
Mesure : dict avec t (datetime UTC), pattern_type, et les colonnes de NUM (float ou None).
Une mesure rejetée (voir rejetee) a corr_x, corr_y vides, comme le dit la doc des colonnes : la station
les vide pour une cible, mais en laisse de remplis (souvent absurdes) pour une référence ; on les vide ici.
"""
par = {}
for chemin in trouver(dossier)["advanced"]:
with open(chemin, newline="") as f:
for r in csv.DictReader(f):
m = {c: _nombre(r.get(c) or "") for c in NUM} # inlier manque dans les exports anciens
m["t"] = instant(r["time_utc"])
m["pattern_type"] = r["pattern_type"]
if rejetee(m):
m["corr_x"] = m["corr_y"] = None
par.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # certains exports finissent le nom par un saut de ligne
return {s: [d[t] for t in sorted(d)] for s, d in par.items()}
def charger_temperature(dossier):
"""Températures de la carte de la station (metrics/, sensor_*_deg_c) -> [(t, °C)] triées : la médiane des composants à
chaque heure ; [] sans ces métriques. Les composants montent et descendent ensemble (l'étiqueteur lit « les
températures de tous les composants ») ; la médiane écarte la pointe d'un seul, le GPU qui calcule (D-124)."""
par = {}
for chemin in glob.glob(os.path.join(dossier, "metrics", "*_deg_c_*.csv")):
with open(chemin, newline="") as f:
for r in csv.DictReader(f):
if (v := _nombre(r.get("metric") or "")) is not None:
par.setdefault(instant(r["time_utc"]), []).append(v)
return [(t, statistics.median(v)) for t, v in sorted(par.items())]
def charger_camera(dossier):
"""Mouvements de la caméra du dossier -> [pose] triées par t ; pose : t et les colonnes de POSE (float ou None).
Une ligne par recalage de la caméra ; on lit les médianes (min = max dans les fichiers reçus : une mesure par minute).
Un instant de mesure sans pose : la caméra n'a pas été recalée.
"""
par = {}
for chemin in trouver(dossier)["camera"]:
with open(chemin, newline="") as f:
for r in csv.DictReader(f):
p = {c: _nombre(r.get("median_" + c, "")) for c in POSE}
p["t"] = instant(r["time_utc"])
par[p["t"]] = p
return [par[t] for t in sorted(par)]
IDENTIFIANTS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # identifiants de la station : jamais lus
METRIQUE_RE = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
def charger_metriques(dossier):
"""Métriques de la station (metrics/, un CSV par métrique et par jour) -> {nom: [(t, valeur)]} triées : les
valeurs numériques seulement (BusVoltage, sensor_CPU_deg_c, rsrp_dBm…) ; les métriques texte (link, mode…) et les
identifiants (SIM_CCID, ip*) ne sont pas lus. Nom : celui du fichier, sans la station ni la date."""
# ponytail: les métriques texte sont sautées ; les lire (link up/down) quand un détecteur en a besoin
station, par = os.path.basename(os.path.normpath(dossier)) + "_", {}
for chemin in glob.glob(os.path.join(dossier, "metrics", "*.csv")):
if not (m := METRIQUE_RE.match(os.path.basename(chemin))):
continue
nom = m.group(1).removeprefix(station)
if IDENTIFIANTS.match(nom):
continue
with open(chemin, newline="") as f:
for r in csv.DictReader(f):
try:
v = float((r.get("metric") or "").strip("[]'\" ")) # certaines métriques arrivent en liste : ['-95']
except ValueError:
continue
par.setdefault(nom, {})[instant(r["time_utc"])] = v
return {n: sorted(d.items()) for n, d in sorted(par.items())}
def est_reference(mesures):
"""Référence du réseau de compensation (pattern_type global_ref), sinon cible. Certains exports ont pattern_type vide,
sur toute la période ou au début : on lit alors le premier pattern_type rempli et, s'il n'y en a aucun, reproj_err
ou inlier remplis (références seulement : labeling/connaissances/colonnes-des-deplacements.md)."""
t = next((m["pattern_type"] for m in mesures if m["pattern_type"]), None)
if t is not None:
return t == "global_ref"
return any(m["reproj_err"] is not None or m["inlier"] is not None for m in mesures)
def rejetee(m):
"""Mesure rejetée par la station : error_value au-dessus du seuil du capteur (error_threshold) ; sans ces deux valeurs, corr_x vide."""
if m["error_value"] is not None and m["error_threshold"] is not None:
return m["error_value"] > m["error_threshold"]
return m["corr_x"] is None
def filtrer(mesures, debut, fin):
"""Mesures entre debut et fin inclus, à la minute près (comme iso() les affiche) ; None = pas de borne."""
return [m for m in mesures if (debut is None or m["t"] >= debut)
and (fin is None or m["t"].replace(second=0, microsecond=0) <= fin)]
def heure_image(chemin):
"""'20261013T1200Z_10167931.jpg' -> datetime UTC ; None si le nom ne suit pas ce format (D-045)."""
m = IMAGE_RE.match(os.path.basename(chemin))
return datetime.strptime(m.group(1), "%Y%m%dT%H%M").replace(tzinfo=timezone.utc) if m else None
def instant(texte):
"""'2026-01-01T00:00Z' ou '2026-01-01 00:00' -> datetime UTC ; une heure sans fuseau est lue en UTC."""
t = datetime.fromisoformat(texte)
return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc)
def iso(t):
return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")
-491
View File
@@ -1,491 +0,0 @@
"""Vue d'ensemble de tous les capteurs d'un export « advanced » : ce que font les capteurs ensemble.
Usage : .venv/bin/python tools/scripts/ensemble.py data/PS_DES_COUDRES/
[--colonnes x,y,sigma0] [--colonne error_value] [--pas 15min] [--k 5] [--min 5]
[--residus corr_x,corr_y,error_value] [--hors T1/T2] [--autres] [--rigide BASE [--sans A,B] [--suivre A,B]] [--table T1/T2|non] [--capteurs A,B] [--debut ...] [--fin ...] [--sortie DOSSIER]
Chaque capteur est comparé à son propre niveau sur la fenêtre --debut/--fin (médiane et MAD de la fenêtre).
Deux PNG, dont les chemins sont imprimés :
1. « commun » : d'abord, si le dossier a des mouvements de la caméra (camera/), un panneau par grandeur de la pose
(pan, tilt, rotation [deg], scale, sigma0 du recalage), valeurs brutes, un point par recalage ; puis un panneau par colonne de --colonnes (défaut x,y,sigma0 : position brute et sigma0 de la
caméra) : médiane, sur les capteurs, de l'écart de chaque capteur à sa médiane ; bande grise = 10e à 90e
centile entre capteurs (la dispersion) ; sur chaque panneau sauf sigma0, en plus, chaque référence globale à part
(son écart à sa médiane, un trait par référence) et la médiane des cibles seules (trait noir) : on voit si les
références bougent dans l'image avec les cibles ou sans elles ; dernier panneau : nombre de capteurs dont --colonne
(défaut error_value) dépasse sa médiane de plus de --k MAD à chaque pas.
2. « multiples » : un petit graphique par capteur (références d'abord) de --colonne, médiane par pas de --pas,
trait pointillé = médiane de la fenêtre ; chaque graphique a son échelle, un palier se voit.
3. « residus-<colonne> » (un PNG par colonne de --residus, défaut corr_x,corr_y,error_value) : un petit graphique par
capteur de la colonne moins le niveau propre du capteur (sa médiane sur la fenêtre, ou hors de --hors T1/T2 si
donné : le niveau est alors celui du reste de la fenêtre, pas celui du moment lu), une échelle par capteur, zéro en
pointillé ; un écart de quelques centièmes sur une colonne `corr` que les bandes parallèles de `graphique.py` cachent
s'y voit. Imprime, par capteur, le niveau, le bruit propre (médiane des écarts entre deux mesures consécutives) et
l'écart minimum et maximum de la médiane par pas de --pas, avec leur instant, en unités de la colonne et en bruits
(bruit 0 et « - » en bruits : valeurs constantes à l'arrondi près, 1e-9 ; --residus non : aucun PNG de résidus).
Avec --autres, chaque capteur est de plus ramené à l'écart à la médiane des AUTRES capteurs au même instant (on
retranche, à chaque mesure, la médiane des autres capteurs de la même colonne moins leur niveau propre) : ce qui est
partagé par tous (le reste de la compensation d'un mouvement commun) s'annule, ce qui est propre au capteur reste.
PNG « residus-<colonne>-autres » ; la table imprime en plus, par capteur, la pente de ce résidu sur chaque
colonne de --colonnes (médiane commune, ex. y) et la corrélation r : un résidu proportionnel au mouvement commun
(même s'il n'est pas partagé par tous) a une pente non nulle et un r élevé.
4. « rigide » (avec --rigide BASE) : à chaque mesure, le mouvement d'ensemble des cibles depuis l'instant BASE
(similitude : translation, rotation, échelle, estimée par médianes sur les cibles, hors --sans) ; pour chaque
capteur, le résidu dx, dy = position brute moins sa position à BASE transportée par ce mouvement [px]. Panneaux :
dx et dy de chaque référence globale (un trait chacune) face à la médiane et au 10e-90e centile des cibles ;
médiane et 90e centile de |résidu| des cibles ; rotation et échelle ajustées ; reproj_err des références. Imprime
la table par pas de --pas (rot_deg, echelle, tx, ty, cib_med, cib_max, puis dx, dy de chaque référence et de chaque
capteur de --suivre A,B), puis par capteur le min et le max de dx, dy avec leur instant et la valeur au dernier
instant. Les mesures rejetées n'entrent ni dans l'ajustement, ni dans la bande et la médiane des cibles, cib_med,
cib_max, ni dans les colonnes de la table ; sur le PNG, celles des références sont des croix, qui ne fixent pas l'échelle ; dans le résumé par
capteur, elles viennent à part (nombre, min et max).
Imprime aussi la table de la médiane commune (une ligne par pas de --pas : heure, écart à la médiane de la fenêtre
pour chaque colonne de --colonnes, puis <colonne>_cibles, la médiane des cibles seules, puis la pose brute de la caméra, vide sans recalage) puis le minimum et le maximum de chaque colonne avec leur instant, et ceux de chaque référence globale ; pour dater
un début ou une fin, la table est au pas de --pas (défaut 15min, la cadence des mesures ; un pas plus large,
1h par ex., aplatit un pic d'un à quatre pas et date à l'heure).
Imprime aussi, pour chaque instant où au moins --min capteurs dépassent leur médiane de plus de --k MAD,
le nombre et les capteurs (--min 1 : aussi les instants où un seul capteur sort de son niveau), et les instants de
mesure sans pose de la caméra (pas de recalage).
Les tables par pas (médiane commune, --rigide) répètent leur en-tête toutes les 24 lignes. Avec --table T1/T2, elles ne
s'impriment qu'entre T1 et T2 (« non » : pas du tout) ; niveaux, minimum et maximum, PNG et le reste portent toujours sur
toute la fenêtre --debut/--fin : sur un mois, on lit tout le reste, puis la table d'un jour aux mêmes niveaux.
Rend des valeurs, pas de verdict.
"""
import argparse
import math
import os
import statistics
import sys
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402
from donnees import POSE, charger, charger_camera, est_reference, filtrer, instant, iso, rejetee # noqa: E402
from graphique import COLONNES, borne # noqa: E402
from resume import bruit_propre, duree # noqa: E402
def niveau(ms, col):
"""(médiane, MAD) des valeurs non vides ; None si aucune."""
v = [m[col] for m in ms if m[col] is not None]
if not v:
return None
med = statistics.median(v)
return med, statistics.median(abs(x - med) for x in v)
def similitude(p, q):
"""Similitude (a, b, tx, ty) qui envoie les points p sur q : q ≈ (a·x − b·y + tx, b·x + a·y + ty).
Estimation par médianes (une cible qui saute seule ne tire pas l'ajustement) : rotation et échelle, médiane sur
toutes les paires de cibles du rapport complexe (qi − qj) / (pi − pj) ; translation, médiane de q − (rotation, échelle)·p.
"""
p, q = [complex(*u) for u in p], [complex(*u) for u in q]
z = [(q[i] - q[j]) / (p[i] - p[j]) for i in range(len(p)) for j in range(i) if p[i] != p[j]]
r = complex(statistics.median(v.real for v in z), statistics.median(v.imag for v in z))
d = [w - r * u for u, w in zip(p, q)]
return r.real, r.imag, statistics.median(v.real for v in d), statistics.median(v.imag for v in d)
def rigide(o, noms, series, temps, base, station):
"""4. Chaque capteur face au mouvement d'ensemble des cibles depuis l'instant --rigide : PNG « rigide » et table."""
t_b = min(temps, key=lambda t: abs(t - o.rigide))
pos = {s: {m["t"]: (m["x"], m["y"]) for m in series[s] if m["x"] is not None and m["y"] is not None} for s in noms}
sans = set(o.sans.split(",")) - {""}
acc = {s: {m["t"] for m in series[s] if not rejetee(m)} for s in noms} # mesures acceptées : seules à entrer dans l'ajustement et les statistiques des cibles
refs = [s for s in noms if series[s] and est_reference(series[s])]
cib = [s for s in noms if s not in refs and t_b in pos[s]]
ajust = [s for s in cib if s not in sans and t_b in acc[s]]
suivis = [s for s in o.suivre.split(",") if s and s not in refs]
res, fit = {s: {} for s in noms}, {} # res : {capteur: {t: (dx, dy)}} ; fit : {t: (rotation deg, échelle, tx, ty, |r| médiane, max des cibles)}
for t in temps:
ok = [s for s in ajust if t in pos[s] and t in acc[s]]
if len(ok) < 2:
continue
a, b, tx, ty = similitude([pos[s][t_b] for s in ok], [pos[s][t] for s in ok])
for s in noms:
if t in pos[s] and t_b in pos[s]:
(x0, y0), (x, y) = pos[s][t_b], pos[s][t]
res[s][t] = (x - (a * x0 - b * y0 + tx), y - (b * x0 + a * y0 + ty))
r = [math.hypot(*res[s][t]) for s in cib if t in res[s] and t in acc[s]]
fit[t] = (math.degrees(math.atan2(b, a)), math.hypot(a, b), tx, ty, statistics.median(r), max(r))
if not fit:
print(f"\n--rigide : moins de deux cibles avec x, y à {iso(t_b)} et ensuite.")
return
tf = sorted(fit)
reproj = {s: {m["t"]: m["reproj_err"] for m in series[s] if m["reproj_err"] is not None} for s in refs}
panneaux = [("dx", 0), ("dy", 1)]
fig, axes = plt.subplots(6, 1, sharex=True, figsize=(14, 18), squeeze=False)
for ax, (nom, k) in zip(axes[:2, 0], panneaux):
band = [sorted(res[s][t][k] for s in cib if t in res[s] and t in acc[s]) for t in tf]
q = [statistics.quantiles(v, n=10, method="inclusive") if len(v) >= 2 else [v[0]] * 9 for v in band]
ax.fill_between(tf, [x[0] for x in q], [x[-1] for x in q], color="0.85", label="cibles : 10e-90e centile")
ax.plot(tf, [statistics.median(v) for v in band], "-", lw=1.2, color="k", label="cibles : médiane")
for i, s in enumerate(refs):
ta = [t for t in res[s] if t in acc[s]]
ax.plot(ta, [res[s][t][k] for t in ta], "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (réf.)")
cadre = ax.get_ylim() # l'échelle des mesures acceptées : une référence rejetée loin est hors cadre
for i, s in enumerate(refs):
tr = [t for t in res[s] if t not in acc[s]]
if tr: # mesures rejetées : à part, une croix chacune, hors du trait
ax.plot(tr, [res[s][t][k] for t in tr], "x", ms=3, color=f"C{(i + 1) % 10}", label=f"{s} rejetée")
ax.set_ylim(cadre)
ax.set_ylabel(f"{nom} [px]\nposition brute moins\nle mouvement des cibles")
ax.legend(loc="upper left", fontsize=8, ncol=1 + len(refs) // 4)
axes[2, 0].plot(tf, [fit[t][4] for t in tf], "-", lw=1, color="k", label="cibles : médiane de |résidu|")
p90 = {t: statistics.quantiles([math.hypot(*res[s][t]) for s in cib if t in res[s] and t in acc[s]], n=10, method="inclusive")[-1] for t in tf}
axes[2, 0].plot(tf, [p90[t] for t in tf], "-", lw=0.8, color="C3", label="cibles : 90e centile de |résidu| (le max est dans la table)")
axes[2, 0].set_ylabel("|résidu| des cibles [px]")
axes[2, 0].legend(loc="upper left", fontsize=8)
axes[3, 0].plot(tf, [fit[t][0] for t in tf], "-", lw=1, color="C2")
axes[3, 0].set_ylabel("rotation ajustée [deg]")
axes[4, 0].plot(tf, [fit[t][1] for t in tf], "-", lw=1, color="C2")
axes[4, 0].set_ylabel("échelle ajustée")
for i, s in enumerate(refs):
axes[5, 0].plot(list(reproj[s]), list(reproj[s].values()), "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (réf.)")
axes[5, 0].set_ylabel("reproj_err\ndes références")
if refs:
axes[5, 0].legend(loc="upper left", fontsize=8, ncol=1 + len(refs) // 4)
for ax in axes[:, 0]:
ax.grid(alpha=0.4)
ax.axvline(t_b, color="C3", ls=":", lw=0.8)
axes[-1, 0].set_xlim(tf[0], tf[-1])
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
fig.suptitle(f"{station} : mouvement d'ensemble (translation, rotation, échelle) ajusté sur {len(ajust)} cibles depuis {iso(t_b)} (pointillé), "
"résidu de chaque capteur, mesures acceptées (croix : référence rejetée) (heures UTC)")
fig.tight_layout(rect=(0, 0, 1, 1 - 0.4 / fig.get_figheight()))
chemin = base.format("rigide")
fig.savefig(chemin, dpi=100, bbox_inches="tight")
plt.close(fig)
print(chemin)
print(f"\nMouvement d'ensemble ajusté sur les cibles depuis {iso(t_b)} : {', '.join(ajust)}"
+ (f" (hors ajustement : {', '.join(sorted(sans & set(cib)))})" if sans & set(cib) else "")
+ f".\nRésidu = position brute moins la position de {iso(t_b)} transportée par ce mouvement [px] ; mesures rejetées hors de l'ajustement, "
f"de cib_med, cib_max et des colonnes des capteurs (à part dans le résumé par capteur) ; médiane par pas de {int(o.pas.total_seconds() // 60)} min{' (table : ' + o.table + ')' if o.table else ''} :")
cols = ["rot_deg", "echelle", "tx", "ty", "cib_med", "cib_max"] + [f"{s}_{c}" for s in refs + suivis for c in ("dx", "dy")]
minuit = tf[0].replace(hour=0, minute=0, second=0, microsecond=0)
t0 = minuit + ((tf[0] - minuit) // o.pas) * o.pas
par_pas = {}
for t in tf:
ligne = par_pas.setdefault(t0 + ((t - t0) // o.pas) * o.pas, {})
for c, v in zip(cols, fit[t]):
ligne.setdefault(c, []).append(v)
for s in refs + suivis:
if t in res[s] and t in acc[s]:
ligne.setdefault(f"{s}_dx", []).append(res[s][t][0]), ligne.setdefault(f"{s}_dy", []).append(res[s][t][1])
w = max(10, max(len(c) for c in cols) + 1)
entete = f"{'instant':17}" + "".join(f"{c:>{w}}" for c in cols)
for i, t in enumerate(t for t in sorted(par_pas) if o.dans_table(t)):
if i % 24 == 0:
print(entete)
print(f"{iso(t):17}" + "".join(f"{statistics.median(par_pas[t][c]):{w}.{7 if c == 'echelle' else 3}g}" if c in par_pas[t] else " " * w for c in cols))
print("\nPar capteur, mesures acceptées : min et max de dx, dy, valeur au dernier instant ; puis les rejetées à part :")
for s in noms:
bons = {t: v for t, v in res[s].items() if t in acc[s]}
rej = {t: v for t, v in res[s].items() if t not in acc[s]}
for k, c in ((0, "dx"), (1, "dy")):
ligne = f"{s:9}{' (réf.)' if s in refs else ' '} {c}"
if bons:
lo, hi = min(bons.items(), key=lambda x: x[1][k]), max(bons.items(), key=lambda x: x[1][k])
der = res[s].get(tf[-1])
ligne += (f" min {lo[1][k]:+.3g} à {iso(lo[0])}, max {hi[1][k]:+.3g} à {iso(hi[0])}, au dernier instant "
f"{der[k] if der else math.nan:+.3g}{' (rejetée)' if der and tf[-1] not in acc[s] else ''}")
else:
ligne += " aucune mesure acceptée"
if rej:
lo, hi = min(rej.items(), key=lambda x: x[1][k]), max(rej.items(), key=lambda x: x[1][k])
ligne += f" ; {len(rej)} rejetées : min {lo[1][k]:+.3g} à {iso(lo[0])}, max {hi[1][k]:+.3g} à {iso(hi[0])}"
if bons or rej:
print(ligne)
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossier")
a.add_argument("--colonnes", default="x,y,sigma0")
a.add_argument("--colonne", default="error_value", choices=COLONNES)
a.add_argument("--pas", default="15min", type=duree, help="pas des tables et des courbes en bleu (défaut 15min, la cadence des mesures : un pas plus large aplatit et décale les événements courts)")
a.add_argument("--k", type=float, default=5, help="écart compté, en MAD de la fenêtre")
a.add_argument("--min", type=int, default=5, help="nombre de capteurs à partir duquel un instant est imprimé")
a.add_argument("--residus", default="corr_x,corr_y,error_value", help="colonnes tracées moins le niveau propre de chaque capteur (« non » ou '' : aucune)")
a.add_argument("--hors", help="T1/T2 : le niveau propre des résidus se calcule hors de ce moment")
a.add_argument("--autres", action="store_true", help="résidus de plus ramenés à la médiane des autres capteurs au même instant, avec pente et corrélation sur le mouvement commun")
a.add_argument("--rigide", type=instant, metavar="BASE", help="résidus face au mouvement d'ensemble des cibles (translation, rotation, échelle) depuis l'instant BASE")
a.add_argument("--sans", default="", help="A,B : cibles laissées hors de l'ajustement de --rigide (leur résidu reste tracé)")
a.add_argument("--suivre", default="", help="A,B : avec --rigide, dx, dy de ces capteurs ajoutés à la table au pas (les références y sont déjà)")
a.add_argument("--table", default="", metavar="T1/T2|non",
help="tables par pas imprimées seulement entre T1 et T2 (« non » : pas du tout) ; le reste sur toute la fenêtre")
a.add_argument("--capteurs", help="liste séparée par des virgules (défaut : tous)")
a.add_argument("--debut", type=instant)
a.add_argument("--fin", type=instant)
a.add_argument("--sortie", default=os.path.join("out", "diagnostic"), help="dossier des PNG")
o = a.parse_args()
par = charger(o.dossier)
if not par:
sys.exit(f"Aucun export « advanced » dans {o.dossier} (voir inventaire.py).")
noms = o.capteurs.split(",") if o.capteurs else sorted(par, key=lambda s: (not est_reference(par[s]), s))
colonnes = o.colonnes.split(",")
residus = [c for c in o.residus.split(",") if c and c != "non"] # '' ou « non » : aucune, comme --table
hors = tuple(instant(x) for x in o.hors.split("/")) if o.hors else None
if hors and len(hors) != 2:
sys.exit("--hors attend T1/T2.")
if o.table in ("", "non"):
o.dans_table = lambda t: o.table == ""
else:
bornes = [instant(x) for x in o.table.split("/")]
if len(bornes) != 2:
sys.exit("--table attend T1/T2 ou non.")
o.dans_table = lambda t: bornes[0] <= t <= bornes[1]
for demandes, connus, quoi in ((noms, par, "Capteurs"), ([s for s in o.suivre.split(",") if s], noms, "Capteurs de --suivre (pris parmi --capteurs)"), (colonnes + residus + [o.colonne], COLONNES, "Colonnes")):
absents = [x for x in demandes if x not in connus]
if absents:
sys.exit(f"{quoi} inconnus : {', '.join(absents)}. Connus : {', '.join(sorted(connus))}.")
series = {s: filtrer(par[s], o.debut, o.fin) for s in noms}
if not any(series.values()):
sys.exit("Aucune mesure dans la fenêtre demandée.")
temps = sorted({m["t"] for s in noms for m in series[s]})
poses = filtrer(charger_camera(o.dossier), temps[0].replace(second=0, microsecond=0), temps[-1]) # poses à la minute, mesures à la seconde
pose_cols = POSE if poses else ()
station = os.path.basename(os.path.normpath(o.dossier))
os.makedirs(o.sortie, exist_ok=True)
base = os.path.join(o.sortie, f"{station}_{{}}_{borne(o.debut, 'debut')}_{borne(o.fin, 'fin')}.png")
def ecarts(col):
"""{capteur: {t: valeur - médiane de la fenêtre}}, et les MAD."""
e, mad = {}, {}
for s in noms:
n = niveau(series[s], col)
if n:
e[s] = {m["t"]: m[col] - n[0] for m in series[s] if m[col] is not None}
mad[s] = n[1]
return e, mad
# 1. commun
n_ax = len(pose_cols) + len(colonnes) + 1
fig, axes = plt.subplots(n_ax, 1, sharex=True, figsize=(14, 3 * n_ax), squeeze=False)
for ax, col in zip(axes[:, 0], pose_cols):
pose_t = {p["t"].replace(second=0, microsecond=0): p[col] for p in poses if p[col] is not None}
cam_t = sorted(set(pose_t) | {t.replace(second=0, microsecond=0) for t in temps})
# trait coupé aux instants de mesure sans recalage : il ne relie pas deux poses par-dessus un trou
ax.plot(cam_t, [pose_t.get(t, math.nan) for t in cam_t], ".-", ms=3, lw=0.5, color="C2", label="pose de la caméra (un point par recalage)")
ax.set_ylabel(f"caméra : {col}" + (" [deg]" if col in ("pan", "tilt", "rotation") else ""))
ax.legend(loc="upper left", fontsize=8)
communes, refs_par_col, ref_ecarts = {}, {}, {}
for ax, col in zip(axes[len(pose_cols):, 0], colonnes):
e, _ = ecarts(col)
med, bas, haut, tt = [], [], [], []
for t in temps:
v = sorted(e[s][t] for s in e if t in e[s])
if len(v) >= 2:
q = statistics.quantiles(v, n=10, method="inclusive")
tt.append(t), med.append(statistics.median(v)), bas.append(q[0]), haut.append(q[-1])
communes[col] = dict(zip(tt, med))
ax.fill_between(tt, bas, haut, color="0.85", label="10e-90e centile entre capteurs")
ax.plot(tt, med, ".-", ms=2, lw=0.8, label="médiane des capteurs")
refs = [s for s in e if est_reference(series[s])]
if refs and col != "sigma0": # sigma0 est le même pour tous : rien à séparer
# références globales une à une, et médiane des seules cibles : qui bouge dans l'image, les références ou tout
cib = {}
for t in temps:
v = [e[s][t] for s in e if s not in refs and t in e[s]]
if v:
cib[t] = statistics.median(v)
communes[col + "_cibles"] = cib
ax.plot(list(cib), list(cib.values()), "-", lw=1.2, color="k", label="médiane des cibles seules")
for i, s in enumerate(refs):
ax.plot(list(e[s]), list(e[s].values()), "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (réf.)")
refs_par_col[col] = refs
ref_ecarts[col] = {s: e[s] for s in refs}
ax.set_ylabel(f"{col}\nécart à sa médiane")
ax.grid(alpha=0.3)
ax.legend(loc="upper left", fontsize=8, ncol=1 + len(refs_par_col.get(col, [])) // 4)
e, mad = ecarts(o.colonne)
compte, qui = [], {}
for t in temps:
hauts = [s for s in e if t in e[s] and mad[s] > 0 and e[s][t] > o.k * mad[s]]
compte.append(len(hauts))
if len(hauts) >= o.min:
qui[t] = hauts
axes[-1, 0].step(temps, compte, where="mid", lw=0.8)
axes[-1, 0].set_ylabel(f"capteurs au-dessus\nde médiane + {o.k:g} MAD\n({o.colonne})")
axes[-1, 0].grid(alpha=0.3)
h = next((h for h in (1, 2, 3, 6, 12) if (temps[-1] - temps[0]).total_seconds() / 3600 / h <= 20), 24)
jours = (temps[-1] - temps[0]).total_seconds() / 86400
# au-delà de 20 jours, une graduation par jour se chevauche : tous les n jours, au plus 20 graduations
axes[-1, 0].xaxis.set_major_locator(mdates.HourLocator(byhour=range(0, 24, h)) if h < 24 else mdates.DayLocator(interval=math.ceil(jours / 20)))
axes[-1, 0].set_xlim(temps[0], temps[-1])
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
for ax in axes[:, 0]:
ax.grid(alpha=0.5, which="major")
fig.suptitle(f"{station} : {len(e)} capteurs ensemble, chacun face à son niveau sur la fenêtre (heures UTC)")
fig.tight_layout(rect=(0, 0, 1, 1 - 0.4 / fig.get_figheight())) # place du titre
commun = base.format("commun")
fig.savefig(commun, dpi=100, bbox_inches="tight")
plt.close(fig)
print(commun)
# table de la médiane commune, par pas
minuit = temps[0].replace(hour=0, minute=0, second=0, microsecond=0)
t0 = minuit + ((temps[0] - minuit) // o.pas) * o.pas # pas calés sur l'horloge (hh:00, hh:15…), pas sur la 1re mesure
lignes_t = {}
for col in colonnes:
for t, v in communes[col].items():
lignes_t.setdefault(t0 + ((t - t0) // o.pas) * o.pas, {}).setdefault(col, []).append(v)
for col in pose_cols:
communes["cam_" + col] = {p["t"]: p[col] for p in poses if p[col] is not None}
for t, v in communes["cam_" + col].items():
lignes_t.setdefault(t0 + ((t - t0) // o.pas) * o.pas, {}).setdefault("cam_" + col, []).append(v)
for col in refs_par_col:
for t, v in communes[col + "_cibles"].items():
lignes_t.setdefault(t0 + ((t - t0) // o.pas) * o.pas, {}).setdefault(col + "_cibles", []).append(v)
cols_t = colonnes + [c + "_cibles" for c in refs_par_col] + ["cam_" + c for c in pose_cols]
print(f"\nMédiane commune (écart à la médiane de la fenêtre){' et pose brute de la caméra (cam_, vide sans recalage)' if pose_cols else ''}, "
f"médiane par pas de {int(o.pas.total_seconds() // 60)} min{' (table : ' + o.table + ')' if o.table else ''} :")
entete = f"{'instant':17}" + "".join(f"{c:>13}" for c in cols_t)
for i, t in enumerate(t for t in sorted(lignes_t) if o.dans_table(t)):
if i % 24 == 0:
print(entete)
print(f"{iso(t):17}" + "".join(f"{statistics.median(lignes_t[t][c]):13.{6 if c.startswith('cam_') else 4}g}" if c in lignes_t[t] else " " * 13 for c in cols_t))
for col in cols_t:
if communes[col]:
lo, hi = min(communes[col].items(), key=lambda x: x[1]), max(communes[col].items(), key=lambda x: x[1])
g = 6 if col.startswith("cam_") else 4
print(f"{col} (mesures) : min {lo[1]:.{g}g} à {iso(lo[0])}, max {hi[1]:.{g}g} à {iso(hi[0])}")
for col, refs in refs_par_col.items():
print(f"\n{col} brut des références globales, écart à leur médiane de la fenêtre (mesures) ; {col}_cibles : médiane des cibles seules :")
for s in refs:
d = ref_ecarts[col][s]
lo, hi = min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
rej = {m["t"] for m in series[s] if rejetee(m)}
r = lambda t: " (rejetée)" if t in rej else ""
print(f"{s:9} min {lo[1]:+.3g} à {iso(lo[0])}{r(lo[0])}, max {hi[1]:+.3g} à {iso(hi[0])}{r(hi[0])}"
+ (f" ; {len(rej)} mesures rejetées dans la fenêtre" if rej else ""))
if not pose_cols:
print("Pose de la caméra : aucune dans la fenêtre (pas de camera/ dans le dossier, ou pas de recalage).")
else:
avec = {p["t"].replace(second=0, microsecond=0) for p in poses}
suites = [] # suites d'instants de mesure consécutifs sans pose : [début, fin, nombre]
for i, t in enumerate(temps):
if t.replace(second=0, microsecond=0) in avec:
continue
if suites and suites[-1][3] == i - 1:
suites[-1][1], suites[-1][2], suites[-1][3] = t, suites[-1][2] + 1, i
else:
suites.append([t, t, 1, i])
print(f"Instants de mesure sans pose de la caméra (pas de recalage) : {sum(x[2] for x in suites)} sur {len(temps)}"
+ (" : " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n, _ in suites) if suites else ""))
# 2. petits multiples
col = o.colonne
n = len(noms)
lignes = math.ceil(n / 5)
fig, axes = plt.subplots(lignes, 5, sharex=True, figsize=(18, 2.5 * lignes), squeeze=False)
for ax, s in zip(axes.flat, noms):
ms = [m for m in series[s] if m[col] is not None]
ax.plot([m["t"] for m in ms], [m[col] for m in ms], "-", lw=0.4, color="0.7")
pas = {}
for m in ms:
pas.setdefault(t0 + ((m["t"] - t0) // o.pas) * o.pas, []).append(m[col])
ax.plot(list(pas), [statistics.median(v) for v in pas.values()], "-", lw=1.1, color="C0")
lv = niveau(series[s], col)
if lv:
ax.axhline(lv[0], color="C3", ls=":", lw=0.8)
ax.set_title(s + (" (réf.)" if series[s] and est_reference(series[s]) else ""), fontsize=9)
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
for ax in list(axes.flat)[n:]:
ax.axis("off")
fig.suptitle(f"{station} : {col}, médiane par pas de {int(o.pas.total_seconds() // 60)} min (bleu), mesures (gris), "
f"médiane de la fenêtre (pointillé) ; une échelle par capteur (heures UTC)")
fig.tight_layout()
multiples = base.format(f"multiples-{col}")
fig.savefig(multiples, dpi=100, bbox_inches="tight")
print(multiples)
# 3. résidus : chaque capteur moins son propre niveau
for col, autres in [(c, False) for c in residus] + [(c, True) for c in residus if o.autres]:
fig, axes = plt.subplots(lignes, 5, sharex=True, figsize=(18, 2.5 * lignes), squeeze=False)
dev = {}
if autres: # {capteur: {t: valeur - niveau propre}} pour la médiane des autres
for q in noms:
mq = [m for m in series[q] if m[col] is not None]
rq = [m for m in mq if not hors or not hors[0] <= m["t"] <= hors[1]] or mq
nq = niveau(rq, col)
if nq:
dev[q] = {m["t"]: m[col] - nq[0] for m in mq}
print(f"\n{col} moins le niveau propre{' et moins la médiane des autres capteurs au même instant' if autres else ''} ({'hors ' + o.hors if hors else 'médiane de la fenêtre'}), médiane par pas de "
f"{int(o.pas.total_seconds() // 60)} min :")
pentes = []
print(f"{'capteur':9}{'niveau':>10}{'bruit':>10}{'écart min':>11} {'à':<17}{'(bruits)':>9}{'écart max':>11} {'à':<17}{'(bruits)':>9}")
for ax, s in zip(axes.flat, noms):
ms = [m for m in series[s] if m[col] is not None]
ref = [m for m in ms if not hors or not hors[0] <= m["t"] <= hors[1]] or ms
lv = niveau(ref, col)
if not lv:
ax.axis("off")
continue
bruit = bruit_propre([m[col] for m in ms])
if autres:
res = {}
for m in ms:
v = [dev[q][m["t"]] for q in dev if q != s and m["t"] in dev[q]]
if len(v) >= 2:
res[m["t"]] = m[col] - lv[0] - statistics.median(v)
if len(res) < 2:
ax.axis("off")
continue
else:
res = {m["t"]: m[col] - lv[0] for m in ms}
pas = {}
for t, v in res.items():
pas.setdefault(t0 + ((t - t0) // o.pas) * o.pas, []).append(v)
pm = {t: statistics.median(v) for t, v in pas.items()}
ax.plot(list(res), list(res.values()), "-", lw=0.4, color="0.7")
ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0")
ax.axhline(0, color="C3", ls=":", lw=0.8)
ax.set_title(s + (" (réf.)" if est_reference(ms) else "") + f" bruit {bruit:.2g}", fontsize=9)
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
lo, hi = min(pm.items(), key=lambda x: x[1]), max(pm.items(), key=lambda x: x[1])
f = lambda v: f"{v / bruit:9.1f}" if bruit else f"{'-':>9}"
print(f"{s:9}{lv[0]:10.4g}{bruit:10.3g}{lo[1]:11.3g} {iso(lo[0]):<17}{f(lo[1])}{hi[1]:11.3g} {iso(hi[0]):<17}{f(hi[1])}")
if autres:
for c in colonnes:
tt = [t for t in res if t in communes[c]]
if len(tt) > 2 and len({communes[c][t] for t in tt}) > 1:
xs, ys = [communes[c][t] for t in tt], [res[t] for t in tt]
pente = statistics.linear_regression(xs, ys).slope
pentes.append(f"{s:9}pente sur {c} commun {pente:+.4g} par unité, r = {statistics.correlation(xs, ys):+.2f}")
for ax in list(axes.flat)[n:]:
ax.axis("off")
if pentes:
print("\n".join(pentes))
fig.suptitle(f"{station} : {col} moins le niveau propre{' et la médiane des autres' if autres else ''} de chaque capteur, médiane par pas de "
f"{int(o.pas.total_seconds() // 60)} min (bleu), mesures (gris) ; une échelle par capteur (heures UTC)")
fig.tight_layout()
chemin = base.format(f"residus-{col}" + ("-autres" if autres else ""))
fig.savefig(chemin, dpi=100, bbox_inches="tight")
plt.close(fig)
print(chemin)
if o.rigide:
rigide(o, noms, series, temps, base, station)
print(f"\nInstants où au moins {o.min} capteurs ont {o.colonne} > médiane de la fenêtre + {o.k:g} MAD :")
for t, hauts in qui.items():
print(f"{iso(t)} {len(hauts):3} {','.join(hauts)}")
if not qui:
print("(aucun)")
if __name__ == "__main__":
main()
+43 -43
View File
@@ -1,12 +1,12 @@
"""GIF des images de la caméra sur une plage, pour joindre à une réponse au client.
"""GIF of the camera images over a range, to attach to a reply to the client.
Usage : .venv/bin/python tools/scripts/gif.py data/LENZSPITZE/
--debut 2026-10-26T18:00Z --fin 2026-10-27T08:00Z [--largeur 720] [--duree 700] [--max 150] [--zone X,Y,L,H] [--sortie DOSSIER]
Prend les images du dossier dont l'heure (nom du fichier, UTC) est entre --debut et --fin inclus, dans l'ordre, les
recadre au besoin sur --zone (pixels de l'image d'origine : coin haut gauche, largeur, hauteur), les met à --largeur px,
écrit la date et l'heure UTC en bas à droite de chacune et les ramène à 64 teintes sans tramage
(le bruit des JPEG gonfle un GIF : ~3 Mo pour une nuit d'images horaires à 720 px). Au-delà de --max images, en garde
--max à intervalle régulier. Imprime l'heure de chaque image retenue, puis le chemin du GIF et sa taille.
Usage: .venv/bin/python tools/scripts/gif.py data/LENZSPITZE/
--start 2026-10-26T18:00Z --end 2026-10-27T08:00Z [--width 720] [--duration 700] [--max 150] [--zone X,Y,W,H] [--out FOLDER]
Takes the images of the folder whose time (file name, UTC) is between --start and --end inclusive, in order,
crops them if needed to --zone (pixels of the original image: top left corner, width, height), scales them to --width px,
writes the UTC date and time at the bottom right of each one and reduces them to 64 colours without dithering
(JPEG noise inflates a GIF: ~3 MB for a night of hourly images at 720 px). Beyond --max images, keeps
--max at regular intervals. Prints the time of each image kept, then the path of the GIF and its size.
"""
import argparse
import os
@@ -14,55 +14,55 @@ import sys
from PIL import Image, ImageDraw, ImageFont
from donnees import heure_image, instant, iso, trouver
from data import find_files, image_time, iso, parse_time
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossier")
a.add_argument("--debut", required=True, help="première heure (UTC)")
a.add_argument("--fin", required=True, help="dernière heure (UTC), incluse")
a.add_argument("--largeur", type=int, default=720, help="largeur du GIF en pixels (défaut 720)")
a.add_argument("--duree", type=int, default=700, help="durée de chaque image en ms (défaut 700)")
a.add_argument("--max", type=int, default=150, help="nombre maximal d'images (défaut 150)")
a.add_argument("--zone", help="X,Y,L,H : recadrage en pixels de l'image d'origine (défaut : image entière)")
a.add_argument("--sortie", default=os.path.join("out", "diagnostic"), help="dossier du GIF")
a.add_argument("folder")
a.add_argument("--start", required=True, help="first time (UTC)")
a.add_argument("--end", required=True, help="last time (UTC), included")
a.add_argument("--width", type=int, default=720, help="width of the GIF in pixels (default 720)")
a.add_argument("--duration", type=int, default=700, help="duration of each image in ms (default 700)")
a.add_argument("--max", type=int, default=150, help="maximum number of images (default 150)")
a.add_argument("--zone", help="X,Y,W,H: crop in pixels of the original image (default: whole image)")
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the GIF")
o = a.parse_args()
debut, fin = instant(o.debut), instant(o.fin)
toutes = sorted((h, p) for p in trouver(o.dossier)["images"] if (h := heure_image(p)))
choisies = [(h, p) for h, p in toutes if debut <= h <= fin]
if not choisies:
dispo = f" (images du {iso(toutes[0][0])} au {iso(toutes[-1][0])})" if toutes else ""
sys.exit(f"Aucune image entre {iso(debut)} et {iso(fin)} dans {o.dossier}{dispo}.")
n = len(choisies)
start, end = parse_time(o.start), parse_time(o.end)
every = sorted((h, p) for p in find_files(o.folder)["images"] if (h := image_time(p)))
chosen = [(h, p) for h, p in every if start <= h <= end]
if not chosen:
available = f" (images from {iso(every[0][0])} to {iso(every[-1][0])})" if every else ""
sys.exit(f"No image between {iso(start)} and {iso(end)} in {o.folder}{available}.")
n = len(chosen)
if n > o.max:
choisies = [choisies[round(i * (n - 1) / (o.max - 1))] for i in range(o.max)]
print(f"{o.max} images sur {n} gardées")
chosen = [chosen[round(i * (n - 1) / (o.max - 1))] for i in range(o.max)]
print(f"{o.max} of {n} images kept")
vues = []
for h, p in choisies:
frames = []
for h, p in chosen:
im = Image.open(p).convert("RGB")
if o.zone:
x, y, l, h_ = map(int, o.zone.split(","))
im = im.crop((x, y, x + l, y + h_))
im = im.resize((o.largeur, round(im.height * o.largeur / im.width)), Image.LANCZOS)
x, y, w, h_ = map(int, o.zone.split(","))
im = im.crop((x, y, x + w, y + h_))
im = im.resize((o.width, round(im.height * o.width / im.width)), Image.LANCZOS)
d = ImageDraw.Draw(im)
texte = h.strftime("%Y-%m-%d %H:%M UTC")
police = ImageFont.load_default(size=max(12, o.largeur // 45))
g, haut, dr, bas = d.textbbox((0, 0), texte, font=police)
x, y, marge = im.width - (dr - g) - 10, im.height - (bas - haut) - 10, 4
d.rectangle((x - marge, y - marge, x + dr - g + marge, y + bas - haut + marge), fill="black")
d.text((x - g, y - haut), texte, fill="white", font=police)
vues.append(im.quantize(64, dither=Image.Dither.NONE))
text = h.strftime("%Y-%m-%d %H:%M UTC")
font = ImageFont.load_default(size=max(12, o.width // 45))
left, top, right, bottom = d.textbbox((0, 0), text, font=font)
x, y, margin = im.width - (right - left) - 10, im.height - (bottom - top) - 10, 4
d.rectangle((x - margin, y - margin, x + right - left + margin, y + bottom - top + margin), fill="black")
d.text((x - left, y - top), text, fill="white", font=font)
frames.append(im.quantize(64, dither=Image.Dither.NONE))
print(iso(h))
os.makedirs(o.sortie, exist_ok=True)
nom = f"images_{choisies[0][0]:%Y%m%dT%H%MZ}_{choisies[-1][0]:%Y%m%dT%H%MZ}.gif"
chemin = os.path.join(o.sortie, nom)
vues[0].save(chemin, save_all=True, append_images=vues[1:], duration=o.duree, loop=0, optimize=True)
print(f"GIF : {chemin} ({os.path.getsize(chemin) / 1e6:.1f} Mo, {len(vues)} images)")
os.makedirs(o.out, exist_ok=True)
name = f"images_{chosen[0][0]:%Y%m%dT%H%MZ}_{chosen[-1][0]:%Y%m%dT%H%MZ}.gif"
path = os.path.join(o.out, name)
frames[0].save(path, save_all=True, append_images=frames[1:], duration=o.duration, loop=0, optimize=True)
print(f"GIF: {path} ({os.path.getsize(path) / 1e6:.1f} MB, {len(frames)} images)")
if __name__ == "__main__":
-93
View File
@@ -1,93 +0,0 @@
"""Graphique PNG des séries d'un export « advanced », pour que l'agent de diagnostic les regarde.
Usage : .venv/bin/python tools/scripts/graphique.py data/PS_DES_COUDRES/
[--capteurs CAPTEUR_A,CAPTEUR_B] [--colonnes error_value,corr_x,corr_y] [--debut ...] [--fin ...]
[--zone DEBUT/FIN ...] [--sortie DOSSIER]
Un panneau par colonne, une courbe par capteur, heures UTC ; chaque --zone est grisée. Colonnes : error_value, corr_x, corr_y,
relative_x, relative_y, x, y, sigma0, reproj_err, et les six métriques de qualité d'image (peak à snr, valeurs brutes ; face à une référence calme : qualite.py).
Imprime le chemin du PNG (out/diagnostic/, ou --sortie : le dossier du rapport, pour une figure de preuve).
"""
import argparse
import os
import sys
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402
from donnees import charger, est_reference, filtrer, instant # noqa: E402
from resume import QUALITE # noqa: E402
COLONNES = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
def zone(texte):
"""'2026-10-13T01:00Z/2026-10-13T03:00Z' -> (début, fin)."""
debut, sep, fin = texte.partition("/")
if not sep:
raise argparse.ArgumentTypeError(f"zone invalide : {texte!r} (ex. 2026-10-13T01:00Z/2026-10-13T03:00Z)")
return instant(debut), instant(fin)
def borne(t, defaut):
return t.strftime("%Y%m%dT%H%MZ") if t else defaut
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossier")
a.add_argument("--capteurs", help="liste séparée par des virgules (défaut : tous)")
a.add_argument("--colonnes", default="error_value,corr_x,corr_y")
a.add_argument("--debut", type=instant)
a.add_argument("--fin", type=instant)
a.add_argument("--zone", type=zone, action="append", default=[], help="fenêtre à griser, DEBUT/FIN (répétable)")
a.add_argument("--sortie", default=os.path.join("out", "diagnostic"), help="dossier du PNG")
o = a.parse_args()
par = charger(o.dossier)
if not par:
sys.exit(f"Aucun export « advanced » dans {o.dossier} (voir inventaire.py).")
noms = o.capteurs.split(",") if o.capteurs else sorted(par, key=lambda s: (not est_reference(par[s]), s))
colonnes = o.colonnes.split(",")
for demandes, connus, quoi in ((noms, par, "Capteurs"), (colonnes, COLONNES + QUALITE, "Colonnes")):
absents = [x for x in demandes if x not in connus]
if absents:
sys.exit(f"{quoi} inconnus : {', '.join(absents)}. Connus : {', '.join(sorted(connus))}.")
series = {s: filtrer(par[s], o.debut, o.fin) for s in noms}
if not any(series.values()):
sys.exit("Aucune mesure dans la fenêtre demandée.")
vides = [c for c in colonnes if all(m[c] is None for s in noms for m in series[s])]
if len(vides) == len(colonnes):
sys.exit(f"Rien à tracer : {', '.join(colonnes)} vide(s) pour ces capteurs dans la fenêtre.")
if vides:
print(f"Panneaux vides (aucune valeur pour ces capteurs) : {', '.join(vides)}")
fig, axes = plt.subplots(len(colonnes), 1, sharex=True, figsize=(14, 3.2 * len(colonnes)), squeeze=False)
tab20 = plt.get_cmap("tab20").colors
couleurs = tab20[::2] + tab20[1::2] # teintes foncées d'abord : deux capteurs voisins ne partagent pas une teinte
for ax, col in zip(axes[:, 0], colonnes):
for i, s in enumerate(noms):
ms = series[s] # une valeur vide devient NaN : la courbe s'interrompt au lieu de relier le trou
ax.plot([m["t"] for m in ms], [float("nan") if m[col] is None else m[col] for m in ms], ".-", ms=2, lw=0.7,
color=couleurs[i % len(couleurs)], label=s)
for debut, fin in o.zone:
ax.axvspan(debut, fin, color="0.85", zorder=0)
ax.set_ylabel(col)
ax.grid(alpha=0.3)
axes[0, 0].legend(loc="upper left", bbox_to_anchor=(1.0, 1.0), fontsize=8)
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
station = os.path.basename(os.path.normpath(o.dossier))
fig.suptitle(f"{station} : {', '.join(colonnes)} (heures UTC)")
os.makedirs(o.sortie, exist_ok=True)
qui = "-".join(noms) if o.capteurs else "tous"
sortie = os.path.join(o.sortie, f"{station}_{'-'.join(colonnes)}_{qui}_"
f"{borne(o.debut, 'debut')}_{borne(o.fin, 'fin')}.png")
fig.savefig(sortie, dpi=100, bbox_inches="tight")
print(sortie)
if __name__ == "__main__":
main()
-314
View File
@@ -1,314 +0,0 @@
"""Inventaire d'un dossier de station : ce qui est disponible, ce qui manque.
Usage : python3 tools/scripts/inventaire.py data/PS_DES_COUDRES/ [--debut T] [--fin T] [--fichiers]
Les comptes portent sur toute la période. Avec --debut et/ou --fin, une section de plus liste, sur cette fenêtre,
les pas absents (datés, avec les capteurs qui n'ont pas de mesure) et, par capteur, les mesures rejetées (datées,
en suites de pas consécutifs ; rejetée : error_value au-dessus du seuil, références comprises), puis le nombre de
références retenues par la compensation (inlier = 1 ; inlier vide sur la fenêtre : reproj_err rempli) à chaque instant
et, datées, les suites d'instants où des références ne le sont pas (lesquelles, et pourquoi : rejetée, valeur d'inlier,
vide). Mouvements de la caméra (camera/) : nombre de poses, période, et les instants de mesure
sans pose (pas de recalage), sur toute la période puis sur la fenêtre. Avant les comptes, ce que contient chaque source
(sur la fenêtre, sinon toute la période) : les colonnes de l'export, de la pose et de quality/, remplies ou vides (avec,
pour une colonne de l'export que tous les capteurs ne portent pas, combien la portent), les noms des métriques système
de metrics/, les images et les images différentielles. Les relevés en double (même capteur, même instant dans deux
exports, ou deux fois dans un) sont comptés par export. Images : par suite d'écarts égaux entre deux images (à 5 min
près ; sur la fenêtre si elle est donnée). Une suite datée qui finit le même jour n'écrit que l'heure de sa fin.
Rend des valeurs, pas de verdict.
"""
import argparse
import collections
import csv
import glob
import os
import re
import statistics
from datetime import datetime, timedelta, timezone
from donnees import charger, charger_camera, est_reference, filtrer, heure_image, instant, iso, rejetee, trouver
from resume import QUALITE
METRIQUE_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_\d{2}_\d{2}") # NIEDERHORN_BusCurrent_2026_09_22_00_00 to ….csv
def suite(a, b, n):
"""Une suite datée : le début en entier, la fin sans la date quand elle tombe le même jour."""
if n == 1:
return iso(a)
return f"{iso(a)} → {iso(b)[11:] if iso(a)[:10] == iso(b)[:10] else iso(b)} ({n})"
def doubles(chemins):
"""Relevés (même capteur, même instant) déjà lus dans un export précédent ou plus haut dans le même : charger n'en
garde qu'un, celui du dernier fichier dans l'ordre des noms. Une ligne par export qui en a."""
vus, lignes = {}, []
for p in chemins:
n = deja = autres = 0
with open(p, newline="") as fh:
for r in csv.DictReader(fh):
n += 1
cle = (r["sensor_name"].strip(), r["time_utc"])
val = tuple(v.strip() for k, v in r.items() if k not in ("sensor_name", None))
if cle in vus:
deja += 1
autres += vus[cle] != val
vus[cle] = val
if deja:
lignes.append(f" {os.path.basename(p)} : {deja} sur {n} déjà lus, dont {autres} de valeurs différentes")
if lignes:
print("Relevés en double (même capteur, même instant dans un export déjà lu, dans l'ordre des noms ; "
"les outils gardent celui du dernier fichier) :")
print("\n".join(lignes))
def liste(chemins, tout):
"""Un dossier de pompage compte des centaines de fichiers : au-delà de 5, le premier et le dernier seulement."""
if tout or len(chemins) <= 5:
shown = chemins
else:
shown = [chemins[0], chemins[-1]]
print(f" {len(chemins)} fichiers, du premier au dernier (--fichiers pour la liste complète) :")
for p in shown:
print(f" {p}")
def fichiers(f, tout=False):
print(f"Exports « advanced » (source des mesures) : {len(f['advanced'])}")
liste(f["advanced"], tout)
doubles(f["advanced"])
print(f"Exports horaires : {len(f['horaire'])} (non lus par les outils : moyennes horaires, "
"heure locale sans fuseau ; voir labeling/connaissances/colonnes-des-deplacements.md)")
liste(f["horaire"], tout)
print(f"Mouvements de la caméra (pose à chaque recalage) : {len(f['camera'])}")
liste(f["camera"], tout)
print(f"Autres fichiers : {len(f['autres'])}")
liste(f["autres"], tout)
def remplies(chemins, debut, fin):
"""CSV d'une source -> (lignes de la fenêtre, {colonne: lignes avec une valeur}, {colonne: capteurs qui en ont une}),
colonnes dans l'ordre des en-têtes ; capteurs : sensor_name, s'il y en a."""
lignes, compte, qui = 0, {}, collections.defaultdict(set)
for p in chemins:
with open(p, newline="") as fh:
r = csv.DictReader(fh)
for c in r.fieldnames or []:
if c not in ("", "time_utc", "sensor_name"):
compte.setdefault(c, 0)
for row in r:
if (debut or fin) and filtrer([{"t": instant(row["time_utc"])}], debut, fin) == []:
continue
lignes += 1
for c in compte:
if (row.get(c) or "").strip():
compte[c] += 1
if row.get("sensor_name"):
qui[c].add(row["sensor_name"].strip())
return lignes, compte, qui
def contenu(dossier, f, debut=None, fin=None):
"""Ce que contient chaque source sur la fenêtre : ses colonnes, remplies ou vides, et les noms des métriques.
Des noms et des comptes, jamais de valeurs (metrics/ porte des identifiants : SIM_CCID, ip_*)."""
quoi = "sur la fenêtre" if debut or fin else "sur toute la période"
print(f"\nCe que contiennent les données, {quoi} (par colonne, les lignes qui ont une valeur ; « vide » : aucune) :")
autres = collections.defaultdict(list)
for p in f["autres"]:
autres[os.path.relpath(p, dossier).split(os.sep)[0]].append(p)
for titre, chemins in (("Export « advanced »", f["advanced"]), ("Pose de la caméra (camera/)", f["camera"]),
("Qualité d'image (quality/)", autres.pop("quality", []))):
if chemins:
n, compte, qui = remplies(chemins, debut, fin)
tous = set().union(*qui.values())
def porte(c): # nombre de capteurs qui portent la colonne, quand ce ne sont pas tous
return f" ({len(qui[c])} capteurs sur {len(tous)})" if tous and len(qui[c]) < len(tous) else ""
print(f" {titre} : {n} lignes ; " + ", ".join(f"{c} {k}{porte(c)}" if k else f"{c} vide" for c, k in compte.items()))
if chemins is f["advanced"]:
q = [c for c in QUALITE if compte.get(c)]
print(f" Qualité d'image de l'export (peak à snr) : " + (f"remplie ({', '.join(q)}) ; lue par qualite.py, "
"labeling/connaissances/metriques-de-qualite.md" if q else "vide"))
jours = collections.Counter() # métrique -> jours (un CSV par métrique et par jour) dans la fenêtre
for p in autres.pop("metrics", []):
m = METRIQUE_RE.match(os.path.basename(p))
if not m:
continue
jour = datetime(*map(int, m.group(2, 3, 4)), tzinfo=timezone.utc)
if (debut is None or jour.date() >= debut.date()) and (fin is None or jour.date() <= fin.date()):
nom = m.group(1)
jours[nom.split("_", 1)[1] if "_" in nom else nom] += 1
if jours:
print(f" Métriques système (metrics/, un CSV par métrique et par jour ; lues par metriques.py) : {len(jours)} : "
+ ", ".join(f"{k} ({n} j)" for k, n in sorted(jours.items())))
for titre, chemins in (("Images de la caméra", f["images"]),
("Images différentielles (images_diff/ ; aucun outil ne les lit)", glob.glob(os.path.join(dossier, "images_diff", "*")))):
heures = [h for h in map(heure_image, chemins) if h and filtrer([{"t": h}], debut, fin)]
if chemins:
print(f" {titre} : {len(heures)}")
if autres:
print(" Autres : " + ", ".join(f"{k} ({len(v)})" for k, v in sorted(autres.items())))
def images(chemins, debut=None, fin=None):
heures = sorted(h for h in map(heure_image, chemins) if h)
print(f"Images : {len(chemins)}" + (f", du {iso(heures[0])} au {iso(heures[-1])}" if heures else ""))
inconnues = [p for p in chemins if heure_image(p) is None]
if inconnues:
print(f" noms sans heure reconnue : {len(inconnues)}")
heures = [h for h in heures if filtrer([{"t": h}], debut, fin)]
if len(heures) < 2:
return
# Suites d'images à écart régulier (arrondi à 5 min, ce qui absorbe les quelques secondes de retard d'une prise) :
# une ligne par changement d'écart, sans rien perdre des heures couvertes.
suites = [] # [première, dernière, écart en min, nombre d'écarts]
for a, b in zip(heures, heures[1:]):
e = round((b - a).total_seconds() / 300) * 5
if suites and suites[-1][2] == e:
suites[-1][1], suites[-1][3] = b, suites[-1][3] + 1
else:
suites.append([a, b, e, 1])
quoi = "sur la fenêtre" if debut or fin else "entre la première et la dernière"
print(f" écart entre deux images {quoi}, par suite d'écarts égaux (à 5 min près) :")
for a, b, e, n in suites:
print(f" {iso(a)} → {iso(b)} : " + (f"{n + 1} images, toutes les {e} min" if n > 1 else f"un écart de {e} min"))
def mesures(par):
if not par:
print("\nAucune mesure : pas d'export « advanced » dans ce dossier.")
return
instants = sorted({m["t"] for ms in par.values() for m in ms})
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
print(f"\nPériode des mesures : {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants")
print(f"Cadence (pas médian) : {cadence.total_seconds() / 60:.0f} min")
trous = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(instants, instants[1:])]
trous = [x for x in trous if x[2] > 0]
print(f"Mesures manquantes (instants absents de la cadence) : {sum(n for *_, n in trous)}")
for a, b, n in trous:
print(f" entre {iso(a)} et {iso(b)} : {n}")
noms = sorted(par, key=lambda s: (not est_reference(par[s]), s))
refs = [s for s in noms if est_reference(par[s])]
print(f"\nCapteurs : {len(par)} ({len(refs)} références, {len(par) - len(refs)} cibles)")
vides = [s for s in noms if not any(m["pattern_type"] for m in par[s])]
if vides:
print(f" pattern_type vide pour {len(vides)} capteurs : référence quand reproj_err ou inlier est rempli "
"(références seulement, labeling/connaissances/colonnes-des-deplacements.md)")
print(f"{'capteur':10} {'type':10} {'motif':11} {'seuil':>5} {'mesures':>7} {'rejetées':>8}")
for s in noms:
ms = par[s]
seuil = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms}))
print(f"{s:10} {'référence' if est_reference(ms) else 'cible':10} {ms[0]['pattern_type']:11} "
f"{seuil:>5} {len(ms):7} {sum(map(rejetee, ms)):8}")
def camera(dossier, par, debut=None, fin=None):
"""Poses de la caméra, et les instants de mesure sans pose (la caméra n'a pas été recalée)."""
poses = filtrer(charger_camera(dossier), debut, fin)
instants = sorted({m["t"] for ms in par.values() for m in filtrer(ms, debut, fin)})
quoi = "dans la fenêtre" if debut or fin else "sur toute la période"
if not poses:
print(f"\nPose de la caméra {quoi} : aucune")
return
avec = {p["t"].replace(second=0, microsecond=0) for p in poses}
print(f"\nPose de la caméra {quoi} : {len(poses)} recalages, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}")
suites = [] # [début, fin, nombre, rang] des instants de mesure consécutifs sans pose
for i, t in enumerate(instants):
if t.replace(second=0, microsecond=0) in avec:
continue
if suites and suites[-1][3] == i - 1:
suites[-1][1], suites[-1][2], suites[-1][3] = t, suites[-1][2] + 1, i
else:
suites.append([t, t, 1, i])
print(f"Instants de mesure sans pose (pas de recalage) : {sum(x[2] for x in suites)} sur {len(instants)}")
if debut or fin:
for a, b, n, _ in suites:
print(f" {suite(a, b, n)}")
def fenetre(par, debut, fin):
"""Pas absents et rejets datés sur la fenêtre debut/fin."""
instants = sorted({m["t"] for ms in par.values() for m in ms})
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
series = {s: {m["t"]: m for m in filtrer(ms, debut, fin)} for s, ms in par.items()}
dans = sorted({t for d in series.values() for t in d})
print(f"\nFenêtre {iso(dans[0])} → {iso(dans[-1])} : {len(dans)} instants présents")
noms = sorted(par, key=lambda s: (not est_reference(par[s]), s))
print(f"Pas absents pour tous les capteurs, entre deux instants présents (cadence de {cadence.total_seconds() / 60:.0f} min) :")
trous = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(dans, dans[1:])]
trous = [x for x in trous if x[2] > 0]
for a, b, n in trous:
print(f" entre {iso(a)} et {iso(b)} : {n}")
if not trous:
print(" (aucun)")
print("Instants présents où des capteurs n'ont pas de mesure :")
partiels = [(t, [s for s in noms if t not in series[s]]) for t in dans]
for t, manque in partiels:
if manque:
print(f" {iso(t)} : {len(manque)} capteurs : {','.join(manque)}")
if not any(m for _, m in partiels):
print(" (aucun)")
print("Mesures rejetées dans la fenêtre, par capteur (début → fin de chaque suite de pas consécutifs, nombre) :")
for s in noms:
rej = [t for t in sorted(series[s]) if rejetee(series[s][t])]
suites = []
for t in rej:
if suites and t - suites[-1][1] <= cadence * 1.5:
suites[-1][1], suites[-1][2] = t, suites[-1][2] + 1
else:
suites.append([t, t, 1])
if rej:
print(f" {s} ({len(rej)}) : " + "; ".join(suite(a, b, n) for a, b, n in suites))
if not any(rejetee(m) for d in series.values() for m in d.values()):
print(" (aucune)")
refs = [s for s in noms if est_reference(par[s])]
# inlier dit quelles références la compensation retient ; certains exports ne le remplissent pas (et parfois
# reproj_err seulement) : on prend alors reproj_err rempli, et le titre le dit.
col = next((c for c in ("inlier", "reproj_err") if any(m[c] is not None for s in refs for m in series[s].values())), None)
if col is None:
print("Références retenues par la compensation : colonnes inlier et reproj_err absentes ou vides.")
return
retenue = (lambda m: m["inlier"] == 1) if col == "inlier" else (lambda m: m["reproj_err"] is not None)
def marque(s, t):
m = series[s].get(t)
return "absente" if m is None else "r" if rejetee(m) else "vide" if m[col] is None else f"{m[col]:g}"
hors = [(t, tuple((s, marque(s, t)) for s in refs if t not in series[s] or not retenue(series[s][t]))) for t in dans]
compte = collections.Counter(len(refs) - len(h) for _, h in hors)
quoi = "inlier = 1" if col == "inlier" else "inlier vide sur la fenêtre : reproj_err rempli à la place"
print(f"Références retenues par la compensation ({quoi}) sur {len(refs)}, nombre d'instants : "
+ ", ".join(f"{k} : {compte[k]}" for k in sorted(compte, reverse=True)))
print("Instants où des références ne sont pas retenues (suites de pas consécutifs avec les mêmes références hors ; "
f"r = rejetée, vide = {col} vide, sinon la valeur d'inlier) :")
suites = []
for i, (t, h) in enumerate(hors):
if h and suites and suites[-1][3] == h and suites[-1][4] == i - 1:
suites[-1][1], suites[-1][2], suites[-1][4] = t, suites[-1][2] + 1, i
elif h:
suites.append([t, t, 1, h, i])
for a, b, n, h, _ in suites:
print(f" {suite(a, b, n)} : {len(refs) - len(h)} retenues ; hors : "
+ ", ".join(f"{s} ({q})" for s, q in h))
if not suites:
print(" (aucun)")
def main(dossier, debut=None, fin=None, tout=False):
print(f"Dossier : {dossier}")
f = trouver(dossier)
fichiers(f, tout)
contenu(dossier, f, debut, fin)
images(f["images"], debut, fin)
par = charger(dossier)
mesures(par)
if par:
camera(dossier, par)
if par and (debut or fin):
fenetre(par, debut, fin)
camera(dossier, par, debut, fin)
if __name__ == "__main__":
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossier")
a.add_argument("--debut", type=instant)
a.add_argument("--fin", type=instant)
a.add_argument("--fichiers", action="store_true", help="liste complète des fichiers (sinon : le premier et le dernier au-delà de 5)")
o = a.parse_args()
main(o.dossier, o.debut, o.fin, o.fichiers)
+314
View File
@@ -0,0 +1,314 @@
"""Inventory of a station folder: what is available, what is missing.
Usage: python3 tools/scripts/inventory.py data/PS_DES_COUDRES/ [--start T] [--end T] [--files]
The counts cover the whole period. With --start and/or --end, one more section lists, over this window,
the absent steps (dated, with the sensors that have no measurement) and, per sensor, the rejected measurements (dated,
in runs of consecutive steps; rejected: error_value above the threshold, references included), then the number of
references retained by the compensation (inlier = 1; inlier empty over the window: reproj_err filled) at each instant
and, dated, the runs of instants where references are not (which ones, and why: rejected, inlier value,
empty). Camera movements (camera/): number of poses, period, and the measurement instants
without a pose (no registration), over the whole period then over the window. Before the counts, what each source
contains (over the window, otherwise the whole period): the columns of the export, of the pose and of quality/, filled or empty (with,
for an export column that not all sensors carry, how many carry it), the names of the system metrics
of metrics/, the images and the differential images. Duplicate readings (same sensor, same instant in two
exports, or twice in one) are counted per export. Images: per run of equal gaps between two images (within 5 min;
over the window if it is given). A dated run that ends on the same day writes only the time of its end.
Gives values, not a verdict.
"""
import argparse
import collections
import csv
import glob
import os
import re
import statistics
from datetime import datetime, timedelta, timezone
from data import filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected
from summary import QUALITY
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_\d{2}_\d{2}") # NIEDERHORN_BusCurrent_2026_09_22_00_00 to ….csv
def span(a, b, n):
"""A dated run: the start in full, the end without the date when it falls on the same day."""
if n == 1:
return iso(a)
return f"{iso(a)} → {iso(b)[11:] if iso(a)[:10] == iso(b)[:10] else iso(b)} ({n})"
def duplicates(paths):
"""Readings (same sensor, same instant) already read in a previous export or higher up in the same one: load
keeps only one, that of the last file in name order. One line per export that has some."""
seen, lines = {}, []
for p in paths:
n = already = different = 0
with open(p, newline="") as fh:
for r in csv.DictReader(fh):
n += 1
key = (r["sensor_name"].strip(), r["time_utc"])
val = tuple(v.strip() for k, v in r.items() if k not in ("sensor_name", None))
if key in seen:
already += 1
different += seen[key] != val
seen[key] = val
if already:
lines.append(f" {os.path.basename(p)}: {already} of {n} already read, {different} of them with different values")
if lines:
print("Duplicate readings (same sensor, same instant in an export already read, in name order; "
"the tools keep the one of the last file):")
print("\n".join(lines))
def list_paths(paths, full):
"""A pumping folder holds hundreds of files: beyond 5, the first and the last only."""
if full or len(paths) <= 5:
shown = paths
else:
shown = [paths[0], paths[-1]]
print(f" {len(paths)} files, first and last (--files for the full list):")
for p in shown:
print(f" {p}")
def files(f, full=False):
print(f'"advanced" exports (source of the measurements): {len(f["advanced"])}')
list_paths(f["advanced"], full)
duplicates(f["advanced"])
print(f"Hourly exports: {len(f['hourly'])} (not read by the tools: hourly averages, "
"local time without a zone; see labeling/knowledge/displacement-columns.md)")
list_paths(f["hourly"], full)
print(f"Camera movements (pose at each registration): {len(f['camera'])}")
list_paths(f["camera"], full)
print(f"Other files: {len(f['others'])}")
list_paths(f["others"], full)
def filled(paths, start, end):
"""CSVs of a source -> (rows of the window, {column: rows with a value}, {column: sensors that have one}),
columns in header order; sensors: sensor_name, if there is one."""
rows, count, who = 0, {}, collections.defaultdict(set)
for p in paths:
with open(p, newline="") as fh:
r = csv.DictReader(fh)
for c in r.fieldnames or []:
if c not in ("", "time_utc", "sensor_name"):
count.setdefault(c, 0)
for row in r:
if (start or end) and filter_window([{"t": parse_time(row["time_utc"])}], start, end) == []:
continue
rows += 1
for c in count:
if (row.get(c) or "").strip():
count[c] += 1
if row.get("sensor_name"):
who[c].add(row["sensor_name"].strip())
return rows, count, who
def contents(folder, f, start=None, end=None):
"""What each source contains over the window: its columns, filled or empty, and the names of the metrics.
Names and counts, never values (metrics/ carries identifiers: SIM_CCID, ip_*)."""
what = "over the window" if start or end else "over the whole period"
print(f'\nWhat the data contain, {what} (per column, the rows that have a value; "empty": none):')
others = collections.defaultdict(list)
for p in f["others"]:
others[os.path.relpath(p, folder).split(os.sep)[0]].append(p)
for title, paths in (('"advanced" export', f["advanced"]), ("Camera pose (camera/)", f["camera"]),
("Image quality (quality/)", others.pop("quality", []))):
if paths:
n, count, who = filled(paths, start, end)
everyone = set().union(*who.values())
def carriers(c): # number of sensors that carry the column, when it is not all of them
return f" ({len(who[c])} sensors of {len(everyone)})" if everyone and len(who[c]) < len(everyone) else ""
print(f" {title}: {n} rows; " + ", ".join(f"{c} {k}{carriers(c)}" if k else f"{c} empty" for c, k in count.items()))
if paths is f["advanced"]:
q = [c for c in QUALITY if count.get(c)]
print(f" Export image quality (peak to snr): " + (f"filled ({', '.join(q)}); read by quality.py, "
"labeling/knowledge/quality-metrics.md" if q else "empty"))
days = collections.Counter() # metric -> days (one CSV per metric and per day) in the window
for p in others.pop("metrics", []):
m = METRIC_RE.match(os.path.basename(p))
if not m:
continue
day = datetime(*map(int, m.group(2, 3, 4)), tzinfo=timezone.utc)
if (start is None or day.date() >= start.date()) and (end is None or day.date() <= end.date()):
name = m.group(1)
days[name.split("_", 1)[1] if "_" in name else name] += 1
if days:
print(f" System metrics (metrics/, one CSV per metric and per day; read by metrics.py): {len(days)}: "
+ ", ".join(f"{k} ({n} d)" for k, n in sorted(days.items())))
for title, paths in (("Camera images", f["images"]),
("Differential images (images_diff/; no tool reads them)", glob.glob(os.path.join(folder, "images_diff", "*")))):
times = [h for h in map(image_time, paths) if h and filter_window([{"t": h}], start, end)]
if paths:
print(f" {title}: {len(times)}")
if others:
print(" Others: " + ", ".join(f"{k} ({len(v)})" for k, v in sorted(others.items())))
def images(paths, start=None, end=None):
times = sorted(h for h in map(image_time, paths) if h)
print(f"Images: {len(paths)}" + (f", from {iso(times[0])} to {iso(times[-1])}" if times else ""))
unknown = [p for p in paths if image_time(p) is None]
if unknown:
print(f" names without a recognised time: {len(unknown)}")
times = [h for h in times if filter_window([{"t": h}], start, end)]
if len(times) < 2:
return
# Runs of images at a regular gap (rounded to 5 min, which absorbs the few seconds of delay of a shot):
# one line per change of gap, without losing any of the times covered.
runs = [] # [first, last, gap in min, number of gaps]
for a, b in zip(times, times[1:]):
e = round((b - a).total_seconds() / 300) * 5
if runs and runs[-1][2] == e:
runs[-1][1], runs[-1][3] = b, runs[-1][3] + 1
else:
runs.append([a, b, e, 1])
what = "over the window" if start or end else "between the first and the last"
print(f" gap between two images {what}, per run of equal gaps (within 5 min):")
for a, b, e, n in runs:
print(f" {iso(a)} → {iso(b)}: " + (f"{n + 1} images, every {e} min" if n > 1 else f"one gap of {e} min"))
def measurements(by):
if not by:
print('\nNo measurement: no "advanced" export in this folder.')
return
instants = sorted({m["t"] for ms in by.values() for m in ms})
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
print(f"\nMeasurement period: {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants")
print(f"Cadence (median step): {cadence.total_seconds() / 60:.0f} min")
gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(instants, instants[1:])]
gaps = [x for x in gaps if x[2] > 0]
print(f"Missing measurements (instants absent from the cadence): {sum(n for *_, n in gaps)}")
for a, b, n in gaps:
print(f" between {iso(a)} and {iso(b)}: {n}")
names = sorted(by, key=lambda s: (not is_reference(by[s]), s))
refs = [s for s in names if is_reference(by[s])]
print(f"\nSensors: {len(by)} ({len(refs)} references, {len(by) - len(refs)} targets)")
empty = [s for s in names if not any(m["pattern_type"] for m in by[s])]
if empty:
print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled "
"(references only, labeling/knowledge/displacement-columns.md)")
print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}")
for s in names:
ms = by[s]
threshold = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms}))
print(f"{s:10} {'reference' if is_reference(ms) else 'target':10} {ms[0]['pattern_type']:11} "
f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8}")
def camera(folder, by, start=None, end=None):
"""Camera poses, and the measurement instants without a pose (the camera was not registered)."""
poses = filter_window(load_camera(folder), start, end)
instants = sorted({m["t"] for ms in by.values() for m in filter_window(ms, start, end)})
what = "in the window" if start or end else "over the whole period"
if not poses:
print(f"\nCamera pose {what}: none")
return
with_pose = {p["t"].replace(second=0, microsecond=0) for p in poses}
print(f"\nCamera pose {what}: {len(poses)} registrations, {iso(poses[0]['t'])} → {iso(poses[-1]['t'])}")
runs = [] # [start, end, count, rank] of the consecutive measurement instants without a pose
for i, t in enumerate(instants):
if t.replace(second=0, microsecond=0) in with_pose:
continue
if runs and runs[-1][3] == i - 1:
runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i
else:
runs.append([t, t, 1, i])
print(f"Measurement instants without a pose (no registration): {sum(x[2] for x in runs)} of {len(instants)}")
if start or end:
for a, b, n, _ in runs:
print(f" {span(a, b, n)}")
def window(by, start, end):
"""Absent steps and rejections dated over the start/end window."""
instants = sorted({m["t"] for ms in by.values() for m in ms})
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
series = {s: {m["t"]: m for m in filter_window(ms, start, end)} for s, ms in by.items()}
inside = sorted({t for d in series.values() for t in d})
print(f"\nWindow {iso(inside[0])} → {iso(inside[-1])}: {len(inside)} instants present")
names = sorted(by, key=lambda s: (not is_reference(by[s]), s))
print(f"Steps absent for all sensors, between two present instants (cadence of {cadence.total_seconds() / 60:.0f} min):")
gaps = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(inside, inside[1:])]
gaps = [x for x in gaps if x[2] > 0]
for a, b, n in gaps:
print(f" between {iso(a)} and {iso(b)}: {n}")
if not gaps:
print(" (none)")
print("Present instants where sensors have no measurement:")
partial = [(t, [s for s in names if t not in series[s]]) for t in inside]
for t, lacking in partial:
if lacking:
print(f" {iso(t)}: {len(lacking)} sensors: {','.join(lacking)}")
if not any(m for _, m in partial):
print(" (none)")
print("Rejected measurements in the window, per sensor (start → end of each run of consecutive steps, count):")
for s in names:
rej = [t for t in sorted(series[s]) if rejected(series[s][t])]
runs = []
for t in rej:
if runs and t - runs[-1][1] <= cadence * 1.5:
runs[-1][1], runs[-1][2] = t, runs[-1][2] + 1
else:
runs.append([t, t, 1])
if rej:
print(f" {s} ({len(rej)}): " + "; ".join(span(a, b, n) for a, b, n in runs))
if not any(rejected(m) for d in series.values() for m in d.values()):
print(" (none)")
refs = [s for s in names if is_reference(by[s])]
# inlier says which references the compensation retains; some exports do not fill it (and sometimes
# only reproj_err): filled reproj_err is taken then, and the title says so.
col = next((c for c in ("inlier", "reproj_err") if any(m[c] is not None for s in refs for m in series[s].values())), None)
if col is None:
print("Retained references by the compensation: columns inlier and reproj_err absent or empty.")
return
retained = (lambda m: m["inlier"] == 1) if col == "inlier" else (lambda m: m["reproj_err"] is not None)
def mark(s, t):
m = series[s].get(t)
return "absent" if m is None else "r" if rejected(m) else "empty" if m[col] is None else f"{m[col]:g}"
out = [(t, tuple((s, mark(s, t)) for s in refs if t not in series[s] or not retained(series[s][t]))) for t in inside]
count = collections.Counter(len(refs) - len(h) for _, h in out)
what = "inlier = 1" if col == "inlier" else "inlier empty over the window: reproj_err filled instead"
print(f"Retained references by the compensation ({what}) out of {len(refs)}, number of instants: "
+ ", ".join(f"{k}: {count[k]}" for k in sorted(count, reverse=True)))
print("Instants where references are not retained (runs of consecutive steps with the same references out; "
f"r = rejected, empty = {col} empty, otherwise the inlier value):")
runs = []
for i, (t, h) in enumerate(out):
if h and runs and runs[-1][3] == h and runs[-1][4] == i - 1:
runs[-1][1], runs[-1][2], runs[-1][4] = t, runs[-1][2] + 1, i
elif h:
runs.append([t, t, 1, h, i])
for a, b, n, h, _ in runs:
print(f" {span(a, b, n)}: {len(refs) - len(h)} retained; out: "
+ ", ".join(f"{s} ({q})" for s, q in h))
if not runs:
print(" (none)")
def main(folder, start=None, end=None, full=False):
print(f"Folder: {folder}")
f = find_files(folder)
files(f, full)
contents(folder, f, start, end)
images(f["images"], start, end)
by = load(folder)
measurements(by)
if by:
camera(folder, by)
if by and (start or end):
window(by, start, end)
camera(folder, by, start, end)
if __name__ == "__main__":
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder")
a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time)
a.add_argument("--files", action="store_true", help="full list of the files (otherwise: the first and the last beyond 5)")
o = a.parse_args()
main(o.folder, o.start, o.end, o.files)
-333
View File
@@ -1,333 +0,0 @@
"""Ce qui revient d'un jour à l'autre : niveau de chaque nuit, et jours superposés par heure du jour, pour l'agent de diagnostic.
Usage : .venv/bin/python tools/scripts/jours.py data/PS_DES_COUDRES/ [--colonnes corr_x,corr_y,error_value]
[--nuit 20-3] [--k 5] [--jours 2] [--pas 15min] [--capteurs A,B] [--bornes [--lissage N] [--fraction F]]
[--debut ...] [--fin ...] [--sortie DOSSIER]
Mesures acceptées seulement : une mesure rejetée (error_value au-dessus du seuil) n'entre nulle part. Une « nuit » :
les heures --nuit H1-H2 (UTC, H1 inclus, H2 exclu, à cheval sur minuit si H1 > H2), datée du jour où elle commence ;
la nuit en cours au début de la fenêtre compte aussi (ses heures après minuit). Bruit propre du capteur : médiane des
écarts entre deux mesures consécutives sur la fenêtre. Pour chaque colonne de --colonnes :
1. Niveau de chaque nuit, par capteur : médiane, MAD et nombre de mesures ; puis, d'une nuit à la suivante, la
marche (différence des médianes, en unités et en bruits) et où elle se place : la suite des mesures de nuit des
deux nuits est coupée là où la somme des écarts absolus au niveau de la première nuit (avant) et de la seconde
(après) est la plus petite ; « entre A et B » : dernière mesure de nuit avant la coupure et première après (B en
début de nuit : la coupure tombe entre deux nuits, de jour).
2. Jours superposés : chaque mesure moins le niveau de la dernière nuit commencée à son instant (la précédente, ou
celle en cours ; sans elle, la nuit la plus proche). PNG « parheure-<colonne> » : un petit graphique par capteur,
en abscisse l'heure du jour (UTC), une courbe par jour, heures de nuit grisées, une échelle par capteur, cadrée du 1er au 99e centile élargi de moitié (le titre dit
combien de mesures sortent du cadre : un pic isolé n'écrase pas l'échelle ; sa valeur est dans les tables).
Table : par créneau de --pas de l'heure du jour, les capteurs dont l'écart dépasse --k bruits à ce créneau, du
même côté, au moins --jours jours différents, avec le nombre de ces jours sur le nombre de jours mesurés et la
médiane de ces écarts (signée, en unités et en bruits) ; un capteur au-dessus certains jours et au-dessous
d'autres est cité deux fois.
3. Avec --bornes : chaque jour face à la nuit qui le précède, pour la médiane des cibles de chaque colonne (avec
--capteurs : la médiane de ces capteurs, références comprises ; un seul capteur : le sien) et pour la pose de la
caméra (pan, tilt, rotation, scale, sigma0) si camera/ existe. Un « jour » : de la fin d'une nuit à la fin de la
suivante (24 h), daté du jour où il commence ; son niveau de référence est celui de la nuit qui vient de finir (sans
elle, la dernière nuit mesurée avant, ou au début du lot la première : nommée, « pas la précédente »). Écart de chaque mesure à ce niveau (médiane des cibles : à chaque
instant, médiane des écarts de chaque cible à son propre niveau de cette nuit) ; bruit : médiane des écarts entre
deux valeurs consécutives de cette suite. Par jour : nombre de mesures, première et dernière ; premier et dernier
instant où |écart| dépasse --k bruits, avec la mesure juste avant et juste après (au pas natif) ; le maximum et le
minimum signés et leur instant ; autour du plus grand des deux en valeur absolue, la suite de mesures consécutives
au-delà de --k bruits du même côté qui le contient, avec la mesure juste avant et juste après ; la nuit suivante
moins la nuit de référence. Un jour sans mesure le dit. Chaque ligne de jour commence par le nom court de la suite
(la colonne, ou pose_tilt…) : `| grep '^pose_tilt '` en garde une, `| grep ' 12/10 '` un jour. Avec --lissage N,
l'écart de chaque mesure est la médiane des N mesures du jour centrées sur elle (le bruit reste celui des mesures,
la figure trace aussi l'écart brut en gris) : un point isolé ne coupe plus la suite autour de l'extrême et ne fait
plus un premier ou un dernier dépassement. Avec --fraction F, le seuil de chaque jour est F fois le plus grand
|écart| de ce jour (au lieu de --k bruits ; il est imprimé sur la ligne du jour) : sur une suite très bruitée, ou
dont le bruit au pas natif est sans rapport avec l'ampleur du cycle. PNG
« bornes » : un panneau par suite, l'écart sur le lot, ±k bruits en pointillé, de chaque jour [premier, dernier]
grisé clair et la suite autour de l'extrême grisée foncé, le maximum et le minimum marqués ; cadré du 1er au 99e
centile élargi de moitié (le titre dit combien de mesures sortent du cadre).
Rend des valeurs, pas de verdict.
"""
import argparse
import math
import os
import statistics
import sys
from datetime import datetime, timedelta, timezone
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from donnees import POSE, charger, charger_camera, est_reference, filtrer, instant, iso, rejetee # noqa: E402
from graphique import COLONNES, borne # noqa: E402
from resume import ARRONDI, bruit_propre, duree # noqa: E402
def heures(texte):
"""'20-3' -> (20, 3)."""
try:
h1, h2 = (int(x) for x in texte.split("-"))
except ValueError:
raise argparse.ArgumentTypeError(f"heures invalides : {texte!r} (ex. 20-3)")
return h1, h2
def de_nuit(t, h1, h2):
return (h1 <= t.hour or t.hour < h2) if h1 > h2 else h1 <= t.hour < h2
def coupure(pts, la, lb):
"""Indice j (1..n-1) où sum|v-la| avant j + sum|v-lb| à partir de j est le plus petit."""
avant, apres = 0.0, sum(abs(v - lb) for _, v in pts)
meilleur = None
for j in range(1, len(pts)):
v = pts[j - 1][1]
avant += abs(v - la)
apres -= abs(v - lb)
if meilleur is None or avant + apres < meilleur[0]:
meilleur = (avant + apres, j)
return meilleur[1]
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossier")
a.add_argument("--colonnes", default="corr_x,corr_y,error_value")
a.add_argument("--nuit", default="20-3", type=heures, help="heures de nuit H1-H2, UTC (défaut 20-3)")
a.add_argument("--k", type=float, default=5, help="écart au niveau de la nuit compté, en bruits propres du capteur")
a.add_argument("--jours", type=int, default=2, help="nombre de jours à partir duquel un capteur est cité à un créneau")
a.add_argument("--pas", default="15min", type=duree, help="largeur des créneaux de l'heure du jour (défaut 15min)")
a.add_argument("--capteurs", help="liste séparée par des virgules (défaut : tous)")
a.add_argument("--bornes", action="store_true", help="ajoute, jour par jour, les bornes de l'écart au niveau de la nuit précédente")
a.add_argument("--lissage", type=int, default=1, metavar="N", help="--bornes : écart lissé par la médiane de N mesures centrées (défaut 1 : aucun)")
a.add_argument("--fraction", type=float, metavar="F", help="--bornes : seuil de chaque jour = F fois son plus grand |écart| (au lieu de --k bruits)")
a.add_argument("--debut", type=instant)
a.add_argument("--fin", type=instant)
a.add_argument("--sortie", default=os.path.join("out", "diagnostic"), help="dossier des PNG")
o = a.parse_args()
par = charger(o.dossier)
if not par:
sys.exit(f"Aucun export « advanced » dans {o.dossier} (voir inventaire.py).")
noms = o.capteurs.split(",") if o.capteurs else sorted(par, key=lambda s: (not est_reference(par[s]), s))
colonnes = o.colonnes.split(",")
for demandes, connus, quoi in ((noms, par, "Capteurs"), (colonnes, COLONNES, "Colonnes")):
absents = [x for x in demandes if x not in connus]
if absents:
sys.exit(f"{quoi} inconnus : {', '.join(absents)}. Connus : {', '.join(sorted(connus))}.")
series = {s: [m for m in filtrer(par[s], o.debut, o.fin) if not rejetee(m)] for s in noms}
if not any(series.values()):
sys.exit("Aucune mesure acceptée dans la fenêtre demandée.")
h1, h2 = o.nuit
pas = int(o.pas.total_seconds() // 60)
station = os.path.basename(os.path.normpath(o.dossier))
os.makedirs(o.sortie, exist_ok=True)
lignes = math.ceil(len(noms) / 5)
for col in colonnes:
print(f"\n== {col} : niveau de chaque nuit ({h1} h → {h2} h UTC), médiane ±MAD (mesures) ; marches d'une nuit à la suivante ==")
creneaux = {} # minute du jour -> {(capteur, au-dessus): {jour: écart}}
bruits = {}
fig, axes = plt.subplots(lignes, 5, sharex=True, figsize=(18, 2.5 * lignes), squeeze=False)
for ax, s in zip(axes.flat, noms):
ms = [(m["t"], m[col]) for m in series[s] if m[col] is not None]
if len(ms) < 2:
print(f"{s:9}(moins de 2 valeurs acceptées)")
ax.axis("off")
continue
bruit = bruits[s] = bruit_propre([v for _, v in ms])
nuits = {}
for t, v in ms:
if de_nuit(t, h1, h2):
nuits.setdefault((t - timedelta(hours=h1)).date(), []).append((t, v))
niv = {d: statistics.median(v for _, v in p) for d, p in nuits.items()}
cases = []
for d in sorted(nuits):
vs = [v for _, v in nuits[d]]
cases.append(f"{d:%d/%m} {niv[d]:+.6g} ±{statistics.median(abs(v - niv[d]) for v in vs):.2g} ({len(vs)})")
print(f"{s:9}bruit {bruit:.3g} | " + (" | ".join(cases) if cases else "aucune mesure de nuit"))
jours = sorted(nuits)
for da, db in zip(jours, jours[1:]):
pts = nuits[da] + nuits[db]
j = coupure(pts, niv[da], niv[db])
d = niv[db] - niv[da]
enb = f"{d / bruit:+.1f}" if bruit else "-"
print(f"{'':9}{da:%d/%m} → {db:%d/%m} : {d:+.4g} ({enb} bruits), entre {iso(pts[j - 1][0])} et {iso(pts[j][0])}")
def niveau_de(t):
d = (t - timedelta(hours=h1)).date()
if d in niv:
return niv[d]
return niv[min(niv, key=lambda x: abs((x - d).days))] if niv else statistics.median(v for _, v in ms)
par_jour = {}
for t, v in ms:
e = v - niveau_de(t)
par_jour.setdefault(t.date(), []).append((t.hour + t.minute / 60, e))
if bruit and abs(e) > o.k * bruit:
c = (t.hour * 60 + t.minute) // pas * pas
creneaux.setdefault(c, {}).setdefault((s, e > 0), {})[t.date()] = e
for i, (d, p) in enumerate(sorted(par_jour.items())):
ax.plot([x for x, _ in p], [e for _, e in p], "-", lw=0.8, color=f"C{i % 10}", label=f"{d:%d/%m}")
ax.axhline(0, color="0.3", ls=":", lw=0.8)
tous = sorted(e for p in par_jour.values() for _, e in p)
q1, q2 = tous[len(tous) // 100], tous[-1 - len(tous) // 100] # cadre du 1er au 99e centile : un pic isolé n'écrase pas l'échelle
marge = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - marge, q2 + marge)
hors_cadre = sum(not q1 - marge <= e <= q2 + marge for e in tous)
for x0, x1 in ((h1, 24), (0, h2)) if h1 > h2 else ((h1, h2),):
ax.axvspan(x0, x1, color="0.9", lw=0)
ax.set_title(s + (" (réf.)" if est_reference(series[s]) else "") + f" bruit {bruit:.2g}"
+ (f" ({hors_cadre} hors cadre)" if hors_cadre else ""), fontsize=9)
ax.set_xlim(0, 24)
ax.set_xticks(range(0, 25, 3))
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)
if ax is axes.flat[0]:
ax.legend(fontsize=6, ncol=2)
for ax in list(axes.flat)[len(noms):]:
ax.axis("off")
fig.suptitle(f"{station} : {col} moins le niveau de la dernière nuit ({h1} h → {h2} h, grisée), "
"une courbe par jour, en abscisse l'heure du jour (UTC) ; une échelle par capteur")
fig.tight_layout()
chemin = os.path.join(o.sortie, f"{station}_parheure-{col}_{borne(o.debut, 'debut')}_{borne(o.fin, 'fin')}.png")
fig.savefig(chemin, dpi=100, bbox_inches="tight")
plt.close(fig)
print(chemin)
nb_jours = {s: {} for s in noms} # capteur -> créneau -> jours mesurés
for s in noms:
for m in series[s]:
if m[col] is not None:
nb_jours[s].setdefault((m["t"].hour * 60 + m["t"].minute) // pas * pas, set()).add(m["t"].date())
print(f"\n{col} : par créneau de {pas} min de l'heure du jour, capteurs à plus de {o.k:g} bruits du niveau de la nuit "
f"au moins {o.jours} jours (jours / jours mesurés, médiane de ces écarts, en bruits)")
vide = True
for c in sorted(creneaux):
cites = []
for s, haut in ((s, h) for s in noms for h in (True, False)):
ec = creneaux[c].get((s, haut), {})
if len(ec) >= o.jours:
med = statistics.median(ec.values())
cites.append(f"{s} {len(ec)}/{len(nb_jours[s].get(c, ()))} {med:+.3g} ({med / bruits[s]:+.1f})")
if cites:
vide = False
print(f"{c // 60:02d}:{c % 60:02d} {len(cites):2} capteurs : " + ", ".join(cites))
if vide:
print("(aucun)")
if o.bornes:
bornes(o, noms, series, colonnes, station)
def niveaux_de_nuit(pts, h1, h2):
"""[(t, v)] -> {date de la nuit: médiane}."""
nuits = {}
for t, v in pts:
if de_nuit(t, h1, h2):
nuits.setdefault((t - timedelta(hours=h1)).date(), []).append(v)
return {d: statistics.median(v) for d, v in nuits.items()}
def bornes(o, noms, series, colonnes, station):
"""Section 3 : par jour, l'écart au niveau de la nuit qui le précède, pour la médiane des cibles et la pose."""
h1, h2 = o.nuit
duree_nuit = timedelta(hours=(h2 - h1) % 24)
choisis = noms if o.capteurs else [s for s in noms if series[s] and not est_reference(series[s])] or noms
suites = [(f"{col} médiane de {len(choisis)} capteur(s)" if len(choisis) > 1 else f"{col} {choisis[0]}", col,
{s: [(m["t"], m[col]) for m in series[s] if m[col] is not None] for s in choisis}) for col in colonnes]
poses = filtrer(charger_camera(o.dossier), o.debut, o.fin)
suites += [(f"pose {c}", f"pose_{c}", {c: [(p["t"], p[c]) for p in poses if p[c] is not None]}) for c in POSE
if any(p[c] is not None for p in poses)]
tous = [t for _, _, d in suites for p in d.values() for t, _ in p]
t0, t1 = min(tous), max(tous)
nuit_de = lambda t: (t - duree_nuit - timedelta(hours=h1)).date() # nuit qui finit avant t (le jour de t commence à sa fin)
jours = [nuit_de(t0) + timedelta(days=i) for i in range((nuit_de(t1) - nuit_de(t0)).days + 1)]
hm = lambda t: f"{t:%d/%m %H:%M}"
seuil_txt = f"seuil {o.fraction:g} fois le plus grand |écart| du jour" if o.fraction else f"seuil {o.k:g} bruits"
print(f"\n== Bornes de chaque jour : écart au niveau de la nuit qui le précède ({h1} h → {h2} h UTC), {seuil_txt}"
+ (f", écart lissé sur {o.lissage} mesures" if o.lissage > 1 else "") + " ==")
print("jour : de la fin d'une nuit à la fin de la suivante, daté de son début ; « avant », « après » : mesure juste avant le premier "
"dépassement, juste après le dernier ; nuit suivante : son niveau moins celui de la nuit de référence")
fig, axes = plt.subplots(len(suites), 1, sharex=True, figsize=(18, 2.2 * len(suites)), squeeze=False)
for ax, (nom, court, par_capteur) in zip(axes.flat, suites):
niv = {s: niveaux_de_nuit(p, h1, h2) for s, p in par_capteur.items()}
nuits = sorted({d for n in niv.values() for d in n})
ecarts = [] # (jour, nuit de réf., [(t, écart)], écart de la nuit suivante)
for d in jours:
debut = datetime.combine(d, datetime.min.time(), tzinfo=timezone.utc) + timedelta(hours=h1) + duree_nuit
fin = debut + timedelta(days=1)
ref = max((n for n in nuits if n <= d), default=None) or (min(nuits) if nuits else None)
par_t = {}
for s, p in par_capteur.items():
if ref in niv[s]:
for t, v in p:
if debut <= t < fin:
par_t.setdefault(t, []).append(v - niv[s][ref])
suivante = [niv[s][d + timedelta(days=1)] - niv[s][ref] for s in niv if ref in niv[s] and d + timedelta(days=1) in niv[s]]
ecarts.append((debut.date(), d, ref, [(t, statistics.median(e)) for t, e in sorted(par_t.items())],
statistics.median(suivante) if suivante else None))
diffs = [abs(b[1] - a_[1]) for *_, p, _ in ecarts for a_, b in zip(p, p[1:])]
bruit = statistics.median(diffs) if diffs else 0
bruit = bruit if bruit >= ARRONDI else 0
print(f"\n{nom} : bruit {bruit:.3g}")
enb = lambda e: f"{e:+.4g} ({e / bruit:+.1f})" if bruit else f"{e:+.4g}"
h = o.lissage // 2
for jour, d, ref, brut, suiv in ecarts:
if not brut:
print(f"{court} {jour:%d/%m} aucune mesure" + ("" if nuits else " de nuit"))
continue
p = [(t, statistics.median(e for _, e in brut[max(0, i - h):i + h + 1])) for i, (t, _) in enumerate(brut)] if h else brut
seuil = o.fraction * max(abs(e) for _, e in p) if o.fraction else o.k * bruit
ligne = (f"{court} {jour:%d/%m} nuit {ref:%d/%m}" + (" (pas la précédente)" if ref != d else "") + f" | {len(p)} mesures {hm(p[0][0])} → {hm(p[-1][0])} | "
+ (f"seuil {seuil:.4g} | " if o.fraction else ""))
hors = [i for i, (_, e) in enumerate(p) if seuil and abs(e) > seuil]
if hors:
i, j = hors[0], hors[-1]
ligne += (f"> {'seuil' if o.fraction else f'{o.k:g} bruits'} : {len(hors)} mesures, de {hm(p[i][0])} (avant {hm(p[i - 1][0]) if i else '-'}) "
f"à {hm(p[j][0])} (après {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
ax.axvspan(p[i][0], p[j][0], color="0.85", lw=0)
else:
ligne += f"jamais > {'seuil' if o.fraction else f'{o.k:g} bruits'}"
haut, bas = max(p, key=lambda x: x[1]), min(p, key=lambda x: x[1])
ligne += f" | max {enb(haut[1])} à {hm(haut[0])}, min {enb(bas[1])} à {hm(bas[0])}"
x = max(range(len(p)), key=lambda i: abs(p[i][1]))
signe = 1 if p[x][1] > 0 else -1
if seuil and signe * p[x][1] > seuil:
i = j = x
while i > 0 and signe * p[i - 1][1] > seuil:
i -= 1
while j + 1 < len(p) and signe * p[j + 1][1] > seuil:
j += 1
ligne += (f" | autour du {'max' if signe > 0 else 'min'} : {j - i + 1} mesures, de {hm(p[i][0])} (avant {hm(p[i - 1][0]) if i else '-'}) "
f"à {hm(p[j][0])} (après {hm(p[j + 1][0]) if j + 1 < len(p) else '-'})")
ax.axvspan(p[i][0], p[j][0], color="0.7", lw=0)
ligne += f" | nuit suivante {enb(suiv)}" if suiv is not None else " | nuit suivante non mesurée"
print(ligne)
if h:
ax.plot([t for t, _ in brut], [e for _, e in brut], "-", lw=0.5, color="0.6")
ax.plot([t for t, _ in p], [e for _, e in p], "-", lw=0.8, color="C0")
ax.plot([haut[0], bas[0]], [haut[1], bas[1]], "o", ms=3, color="C3")
if bruit and not o.fraction:
for y in (o.k * bruit, -o.k * bruit):
ax.axhline(y, color="0.4", ls=":", lw=0.8)
ax.axhline(0, color="0.3", lw=0.5)
tous_e = sorted(e for *_, p, _ in ecarts for _, e in p)
hors_cadre = 0
if tous_e: # cadre du 1er au 99e centile élargi de moitié, comme parheure : un pic isolé n'écrase pas l'échelle
q1, q2 = tous_e[len(tous_e) // 100], tous_e[-1 - len(tous_e) // 100]
marge = (q2 - q1) * 0.5 or 1
ax.set_ylim(q1 - marge, q2 + marge)
hors_cadre = sum(not q1 - marge <= e <= q2 + marge for e in tous_e)
ax.set_title(f"{nom} bruit {bruit:.2g}" + (f" ({hors_cadre} hors cadre)" if hors_cadre else ""), fontsize=9)
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)
fig.suptitle(f"{station} : écart au niveau de la nuit qui précède chaque jour ({h1} h → {h2} h) ; "
f"grisé clair : du premier au dernier dépassement {'du seuil (' + seuil_txt + ')' if o.fraction else f'de {o.k:g} bruits'}, foncé : la suite autour de l'extrême ; points : max et min du jour"
+ (f" ; écart lissé sur {o.lissage} mesures (bleu), brut (gris)" if o.lissage > 1 else ""))
fig.tight_layout()
chemin = os.path.join(o.sortie, f"{station}_bornes_{borne(o.debut, 'debut')}_{borne(o.fin, 'fin')}.png")
fig.savefig(chemin, dpi=100, bbox_inches="tight")
plt.close(fig)
print(chemin)
if __name__ == "__main__":
main()
+264
View File
@@ -0,0 +1,264 @@
"""System metrics of the station (metrics/: temperatures, voltage, battery, link…), as a table and a figure,
against the target median of corr_x, corr_y and the camera pose.
Usage: .venv/bin/python tools/scripts/metrics.py data/PS_DES_COUDRES/
[--names sensor_A0_deg_c,BusVoltage] [--step 1h] [--stat median|min|max] [--against temperature] [--smoothing 24h]
[--metrics-only] [--start ...] [--end ...] [--out FOLDER]
Columns: `temperature` (median of the sensor_*_deg_c components at each hour, as the detectors read it,
data.load_temperature), then each metric of --names (default: the *_deg_c and BusVoltage present), then,
unless --metrics-only, tgt_corr_x, tgt_corr_y (at each measurement, median over the targets of corr minus the target's
own level, its median over the window; accepted measurements only) and the raw camera pose (cam_pan,
cam_tilt, cam_rotation, cam_scale, cam_sigma0). An empty cell of a metrics/ CSV is skipped, not read as zero.
A text metric (BatteryState, link, last_sync…) is given as it is (the last one of the step).
Never an identifier: SIM_CCID and ip* are neither read nor listed.
Prints: the available metrics; the table, one line per step of --step (start of the step, UTC; --stat of the values of the
step; "-": no value), steps where no metric has a value are left out; per column, the number of filled
steps, the minimum and the maximum with their time; the correlation r and the slope (least squares, over the steps where
both have a value) of each numeric column on --against (default temperature; with --smoothing D, on its
moving average centred over D, column <against>_smooth); the dated runs of steps without any metric.
One PNG, path printed: temperatures on one panel, each other numeric metric on its own, then tgt_corr_x,
tgt_corr_y and the pose, at native times (line broken over a gap). Gives values, not a verdict.
"""
import argparse
import csv
import glob
import os
import statistics
import sys
from datetime import date, timedelta
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402
from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time # noqa: E402
from inventory import METRIC_RE # noqa: E402
from plot import bound # noqa: E402
from summary import STATS, duration # noqa: E402
def is_identifier(name):
"""Metric that carries an identifier (SIM card, IP addresses): never read."""
return name == "SIM_CCID" or name.startswith("ip")
def parse_value(text):
v = text.strip().strip("[]").strip("'\"")
try:
return float(v)
except ValueError:
return v or None
def read_metrics(folder, start, end):
"""metrics/ -> {name: [{'t', 'v'}]} sorted, over the window; v float or text; empty cell skipped; name without a value: []."""
prefix = os.path.basename(os.path.normpath(folder)) + "_"
by_name = {}
for p in glob.glob(os.path.join(folder, "metrics", "*.csv")):
m = METRIC_RE.match(os.path.basename(p))
if not m:
continue
name = m.group(1)
name = name[len(prefix):] if name.startswith(prefix) else name.split("_", 1)[-1]
if is_identifier(name):
continue
day = date(*map(int, m.group(2, 3, 4)))
series = by_name.setdefault(name, {})
if (start and day < start.date() - timedelta(days=1)) or (end and day > end.date()):
continue
with open(p, newline="") as f:
for r in csv.DictReader(f):
if (v := parse_value(r.get("metric") or "")) is not None:
series[parse_time(r["time_utc"])] = v
return {n: filter_window([{"t": t, "v": s[t]} for t in sorted(s)], start, end) for n, s in by_name.items()}
def targets(by_sensor, col, start, end):
"""At each measurement, median over the targets of col minus the target's own level (median over the window)."""
deviations = {}
for s, ms in by_sensor.items():
if is_reference(ms):
continue
ms = [m for m in filter_window(ms, start, end) if m[col] is not None]
if ms:
level = statistics.median(m[col] for m in ms)
for m in ms:
deviations.setdefault(m["t"], []).append(m[col] - level)
return [{"t": t, "v": statistics.median(v)} for t, v in sorted(deviations.items())]
def smooth(series, width):
"""Moving average centred over width (numeric values)."""
pts = [p for p in series if isinstance(p["v"], float)]
# ponytail: O(n²), enough for hourly metrics over a few weeks
return [{"t": p["t"], "v": statistics.fmean(q["v"] for q in pts if abs(q["t"] - p["t"]) <= width / 2)} for p in pts]
def fmt(v):
return "-" if v is None else f"{v:.6g}" if isinstance(v, float) else str(v)
def runs(empty_steps, step):
"""[t] sorted -> [(start, end, n)] of consecutive steps."""
out = []
for t in empty_steps:
if out and t - out[-1][1] == step:
out[-1] = (out[-1][0], t, out[-1][2] + 1)
else:
out.append((t, t, 1))
return out
def cut(series):
"""Times and numeric values, NaN inserted over a gap of more than 1.5 times the usual step (the line breaks)."""
pts = [(p["t"], p["v"]) for p in series if isinstance(p["v"], float)]
if len(pts) < 3:
return [t for t, _ in pts], [v for _, v in pts]
step = statistics.median(b[0] - a[0] for a, b in zip(pts, pts[1:]))
ts, vs = [pts[0][0]], [pts[0][1]]
for (a, _), (b, v) in zip(pts, pts[1:]):
if b - a > 1.5 * step:
ts.append(a + (b - a) / 2)
vs.append(float("nan"))
ts.append(b)
vs.append(v)
return ts, vs
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder")
a.add_argument("--names", help="metrics, comma-separated (default: the *_deg_c and BusVoltage present)")
a.add_argument("--step", default="1h")
a.add_argument("--stat", default="median", choices=STATS)
a.add_argument("--against", default="temperature", help="column on which r and the slope are computed")
a.add_argument("--smoothing", type=duration, help="centred moving average of --against, e.g. 24h")
a.add_argument("--metrics-only", action="store_true", help="without tgt_corr_x, tgt_corr_y or the pose")
a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time)
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG")
o = a.parse_args()
step = duration(o.step)
met = read_metrics(o.folder, o.start, o.end)
if not met:
print(f"No metrics/ in {o.folder}.")
else:
print("Available metrics (values over the window; identifiers SIM_CCID, ip* excluded): "
+ ", ".join(f"{n} ({len(s)})" if s else f"{n} (empty)" for n, s in sorted(met.items())))
if o.names:
names = o.names.split(",")
if forbidden := [n for n in names if is_identifier(n)]:
sys.exit(f"Identifiers, never read: {', '.join(forbidden)}.")
if missing := [n for n in names if n not in met]:
sys.exit(f"Unknown metrics: {', '.join(missing)}. Known: {', '.join(sorted(met))}.")
else:
names = sorted(n for n in met if n.endswith("_deg_c")) + [n for n in ("BusVoltage",) if n in met]
series = {"temperature": filter_window([{"t": t, "v": v} for t, v in load_temperature(o.folder)], o.start, o.end)}
series.update({n: met[n] for n in names})
if o.smoothing:
if o.against not in series:
sys.exit(f"--against {o.against}: not a column ({', '.join(series)}).")
series[o.against + "_smooth"] = smooth(series[o.against], o.smoothing)
metric_cols = list(series)
if not o.metrics_only:
by_sensor = load(o.folder)
for c in ("corr_x", "corr_y"):
series["tgt_" + c] = targets(by_sensor, c, o.start, o.end)
poses = filter_window(load_camera(o.folder), o.start, o.end)
for c in POSE:
series["cam_" + c] = [{"t": p["t"], "v": p[c]} for p in poses if p[c] is not None]
reference = o.against + "_smooth" if o.smoothing else o.against
if reference not in series:
sys.exit(f"--against {o.against}: not a column ({', '.join(series)}).")
times = [p["t"] for s in series.values() for p in s]
if not times:
sys.exit("No value in the requested window.")
t0 = (o.start or min(times)).replace(minute=0, second=0, microsecond=0)
t1 = o.end or max(times)
cells = {} # column -> start of the step -> values
for c, s in series.items():
for p in s:
cells.setdefault(c, {}).setdefault(t0 + ((p["t"] - t0) // step) * step, []).append(p["v"])
stat = STATS[o.stat]
table = {c: {k: (stat(v) if all(isinstance(x, float) for x in v) else v[-1]) for k, v in d.items()}
for c, d in cells.items()}
grid = [t0 + i * step for i in range(int((t1 - t0) / step) + 1)]
columns = list(series)
print(f"\n{o.stat} per step of {o.step} (start of the step, UTC); \"-\": no value; tgt_: deviation from the targets' own level")
print("\t".join(["time"] + columns))
for k in grid:
line = [table.get(c, {}).get(k) for c in columns]
if any(v is not None for v in line):
print("\t".join([iso(k)] + [fmt(v) for v in line]))
print(f"\nPer column ({len(grid)} steps):")
for c in columns:
d = table.get(c, {})
nums = [(v, k) for k, v in d.items() if isinstance(v, float)]
if not d:
print(f" {c}: empty")
elif nums:
(vmin, kmin), (vmax, kmax) = min(nums), max(nums)
print(f" {c}: {len(d)} steps; min {fmt(vmin)} ({iso(kmin)}); max {fmt(vmax)} ({iso(kmax)})")
else:
ks = sorted(d)
print(f" {c}: {len(d)} steps, text; {len(set(d.values()))} distinct values; "
f"first {d[ks[0]]} ({iso(ks[0])}); last {d[ks[-1]]} ({iso(ks[-1])})")
ref = table.get(reference, {})
print(f"\nCorrelation r and slope on {reference} (over the steps where both have a value):")
if not any(isinstance(v, float) for v in ref.values()):
print(f" {reference} empty: none")
for c in columns:
if c == reference or not ref or not any(isinstance(v, float) for v in table.get(c, {}).values()):
continue
pairs = [(ref[k], v) for k, v in table.get(c, {}).items()
if isinstance(v, float) and isinstance(ref.get(k), float)]
try:
r = statistics.correlation(*zip(*pairs))
slope = statistics.linear_regression(*zip(*pairs)).slope
print(f" {c}: r = {r:+.2f}; slope = {slope:.4g} per unit of {reference}; n = {len(pairs)}")
except (statistics.StatisticsError, TypeError, ValueError):
print(f" {c}: - (n = {len(pairs)})")
empty = [k for k in grid if all(k not in table.get(c, {}) for c in metric_cols)]
print(f"\nSteps without any metric ({', '.join(metric_cols)}): {len(empty)} of {len(grid)}")
for d, f, n in runs(empty, step):
print(f" {iso(d)} → {iso(f)} ({n} steps)")
panels = {} # title -> columns
for c in columns:
if any(isinstance(p["v"], float) for p in series[c]):
key = "°C" if c.startswith("temperature") or "_deg_c" in c else c
panels.setdefault(key, []).append(c)
if not panels:
print("\nNothing to plot.")
return
fig, axes = plt.subplots(len(panels), 1, sharex=True, figsize=(14, 2.4 * len(panels)), squeeze=False,
layout="constrained")
for ax, (title, cols) in zip(axes[:, 0], panels.items()):
for c in cols:
black = c in ("temperature", reference)
ax.plot(*cut(series[c]), ".-", ms=2, lw=1.4 if black else 0.7, label=c, zorder=3 if black else 2,
color="k" if c == "temperature" else None)
if len(cols) > 1:
ax.legend(loc="upper left", bbox_to_anchor=(1.0, 1.0), fontsize=8)
ax.set_ylabel(title if title == "°C" else title.replace("_", "_\n", 1), fontsize=8)
ax.grid(alpha=0.3)
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
station = os.path.basename(os.path.normpath(o.folder))
fig.suptitle(f"{station}: metrics, targets and pose (UTC times)")
os.makedirs(o.out, exist_ok=True)
out = os.path.join(o.out, f"{station}_metrics_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png")
fig.savefig(out, dpi=100, bbox_inches="tight")
print(f"\n{out}")
if __name__ == "__main__":
main()
-264
View File
@@ -1,264 +0,0 @@
"""Métriques système de la station (metrics/ : températures, tension, batterie, lien…), en table et en figure,
face à la médiane des cibles de corr_x, corr_y et à la pose de la caméra.
Usage : .venv/bin/python tools/scripts/metriques.py data/PS_DES_COUDRES/
[--noms sensor_A0_deg_c,BusVoltage] [--pas 1h] [--stat median|min|max] [--contre temperature] [--lissage 24h]
[--metriques-seules] [--debut ...] [--fin ...] [--sortie DOSSIER]
Colonnes : `temperature` (médiane des composants sensor_*_deg_c à chaque heure, comme la lisent les détecteurs,
donnees.charger_temperature), puis chaque métrique de --noms (défaut : les *_deg_c et BusVoltage présents), puis,
sauf --metriques-seules, cib_corr_x, cib_corr_y (à chaque mesure, médiane sur les cibles de corr moins le niveau
propre de la cible, sa médiane sur la fenêtre ; mesures acceptées seulement) et la pose brute de la caméra (cam_pan,
cam_tilt, cam_rotation, cam_scale, cam_sigma0). Une case vide d'un CSV de metrics/ est sautée, pas lue comme zéro.
Une métrique texte (BatteryState, link, last_sync…) est rendue telle quelle (la dernière du pas).
Jamais d'identifiant : SIM_CCID et ip* ne sont ni lus ni listés.
Imprime : les métriques disponibles ; la table, une ligne par pas de --pas (début du pas, UTC ; --stat des valeurs du
pas ; « - » : aucune valeur), les pas où aucune métrique n'a de valeur n'y sont pas ; par colonne, le nombre de pas
remplis, le minimum et le maximum avec leur instant ; la corrélation r et la pente (moindres carrés, sur les pas où
les deux ont une valeur) de chaque colonne numérique sur --contre (défaut temperature ; avec --lissage D, sur sa
moyenne glissante centrée sur D, colonne <contre>_lisse) ; les suites datées de pas sans aucune métrique.
Un PNG, chemin imprimé : températures sur un panneau, chaque autre métrique numérique sur le sien, puis cib_corr_x,
cib_corr_y et la pose, aux instants natifs (trait coupé sur un trou). Rend des valeurs, pas de verdict.
"""
import argparse
import csv
import glob
import os
import statistics
import sys
from datetime import date, timedelta
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402
from donnees import POSE, charger, charger_camera, charger_temperature, est_reference, filtrer, instant, iso # noqa: E402
from graphique import borne # noqa: E402
from inventaire import METRIQUE_RE # noqa: E402
from resume import STATS, duree # noqa: E402
def identifiant(nom):
"""Métrique qui porte un identifiant (carte SIM, adresses IP) : jamais lue."""
return nom == "SIM_CCID" or nom.startswith("ip")
def valeur(texte):
v = texte.strip().strip("[]").strip("'\"")
try:
return float(v)
except ValueError:
return v or None
def lire_metriques(dossier, debut, fin):
"""metrics/ -> {nom: [{'t', 'v'}]} triées, sur la fenêtre ; v float ou texte ; case vide sautée ; nom sans valeur : []."""
prefixe = os.path.basename(os.path.normpath(dossier)) + "_"
par = {}
for p in glob.glob(os.path.join(dossier, "metrics", "*.csv")):
m = METRIQUE_RE.match(os.path.basename(p))
if not m:
continue
nom = m.group(1)
nom = nom[len(prefixe):] if nom.startswith(prefixe) else nom.split("_", 1)[-1]
if identifiant(nom):
continue
jour = date(*map(int, m.group(2, 3, 4)))
serie = par.setdefault(nom, {})
if (debut and jour < debut.date() - timedelta(days=1)) or (fin and jour > fin.date()):
continue
with open(p, newline="") as f:
for r in csv.DictReader(f):
if (v := valeur(r.get("metric") or "")) is not None:
serie[instant(r["time_utc"])] = v
return {n: filtrer([{"t": t, "v": s[t]} for t in sorted(s)], debut, fin) for n, s in par.items()}
def cibles(par, col, debut, fin):
"""À chaque mesure, médiane sur les cibles de col moins le niveau propre de la cible (médiane sur la fenêtre)."""
ecarts = {}
for s, ms in par.items():
if est_reference(ms):
continue
ms = [m for m in filtrer(ms, debut, fin) if m[col] is not None]
if ms:
niveau = statistics.median(m[col] for m in ms)
for m in ms:
ecarts.setdefault(m["t"], []).append(m[col] - niveau)
return [{"t": t, "v": statistics.median(v)} for t, v in sorted(ecarts.items())]
def lisser(serie, largeur):
"""Moyenne glissante centrée sur largeur (valeurs numériques)."""
pts = [p for p in serie if isinstance(p["v"], float)]
# ponytail: O(n²), assez pour des métriques horaires sur quelques semaines
return [{"t": p["t"], "v": statistics.fmean(q["v"] for q in pts if abs(q["t"] - p["t"]) <= largeur / 2)} for p in pts]
def fmt(v):
return "-" if v is None else f"{v:.6g}" if isinstance(v, float) else str(v)
def suites(pas_vides, pas):
"""[t] triés -> [(début, fin, n)] de pas consécutifs."""
out = []
for t in pas_vides:
if out and t - out[-1][1] == pas:
out[-1] = (out[-1][0], t, out[-1][2] + 1)
else:
out.append((t, t, 1))
return out
def coupe(serie):
"""Instants et valeurs numériques, NaN inséré sur un écart de plus de 1,5 fois le pas habituel (le trait se coupe)."""
pts = [(p["t"], p["v"]) for p in serie if isinstance(p["v"], float)]
if len(pts) < 3:
return [t for t, _ in pts], [v for _, v in pts]
pas = statistics.median(b[0] - a[0] for a, b in zip(pts, pts[1:]))
ts, vs = [pts[0][0]], [pts[0][1]]
for (a, _), (b, v) in zip(pts, pts[1:]):
if b - a > 1.5 * pas:
ts.append(a + (b - a) / 2)
vs.append(float("nan"))
ts.append(b)
vs.append(v)
return ts, vs
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossier")
a.add_argument("--noms", help="métriques, séparées par des virgules (défaut : les *_deg_c et BusVoltage présents)")
a.add_argument("--pas", default="1h")
a.add_argument("--stat", default="median", choices=STATS)
a.add_argument("--contre", default="temperature", help="colonne sur laquelle r et la pente sont calculés")
a.add_argument("--lissage", type=duree, help="moyenne glissante centrée de --contre, ex. 24h")
a.add_argument("--metriques-seules", action="store_true", help="sans cib_corr_x, cib_corr_y ni la pose")
a.add_argument("--debut", type=instant)
a.add_argument("--fin", type=instant)
a.add_argument("--sortie", default=os.path.join("out", "diagnostic"), help="dossier du PNG")
o = a.parse_args()
pas = duree(o.pas)
met = lire_metriques(o.dossier, o.debut, o.fin)
if not met:
print(f"Pas de metrics/ dans {o.dossier}.")
else:
print("Métriques disponibles (valeurs sur la fenêtre ; identifiants SIM_CCID, ip* exclus) : "
+ ", ".join(f"{n} ({len(s)})" if s else f"{n} (vide)" for n, s in sorted(met.items())))
if o.noms:
noms = o.noms.split(",")
if interdits := [n for n in noms if identifiant(n)]:
sys.exit(f"Identifiants, jamais lus : {', '.join(interdits)}.")
if absents := [n for n in noms if n not in met]:
sys.exit(f"Métriques inconnues : {', '.join(absents)}. Connues : {', '.join(sorted(met))}.")
else:
noms = sorted(n for n in met if n.endswith("_deg_c")) + [n for n in ("BusVoltage",) if n in met]
series = {"temperature": filtrer([{"t": t, "v": v} for t, v in charger_temperature(o.dossier)], o.debut, o.fin)}
series.update({n: met[n] for n in noms})
if o.lissage:
if o.contre not in series:
sys.exit(f"--contre {o.contre} : pas une colonne ({', '.join(series)}).")
series[o.contre + "_lisse"] = lisser(series[o.contre], o.lissage)
metriques = list(series)
if not o.metriques_seules:
par = charger(o.dossier)
for c in ("corr_x", "corr_y"):
series["cib_" + c] = cibles(par, c, o.debut, o.fin)
poses = filtrer(charger_camera(o.dossier), o.debut, o.fin)
for c in POSE:
series["cam_" + c] = [{"t": p["t"], "v": p[c]} for p in poses if p[c] is not None]
reference = o.contre + "_lisse" if o.lissage else o.contre
if reference not in series:
sys.exit(f"--contre {o.contre} : pas une colonne ({', '.join(series)}).")
tous = [p["t"] for s in series.values() for p in s]
if not tous:
sys.exit("Aucune valeur dans la fenêtre demandée.")
t0 = (o.debut or min(tous)).replace(minute=0, second=0, microsecond=0)
t1 = o.fin or max(tous)
cases = {} # colonne -> début du pas -> valeurs
for c, s in series.items():
for p in s:
cases.setdefault(c, {}).setdefault(t0 + ((p["t"] - t0) // pas) * pas, []).append(p["v"])
stat = STATS[o.stat]
table = {c: {k: (stat(v) if all(isinstance(x, float) for x in v) else v[-1]) for k, v in d.items()}
for c, d in cases.items()}
grille = [t0 + i * pas for i in range(int((t1 - t0) / pas) + 1)]
colonnes = list(series)
print(f"\n{o.stat} par pas de {o.pas} (début du pas, UTC) ; « - » : aucune valeur ; cib_ : écart au niveau propre des cibles")
print("\t".join(["heure"] + colonnes))
for k in grille:
ligne = [table.get(c, {}).get(k) for c in colonnes]
if any(v is not None for v in ligne):
print("\t".join([iso(k)] + [fmt(v) for v in ligne]))
print(f"\nPar colonne ({len(grille)} pas) :")
for c in colonnes:
d = table.get(c, {})
nums = [(v, k) for k, v in d.items() if isinstance(v, float)]
if not d:
print(f" {c} : vide")
elif nums:
(vmin, kmin), (vmax, kmax) = min(nums), max(nums)
print(f" {c} : {len(d)} pas ; min {fmt(vmin)} ({iso(kmin)}) ; max {fmt(vmax)} ({iso(kmax)})")
else:
ks = sorted(d)
print(f" {c} : {len(d)} pas, texte ; {len(set(d.values()))} valeurs distinctes ; "
f"première {d[ks[0]]} ({iso(ks[0])}) ; dernière {d[ks[-1]]} ({iso(ks[-1])})")
ref = table.get(reference, {})
print(f"\nCorrélation r et pente sur {reference} (par pas où les deux ont une valeur) :")
if not any(isinstance(v, float) for v in ref.values()):
print(f" {reference} vide : aucune")
for c in colonnes:
if c == reference or not ref or not any(isinstance(v, float) for v in table.get(c, {}).values()):
continue
paires = [(ref[k], v) for k, v in table.get(c, {}).items()
if isinstance(v, float) and isinstance(ref.get(k), float)]
try:
r = statistics.correlation(*zip(*paires))
pente = statistics.linear_regression(*zip(*paires)).slope
print(f" {c} : r = {r:+.2f} ; pente = {pente:.4g} par unité de {reference} ; n = {len(paires)}")
except (statistics.StatisticsError, TypeError, ValueError):
print(f" {c} : - (n = {len(paires)})")
vides = [k for k in grille if all(k not in table.get(c, {}) for c in metriques)]
print(f"\nPas sans aucune métrique ({', '.join(metriques)}) : {len(vides)} sur {len(grille)}")
for d, f, n in suites(vides, pas):
print(f" {iso(d)} → {iso(f)} ({n} pas)")
panneaux = {} # titre -> colonnes
for c in colonnes:
if any(isinstance(p["v"], float) for p in series[c]):
cle = "°C" if c.startswith("temperature") or "_deg_c" in c else c
panneaux.setdefault(cle, []).append(c)
if not panneaux:
print("\nRien à tracer.")
return
fig, axes = plt.subplots(len(panneaux), 1, sharex=True, figsize=(14, 2.4 * len(panneaux)), squeeze=False,
layout="constrained")
for ax, (titre, cols) in zip(axes[:, 0], panneaux.items()):
for c in cols:
noir = c in ("temperature", reference)
ax.plot(*coupe(series[c]), ".-", ms=2, lw=1.4 if noir else 0.7, label=c, zorder=3 if noir else 2,
color="k" if c == "temperature" else None)
if len(cols) > 1:
ax.legend(loc="upper left", bbox_to_anchor=(1.0, 1.0), fontsize=8)
ax.set_ylabel(titre if titre == "°C" else titre.replace("_", "_\n", 1), fontsize=8)
ax.grid(alpha=0.3)
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
station = os.path.basename(os.path.normpath(o.dossier))
fig.suptitle(f"{station} : métriques, cibles et pose (heures UTC)")
os.makedirs(o.sortie, exist_ok=True)
sortie = os.path.join(o.sortie, f"{station}_metriques_{borne(o.debut, 'debut')}_{borne(o.fin, 'fin')}.png")
fig.savefig(sortie, dpi=100, bbox_inches="tight")
print(f"\n{sortie}")
if __name__ == "__main__":
main()
+491
View File
@@ -0,0 +1,491 @@
"""Overview of all the sensors of an "advanced" export: what the sensors do together.
Usage: .venv/bin/python tools/scripts/overview.py data/PS_DES_COUDRES/
[--columns x,y,sigma0] [--column error_value] [--step 15min] [--k 5] [--min 5]
[--residuals corr_x,corr_y,error_value] [--outside T1/T2] [--others] [--rigid BASE [--without A,B] [--follow A,B]] [--table T1/T2|no] [--sensors A,B] [--start ...] [--end ...] [--out FOLDER]
Each sensor is compared with its own level over the window --start/--end (median and MAD of the window).
Two PNGs, whose paths are printed:
1. "common": first, if the folder has camera movements (camera/), one panel per pose quantity
(pan, tilt, rotation [deg], scale, sigma0 of the registration), raw values, one point per registration; then one panel per column of --columns (default x,y,sigma0: raw position and sigma0 of the
camera): median, over the sensors, of each sensor's deviation from its median; grey band = 10th to 90th
percentile between sensors (the spread); on each panel except sigma0, in addition, each global reference separately
(its deviation from its median, one line per reference) and the median of the targets alone (black line): one sees whether the
references move in the image with the targets or without them; last panel: number of sensors whose --column
(default error_value) exceeds its median by more than --k MAD at each step.
2. "multiples": one small plot per sensor (references first) of --column, median per step of --step,
dotted line = median of the window; each plot has its own scale, a plateau shows.
3. "residuals-<column>" (one PNG per column of --residuals, default corr_x,corr_y,error_value): one small plot per
sensor of the column minus the sensor's own level (its median over the window, or outside --outside T1/T2 if
given: the level is then that of the rest of the window, not that of the moment being read), one scale per sensor, zero
dotted; a deviation of a few hundredths on a `corr` column that the parallel bands of `plot.py` hide
shows there. Prints, per sensor, the level, the own noise (median of the differences between two consecutive measurements) and
the minimum and maximum deviation of the median per step of --step, with their time, in units of the column and in noises
(noise 0 and "-" in noises: values constant up to rounding, 1e-9; --residuals no: no residuals PNG).
With --others, each sensor is in addition brought back to the deviation from the median of the OTHER sensors at the same time (one
subtracts, at each measurement, the median of the other sensors of the same column minus their own level): what is
shared by all (the remainder of the compensation of a common movement) cancels out, what is specific to the sensor remains.
PNG "residuals-<column>-others"; the table also prints, per sensor, the slope of this residual on each
column of --columns (common median, e.g. y) and the correlation r: a residual proportional to the common movement
(even if it is not shared by all) has a non-zero slope and a high r.
4. "rigid" (with --rigid BASE): at each measurement, the overall movement of the targets since time BASE
(similarity: translation, rotation, scale, estimated by medians over the targets, except --without); for each
sensor, the residual dx, dy = raw position minus its position at BASE carried by this movement [px]. Panels:
dx and dy of each global reference (one line each) against the median and the 10th-90th percentile of the targets;
median and 90th percentile of |residual| of the targets; fitted rotation and scale; reproj_err of the references. Prints
the table per step of --step (rot_deg, scale, tx, ty, tgt_med, tgt_max, then dx, dy of each reference and of each
sensor of --follow A,B), then per sensor the min and the max of dx, dy with their time and the value at the last
time. Rejected measurements enter neither the fit, nor the band and the median of the targets, tgt_med,
tgt_max, nor the columns of the table; on the PNG, those of the references are crosses, which do not set the scale; in the per-sensor
summary, they come separately (number, min and max).
Also prints the table of the common median (one line per step of --step: time, deviation from the median of the window
for each column of --columns, then <column>_targets, the median of the targets alone, then the raw camera pose, empty without registration) then the minimum and the maximum of each column with their time, and those of each global reference; to date
a start or an end, the table is at the step of --step (default 15min, the measurement cadence; a wider step,
1h for example, flattens a peak of one to four steps and dates to the hour).
Also prints, for each time when at least --min sensors exceed their median by more than --k MAD,
the number and the sensors (--min 1: also the times when a single sensor leaves its level), and the
measurement times without a camera pose (no registration).
The per-step tables (common median, --rigid) repeat their header every 24 lines. With --table T1/T2, they
are printed only between T1 and T2 ("no": not at all); levels, minimum and maximum, PNGs and the rest always cover
the whole window --start/--end: over a month, one reads all the rest, then the table of one day at the same levels.
Gives values, not a verdict.
"""
import argparse
import math
import os
import statistics
import sys
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402
from data import POSE, filter_window, is_reference, iso, load, load_camera, parse_time, rejected # noqa: E402
from plot import COLUMNS, bound # noqa: E402
from summary import duration, own_noise # noqa: E402
def level(ms, col):
"""(median, MAD) of the non-empty values; None if there are none."""
v = [m[col] for m in ms if m[col] is not None]
if not v:
return None
med = statistics.median(v)
return med, statistics.median(abs(x - med) for x in v)
def similarity(p, q):
"""Similarity (a, b, tx, ty) that sends the points p onto q: q ≈ (a·x − b·y + tx, b·x + a·y + ty).
Estimation by medians (a target that jumps alone does not pull the fit): rotation and scale, median over
all pairs of targets of the complex ratio (qi − qj) / (pi − pj); translation, median of q − (rotation, scale)·p.
"""
p, q = [complex(*u) for u in p], [complex(*u) for u in q]
z = [(q[i] - q[j]) / (p[i] - p[j]) for i in range(len(p)) for j in range(i) if p[i] != p[j]]
r = complex(statistics.median(v.real for v in z), statistics.median(v.imag for v in z))
d = [w - r * u for u, w in zip(p, q)]
return r.real, r.imag, statistics.median(v.real for v in d), statistics.median(v.imag for v in d)
def rigid(o, names, series, times, base, station):
"""4. Each sensor against the overall movement of the targets since time --rigid: PNG "rigid" and table."""
t_b = min(times, key=lambda t: abs(t - o.rigid))
pos = {s: {m["t"]: (m["x"], m["y"]) for m in series[s] if m["x"] is not None and m["y"] is not None} for s in names}
without = set(o.without.split(",")) - {""}
acc = {s: {m["t"] for m in series[s] if not rejected(m)} for s in names} # accepted measurements: the only ones to enter the fit and the target statistics
refs = [s for s in names if series[s] and is_reference(series[s])]
tgt = [s for s in names if s not in refs and t_b in pos[s]]
fitted = [s for s in tgt if s not in without and t_b in acc[s]]
followed = [s for s in o.follow.split(",") if s and s not in refs]
res, fit = {s: {} for s in names}, {} # res: {sensor: {t: (dx, dy)}}; fit: {t: (rotation deg, scale, tx, ty, median |r|, max of the targets)}
for t in times:
ok = [s for s in fitted if t in pos[s] and t in acc[s]]
if len(ok) < 2:
continue
a, b, tx, ty = similarity([pos[s][t_b] for s in ok], [pos[s][t] for s in ok])
for s in names:
if t in pos[s] and t_b in pos[s]:
(x0, y0), (x, y) = pos[s][t_b], pos[s][t]
res[s][t] = (x - (a * x0 - b * y0 + tx), y - (b * x0 + a * y0 + ty))
r = [math.hypot(*res[s][t]) for s in tgt if t in res[s] and t in acc[s]]
fit[t] = (math.degrees(math.atan2(b, a)), math.hypot(a, b), tx, ty, statistics.median(r), max(r))
if not fit:
print(f"\n--rigid: fewer than two targets with x, y at {iso(t_b)} and after.")
return
tf = sorted(fit)
reproj = {s: {m["t"]: m["reproj_err"] for m in series[s] if m["reproj_err"] is not None} for s in refs}
panels = [("dx", 0), ("dy", 1)]
fig, axes = plt.subplots(6, 1, sharex=True, figsize=(14, 18), squeeze=False)
for ax, (name, k) in zip(axes[:2, 0], panels):
band = [sorted(res[s][t][k] for s in tgt if t in res[s] and t in acc[s]) for t in tf]
q = [statistics.quantiles(v, n=10, method="inclusive") if len(v) >= 2 else [v[0]] * 9 for v in band]
ax.fill_between(tf, [x[0] for x in q], [x[-1] for x in q], color="0.85", label="targets: 10th-90th percentile")
ax.plot(tf, [statistics.median(v) for v in band], "-", lw=1.2, color="k", label="targets: median")
for i, s in enumerate(refs):
ta = [t for t in res[s] if t in acc[s]]
ax.plot(ta, [res[s][t][k] for t in ta], "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (ref.)")
frame = ax.get_ylim() # the scale of the accepted measurements: a reference rejected far away is out of frame
for i, s in enumerate(refs):
tr = [t for t in res[s] if t not in acc[s]]
if tr: # rejected measurements: separately, one cross each, off the line
ax.plot(tr, [res[s][t][k] for t in tr], "x", ms=3, color=f"C{(i + 1) % 10}", label=f"{s} rejected")
ax.set_ylim(frame)
ax.set_ylabel(f"{name} [px]\nraw position minus\nthe targets' movement")
ax.legend(loc="upper left", fontsize=8, ncol=1 + len(refs) // 4)
axes[2, 0].plot(tf, [fit[t][4] for t in tf], "-", lw=1, color="k", label="targets: median of |residual|")
p90 = {t: statistics.quantiles([math.hypot(*res[s][t]) for s in tgt if t in res[s] and t in acc[s]], n=10, method="inclusive")[-1] for t in tf}
axes[2, 0].plot(tf, [p90[t] for t in tf], "-", lw=0.8, color="C3", label="targets: 90th percentile of |residual| (the max is in the table)")
axes[2, 0].set_ylabel("|residual| of the targets [px]")
axes[2, 0].legend(loc="upper left", fontsize=8)
axes[3, 0].plot(tf, [fit[t][0] for t in tf], "-", lw=1, color="C2")
axes[3, 0].set_ylabel("fitted rotation [deg]")
axes[4, 0].plot(tf, [fit[t][1] for t in tf], "-", lw=1, color="C2")
axes[4, 0].set_ylabel("fitted scale")
for i, s in enumerate(refs):
axes[5, 0].plot(list(reproj[s]), list(reproj[s].values()), "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (ref.)")
axes[5, 0].set_ylabel("reproj_err\nof the references")
if refs:
axes[5, 0].legend(loc="upper left", fontsize=8, ncol=1 + len(refs) // 4)
for ax in axes[:, 0]:
ax.grid(alpha=0.4)
ax.axvline(t_b, color="C3", ls=":", lw=0.8)
axes[-1, 0].set_xlim(tf[0], tf[-1])
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
fig.suptitle(f"{station}: overall movement (translation, rotation, scale) fitted on {len(fitted)} targets since {iso(t_b)} (dotted), "
"residual of each sensor, accepted measurements (cross: rejected reference) (UTC times)")
fig.tight_layout(rect=(0, 0, 1, 1 - 0.4 / fig.get_figheight()))
path = base.format("rigid")
fig.savefig(path, dpi=100, bbox_inches="tight")
plt.close(fig)
print(path)
print(f"\nOverall movement fitted on the targets since {iso(t_b)}: {', '.join(fitted)}"
+ (f" (outside the fit: {', '.join(sorted(without & set(tgt)))})" if without & set(tgt) else "")
+ f".\nResidual = raw position minus the position at {iso(t_b)} carried by this movement [px]; rejected measurements left out of the fit, "
f"of tgt_med, tgt_max and of the sensor columns (separately in the per-sensor summary); median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
cols = ["rot_deg", "scale", "tx", "ty", "tgt_med", "tgt_max"] + [f"{s}_{c}" for s in refs + followed for c in ("dx", "dy")]
midnight = tf[0].replace(hour=0, minute=0, second=0, microsecond=0)
t0 = midnight + ((tf[0] - midnight) // o.step) * o.step
per_step = {}
for t in tf:
line = per_step.setdefault(t0 + ((t - t0) // o.step) * o.step, {})
for c, v in zip(cols, fit[t]):
line.setdefault(c, []).append(v)
for s in refs + followed:
if t in res[s] and t in acc[s]:
line.setdefault(f"{s}_dx", []).append(res[s][t][0]), line.setdefault(f"{s}_dy", []).append(res[s][t][1])
w = max(10, max(len(c) for c in cols) + 1)
header = f"{'time':17}" + "".join(f"{c:>{w}}" for c in cols)
for i, t in enumerate(t for t in sorted(per_step) if o.in_table(t)):
if i % 24 == 0:
print(header)
print(f"{iso(t):17}" + "".join(f"{statistics.median(per_step[t][c]):{w}.{7 if c == 'scale' else 3}g}" if c in per_step[t] else " " * w for c in cols))
print("\nPer sensor, accepted measurements: min and max of dx, dy, value at the last time; then the rejected ones separately:")
for s in names:
good = {t: v for t, v in res[s].items() if t in acc[s]}
rej = {t: v for t, v in res[s].items() if t not in acc[s]}
for k, c in ((0, "dx"), (1, "dy")):
line = f"{s:9}{' (ref.)' if s in refs else ' '} {c}"
if good:
lo, hi = min(good.items(), key=lambda x: x[1][k]), max(good.items(), key=lambda x: x[1][k])
last = res[s].get(tf[-1])
line += (f" min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}, at the last time "
f"{last[k] if last else math.nan:+.3g}{' (rejected)' if last and tf[-1] not in acc[s] else ''}")
else:
line += " no accepted measurement"
if rej:
lo, hi = min(rej.items(), key=lambda x: x[1][k]), max(rej.items(), key=lambda x: x[1][k])
line += f"; {len(rej)} rejected: min {lo[1][k]:+.3g} at {iso(lo[0])}, max {hi[1][k]:+.3g} at {iso(hi[0])}"
if good or rej:
print(line)
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder")
a.add_argument("--columns", default="x,y,sigma0")
a.add_argument("--column", default="error_value", choices=COLUMNS)
a.add_argument("--step", default="15min", type=duration, help="step of the tables and of the blue curves (default 15min, the measurement cadence: a wider step flattens and shifts short events)")
a.add_argument("--k", type=float, default=5, help="deviation counted, in MAD of the window")
a.add_argument("--min", type=int, default=5, help="number of sensors from which a time is printed")
a.add_argument("--residuals", default="corr_x,corr_y,error_value", help="columns plotted minus each sensor's own level (\"no\" or '': none)")
a.add_argument("--outside", help="T1/T2: the own level of the residuals is computed outside this moment")
a.add_argument("--others", action="store_true", help="residuals also brought back to the median of the other sensors at the same time, with slope and correlation on the common movement")
a.add_argument("--rigid", type=parse_time, metavar="BASE", help="residuals against the overall movement of the targets (translation, rotation, scale) since time BASE")
a.add_argument("--without", default="", help="A,B: targets left out of the fit of --rigid (their residual is still plotted)")
a.add_argument("--follow", default="", help="A,B: with --rigid, dx, dy of these sensors added to the per-step table (the references are already there)")
a.add_argument("--table", default="", metavar="T1/T2|no",
help="per-step tables printed only between T1 and T2 (\"no\": not at all); the rest over the whole window")
a.add_argument("--sensors", help="comma-separated list (default: all)")
a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time)
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
o = a.parse_args()
by_sensor = load(o.folder)
if not by_sensor:
sys.exit(f"No \"advanced\" export in {o.folder} (see inventory.py).")
names = o.sensors.split(",") if o.sensors else sorted(by_sensor, key=lambda s: (not is_reference(by_sensor[s]), s))
columns = o.columns.split(",")
residuals = [c for c in o.residuals.split(",") if c and c != "no"] # '' or "no": none, as for --table
outside = tuple(parse_time(x) for x in o.outside.split("/")) if o.outside else None
if outside and len(outside) != 2:
sys.exit("--outside expects T1/T2.")
if o.table in ("", "no"):
o.in_table = lambda t: o.table == ""
else:
limits = [parse_time(x) for x in o.table.split("/")]
if len(limits) != 2:
sys.exit("--table expects T1/T2 or no.")
o.in_table = lambda t: limits[0] <= t <= limits[1]
for requested, known, what in ((names, by_sensor, "sensors"), ([s for s in o.follow.split(",") if s], names, "sensors of --follow (taken from --sensors)"), (columns + residuals + [o.column], COLUMNS, "columns")):
missing = [x for x in requested if x not in known]
if missing:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
series = {s: filter_window(by_sensor[s], o.start, o.end) for s in names}
if not any(series.values()):
sys.exit("No measurement in the requested window.")
times = sorted({m["t"] for s in names for m in series[s]})
poses = filter_window(load_camera(o.folder), times[0].replace(second=0, microsecond=0), times[-1]) # poses to the minute, measurements to the second
pose_cols = POSE if poses else ()
station = os.path.basename(os.path.normpath(o.folder))
os.makedirs(o.out, exist_ok=True)
base = os.path.join(o.out, f"{station}_{{}}_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png")
def deviations(col):
"""{sensor: {t: value - median of the window}}, and the MADs."""
e, mad = {}, {}
for s in names:
n = level(series[s], col)
if n:
e[s] = {m["t"]: m[col] - n[0] for m in series[s] if m[col] is not None}
mad[s] = n[1]
return e, mad
# 1. common
n_ax = len(pose_cols) + len(columns) + 1
fig, axes = plt.subplots(n_ax, 1, sharex=True, figsize=(14, 3 * n_ax), squeeze=False)
for ax, col in zip(axes[:, 0], pose_cols):
pose_t = {p["t"].replace(second=0, microsecond=0): p[col] for p in poses if p[col] is not None}
cam_t = sorted(set(pose_t) | {t.replace(second=0, microsecond=0) for t in times})
# line broken at measurement times without registration: it does not join two poses across a gap
ax.plot(cam_t, [pose_t.get(t, math.nan) for t in cam_t], ".-", ms=3, lw=0.5, color="C2", label="camera pose (one point per registration)")
ax.set_ylabel(f"camera: {col}" + (" [deg]" if col in ("pan", "tilt", "rotation") else ""))
ax.legend(loc="upper left", fontsize=8)
commons, refs_per_col, ref_devs = {}, {}, {}
for ax, col in zip(axes[len(pose_cols):, 0], columns):
e, _ = deviations(col)
med, low, high, tt = [], [], [], []
for t in times:
v = sorted(e[s][t] for s in e if t in e[s])
if len(v) >= 2:
q = statistics.quantiles(v, n=10, method="inclusive")
tt.append(t), med.append(statistics.median(v)), low.append(q[0]), high.append(q[-1])
commons[col] = dict(zip(tt, med))
ax.fill_between(tt, low, high, color="0.85", label="10th-90th percentile between sensors")
ax.plot(tt, med, ".-", ms=2, lw=0.8, label="median of the sensors")
refs = [s for s in e if is_reference(series[s])]
if refs and col != "sigma0": # sigma0 is the same for all: nothing to separate
# global references one by one, and median of the targets alone: what moves in the image, the references or everything
tgt = {}
for t in times:
v = [e[s][t] for s in e if s not in refs and t in e[s]]
if v:
tgt[t] = statistics.median(v)
commons[col + "_targets"] = tgt
ax.plot(list(tgt), list(tgt.values()), "-", lw=1.2, color="k", label="median of the targets alone")
for i, s in enumerate(refs):
ax.plot(list(e[s]), list(e[s].values()), "-", lw=0.9, color=f"C{(i + 1) % 10}", label=f"{s} (ref.)")
refs_per_col[col] = refs
ref_devs[col] = {s: e[s] for s in refs}
ax.set_ylabel(f"{col}\ndeviation from its median")
ax.grid(alpha=0.3)
ax.legend(loc="upper left", fontsize=8, ncol=1 + len(refs_per_col.get(col, [])) // 4)
e, mad = deviations(o.column)
count, who = [], {}
for t in times:
highs = [s for s in e if t in e[s] and mad[s] > 0 and e[s][t] > o.k * mad[s]]
count.append(len(highs))
if len(highs) >= o.min:
who[t] = highs
axes[-1, 0].step(times, count, where="mid", lw=0.8)
axes[-1, 0].set_ylabel(f"sensors above\nmedian + {o.k:g} MAD\n({o.column})")
axes[-1, 0].grid(alpha=0.3)
h = next((h for h in (1, 2, 3, 6, 12) if (times[-1] - times[0]).total_seconds() / 3600 / h <= 20), 24)
days = (times[-1] - times[0]).total_seconds() / 86400
# beyond 20 days, one tick per day overlaps: every n days, at most 20 ticks
axes[-1, 0].xaxis.set_major_locator(mdates.HourLocator(byhour=range(0, 24, h)) if h < 24 else mdates.DayLocator(interval=math.ceil(days / 20)))
axes[-1, 0].set_xlim(times[0], times[-1])
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
for ax in axes[:, 0]:
ax.grid(alpha=0.5, which="major")
fig.suptitle(f"{station}: {len(e)} sensors together, each against its level over the window (UTC times)")
fig.tight_layout(rect=(0, 0, 1, 1 - 0.4 / fig.get_figheight())) # room for the title
common = base.format("common")
fig.savefig(common, dpi=100, bbox_inches="tight")
plt.close(fig)
print(common)
# table of the common median, per step
midnight = times[0].replace(hour=0, minute=0, second=0, microsecond=0)
t0 = midnight + ((times[0] - midnight) // o.step) * o.step # steps aligned on the clock (hh:00, hh:15…), not on the 1st measurement
rows = {}
for col in columns:
for t, v in commons[col].items():
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col, []).append(v)
for col in pose_cols:
commons["cam_" + col] = {p["t"]: p[col] for p in poses if p[col] is not None}
for t, v in commons["cam_" + col].items():
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault("cam_" + col, []).append(v)
for col in refs_per_col:
for t, v in commons[col + "_targets"].items():
rows.setdefault(t0 + ((t - t0) // o.step) * o.step, {}).setdefault(col + "_targets", []).append(v)
table_cols = columns + [c + "_targets" for c in refs_per_col] + ["cam_" + c for c in pose_cols]
print(f"\nCommon median (deviation from the median of the window){' and raw camera pose (cam_, empty without registration)' if pose_cols else ''}, "
f"median per step of {int(o.step.total_seconds() // 60)} min{' (table: ' + o.table + ')' if o.table else ''}:")
header = f"{'time':17}" + "".join(f"{c:>13}" for c in table_cols)
for i, t in enumerate(t for t in sorted(rows) if o.in_table(t)):
if i % 24 == 0:
print(header)
print(f"{iso(t):17}" + "".join(f"{statistics.median(rows[t][c]):13.{6 if c.startswith('cam_') else 4}g}" if c in rows[t] else " " * 13 for c in table_cols))
for col in table_cols:
if commons[col]:
lo, hi = min(commons[col].items(), key=lambda x: x[1]), max(commons[col].items(), key=lambda x: x[1])
g = 6 if col.startswith("cam_") else 4
print(f"{col} (measurements): min {lo[1]:.{g}g} at {iso(lo[0])}, max {hi[1]:.{g}g} at {iso(hi[0])}")
for col, refs in refs_per_col.items():
print(f"\n{col} raw of the global references, deviation from their median of the window (measurements); {col}_targets: median of the targets alone:")
for s in refs:
d = ref_devs[col][s]
lo, hi = min(d.items(), key=lambda x: x[1]), max(d.items(), key=lambda x: x[1])
rej = {m["t"] for m in series[s] if rejected(m)}
r = lambda t: " (rejected)" if t in rej else ""
print(f"{s:9} min {lo[1]:+.3g} at {iso(lo[0])}{r(lo[0])}, max {hi[1]:+.3g} at {iso(hi[0])}{r(hi[0])}"
+ (f"; {len(rej)} rejected measurements in the window" if rej else ""))
if not pose_cols:
print("Camera pose: none in the window (no camera/ in the folder, or no registration).")
else:
posed = {p["t"].replace(second=0, microsecond=0) for p in poses}
runs = [] # runs of consecutive measurement times without a pose: [start, end, number]
for i, t in enumerate(times):
if t.replace(second=0, microsecond=0) in posed:
continue
if runs and runs[-1][3] == i - 1:
runs[-1][1], runs[-1][2], runs[-1][3] = t, runs[-1][2] + 1, i
else:
runs.append([t, t, 1, i])
print(f"Measurement times without a camera pose (no registration): {sum(x[2] for x in runs)} of {len(times)}"
+ (": " + "; ".join(iso(a) if n == 1 else f"{iso(a)} → {iso(b)} ({n})" for a, b, n, _ in runs) if runs else ""))
# 2. small multiples
col = o.column
n = len(names)
nrows = math.ceil(n / 5)
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
for ax, s in zip(axes.flat, names):
ms = [m for m in series[s] if m[col] is not None]
ax.plot([m["t"] for m in ms], [m[col] for m in ms], "-", lw=0.4, color="0.7")
steps = {}
for m in ms:
steps.setdefault(t0 + ((m["t"] - t0) // o.step) * o.step, []).append(m[col])
ax.plot(list(steps), [statistics.median(v) for v in steps.values()], "-", lw=1.1, color="C0")
lv = level(series[s], col)
if lv:
ax.axhline(lv[0], color="C3", ls=":", lw=0.8)
ax.set_title(s + (" (ref.)" if series[s] and is_reference(series[s]) else ""), fontsize=9)
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
for ax in list(axes.flat)[n:]:
ax.axis("off")
fig.suptitle(f"{station}: {col}, median per step of {int(o.step.total_seconds() // 60)} min (blue), measurements (grey), "
f"median of the window (dotted); one scale per sensor (UTC times)")
fig.tight_layout()
multiples = base.format(f"multiples-{col}")
fig.savefig(multiples, dpi=100, bbox_inches="tight")
print(multiples)
# 3. residuals: each sensor minus its own level
for col, others in [(c, False) for c in residuals] + [(c, True) for c in residuals if o.others]:
fig, axes = plt.subplots(nrows, 5, sharex=True, figsize=(18, 2.5 * nrows), squeeze=False)
dev = {}
if others: # {sensor: {t: value - own level}} for the median of the others
for q in names:
mq = [m for m in series[q] if m[col] is not None]
rq = [m for m in mq if not outside or not outside[0] <= m["t"] <= outside[1]] or mq
nq = level(rq, col)
if nq:
dev[q] = {m["t"]: m[col] - nq[0] for m in mq}
print(f"\n{col} minus the own level{' and minus the median of the other sensors at the same time' if others else ''} ({'outside ' + o.outside if outside else 'median of the window'}), median per step of "
f"{int(o.step.total_seconds() // 60)} min:")
slopes = []
print(f"{'sensor':9}{'level':>10}{'noise':>10}{'min dev.':>11} {'at':<17}{'(noises)':>9}{'max dev.':>11} {'at':<17}{'(noises)':>9}")
for ax, s in zip(axes.flat, names):
ms = [m for m in series[s] if m[col] is not None]
ref = [m for m in ms if not outside or not outside[0] <= m["t"] <= outside[1]] or ms
lv = level(ref, col)
if not lv:
ax.axis("off")
continue
noise = own_noise([m[col] for m in ms])
if others:
res = {}
for m in ms:
v = [dev[q][m["t"]] for q in dev if q != s and m["t"] in dev[q]]
if len(v) >= 2:
res[m["t"]] = m[col] - lv[0] - statistics.median(v)
if len(res) < 2:
ax.axis("off")
continue
else:
res = {m["t"]: m[col] - lv[0] for m in ms}
steps = {}
for t, v in res.items():
steps.setdefault(t0 + ((t - t0) // o.step) * o.step, []).append(v)
pm = {t: statistics.median(v) for t, v in steps.items()}
ax.plot(list(res), list(res.values()), "-", lw=0.4, color="0.7")
ax.plot(list(pm), list(pm.values()), "-", lw=1.1, color="C0")
ax.axhline(0, color="C3", ls=":", lw=0.8)
ax.set_title(s + (" (ref.)" if is_reference(ms) else "") + f" noise {noise:.2g}", fontsize=9)
ax.grid(alpha=0.3)
ax.tick_params(labelsize=7)
ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=2, maxticks=5))
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%Hh"))
lo, hi = min(pm.items(), key=lambda x: x[1]), max(pm.items(), key=lambda x: x[1])
f = lambda v: f"{v / noise:9.1f}" if noise else f"{'-':>9}"
print(f"{s:9}{lv[0]:10.4g}{noise:10.3g}{lo[1]:11.3g} {iso(lo[0]):<17}{f(lo[1])}{hi[1]:11.3g} {iso(hi[0]):<17}{f(hi[1])}")
if others:
for c in columns:
tt = [t for t in res if t in commons[c]]
if len(tt) > 2 and len({commons[c][t] for t in tt}) > 1:
xs, ys = [commons[c][t] for t in tt], [res[t] for t in tt]
slope = statistics.linear_regression(xs, ys).slope
slopes.append(f"{s:9}slope on common {c} {slope:+.4g} per unit, r = {statistics.correlation(xs, ys):+.2f}")
for ax in list(axes.flat)[n:]:
ax.axis("off")
if slopes:
print("\n".join(slopes))
fig.suptitle(f"{station}: {col} minus the own level{' and the median of the others' if others else ''} of each sensor, median per step of "
f"{int(o.step.total_seconds() // 60)} min (blue), measurements (grey); one scale per sensor (UTC times)")
fig.tight_layout()
path = base.format(f"residuals-{col}" + ("-others" if others else ""))
fig.savefig(path, dpi=100, bbox_inches="tight")
plt.close(fig)
print(path)
if o.rigid:
rigid(o, names, series, times, base, station)
print(f"\nTimes when at least {o.min} sensors have {o.column} > median of the window + {o.k:g} MAD:")
for t, highs in who.items():
print(f"{iso(t)} {len(highs):3} {','.join(highs)}")
if not who:
print("(none)")
if __name__ == "__main__":
main()
-156
View File
@@ -1,156 +0,0 @@
"""Plages datées par capteur et par colonne, au pas natif des mesures, pour l'agent de diagnostic.
Usage : python3 tools/scripts/plages.py data/PS_DES_COUDRES/ [--colonnes error_value,corr_x,corr_y]
[--k 5] [--fusion 1h] [--min-mesures 1] [--hors T1/T2] [--glissant N|DUREE] [--instants N]
[--capteurs A,B] [--debut ...] [--fin ...]
Pour chaque colonne, chaque capteur est comparé à son propre niveau sur la fenêtre --debut/--fin (médiane des
valeurs non vides ; avec --hors T1/T2, celle du reste de la fenêtre, hors du moment lu : utile quand un épisode
long fausse la médiane) et à son bruit propre (médiane des écarts entre deux mesures consécutives non vides, hors --hors).
1. Un tableau par colonne : niveau, bruit, et le plus grand saut d'une mesure à la suivante (valeur, en bruits,
les deux instants).
2. Les plages où la colonne s'écarte de son niveau de plus de --k bruits : début et fin (première et dernière
mesure hors niveau), nombre de mesures hors niveau, valeur extrême (écart signé au niveau, en unités et en
bruits) et son instant. Deux plages du même sens séparées de moins de --fusion (défaut 1h) n'en font qu'une
(un écart vers le bas puis vers le haut font deux plages) ; les mesures rejetées (colonne vide) ne coupent pas
une plage. Colonne « rejetées » : combien des mesures de la plage sont rejetées par la station (error_value
au-dessus du seuil), et « extrême rejetée » si l'extrême l'est ; un saut dont un bout est rejeté le dit.
corr_x, corr_y d'une mesure rejetée sont vides, références comprises (donnees.charger). Une plage qui touche le bord de la fenêtre est datée au
bord (le niveau est celui de la fenêtre : un palier durable peut faire de l'autre moitié la « plage »).
Avec --min-mesures N, seules les plages d'au moins N mesures hors niveau sont imprimées (leur nombre total aussi).
Avec --glissant N (un entier : N mesures non vides de part et d'autre) ou --glissant DUREE (6h, 1d : les mesures
à moins de DUREE avant et après), le niveau de chaque mesure est la médiane de ses voisines, sans elle (niveau
local) : une dérive longue ou un palier suivent le niveau, seuls restent les écarts plus brefs que la fenêtre.
La colonne « niveau » du tableau 1 reste la médiane de la fenêtre ; le bruit ne change pas.
3. Avec --instants N : les instants où au moins N capteurs sont hors niveau (sur l'une des colonnes), avec, pour
chacun, les capteurs et l'écart en bruits (cx corr_x, cy corr_y, ev error_value, sinon le nom de la colonne),
les capteurs rejetés à cet instant (parmi --capteurs : en retirer les capteurs toujours rejetés allège la liste),
et chaque référence de la station, --capteurs ou non : error_value (« r » si rejetée) / inlier (« - » vide).
Chaque colonne est lue aux mesures, jamais moyennée par pas. Rend des valeurs, pas de verdict.
"""
import argparse
import bisect
import statistics
import sys
from donnees import charger, est_reference, filtrer, instant, iso, rejetee
from resume import COLONNES, bruit_propre, duree
ABREGE = {"corr_x": "cx", "corr_y": "cy", "error_value": "ev"}
def locaux(ms, g):
"""Niveau local de chaque mesure (t, v) : médiane des voisines non vides, sans elle ; g entier (mesures de part et
d'autre) ou timedelta (mesures à moins de g) ; None sans voisine."""
ts, vs = [t for t, _ in ms], [v for _, v in ms]
out = []
for i, t in enumerate(ts):
lo, hi = (max(0, i - g), i + g + 1) if isinstance(g, int) else (bisect.bisect_left(ts, t - g), bisect.bisect_right(ts, t + g))
w = vs[lo:i] + vs[i + 1:hi]
out.append(statistics.median(w) if w else None)
return out
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossier")
a.add_argument("--colonnes", default="error_value,corr_x,corr_y")
a.add_argument("--k", type=float, default=5, help="écart au niveau compté, en bruits propres du capteur")
a.add_argument("--fusion", default="1h", type=duree, help="deux plages séparées de moins que cela n'en font qu'une")
a.add_argument("--min-mesures", type=int, default=1, help="n'imprime que les plages d'au moins N mesures hors niveau")
a.add_argument("--hors", help="T1/T2 : niveau et bruit se calculent hors de ce moment (les plages se cherchent partout)")
a.add_argument("--glissant", type=lambda x: int(x) if x.isdigit() else duree(x),
help="N (mesures de part et d'autre) ou durée (6h, 1d) : niveau local, médiane des voisines sans la mesure")
a.add_argument("--instants", type=int, help="N : instants où au moins N capteurs sont hors niveau, avec rejets et références")
a.add_argument("--capteurs", help="liste séparée par des virgules (défaut : tous)")
a.add_argument("--debut", type=instant)
a.add_argument("--fin", type=instant)
o = a.parse_args()
par = charger(o.dossier)
if not par:
sys.exit(f"Aucun export « advanced » dans {o.dossier} (voir inventaire.py).")
noms = o.capteurs.split(",") if o.capteurs else sorted(par, key=lambda s: (not est_reference(par[s]), s))
colonnes = o.colonnes.split(",")
hors = tuple(instant(x) for x in o.hors.split("/")) if o.hors else None
if hors and len(hors) != 2:
sys.exit("--hors attend T1/T2.")
for demandes, connus, quoi in ((noms, par, "Capteurs"), (colonnes, COLONNES, "Colonnes")):
absents = [x for x in demandes if x not in connus]
if absents:
sys.exit(f"{quoi} inconnus : {', '.join(absents)}. Connus : {', '.join(sorted(connus))}.")
series = {s: filtrer(par[s], o.debut, o.fin) for s in noms}
if not any(series.values()):
sys.exit("Aucune mesure dans la fenêtre demandée.")
glis = (f"{o.glissant} mesures" if isinstance(o.glissant, int) else f"{int(o.glissant.total_seconds() // 60)} min") if o.glissant is not None else ""
par_instant = {} # iso(t) -> ["capteur.col+écart"] (--instants)
for col in colonnes:
print(f"\n== {col} : niveau, bruit, plus grand saut d'une mesure à la suivante ==")
print(f"{'capteur':9}{'n':>5}{'niveau':>10}{'bruit':>10}{'saut':>10}{'(bruits)':>9} entre")
plages = []
for s in noms:
ms = [(m["t"], m[col]) for m in series[s] if m[col] is not None]
rej = {m["t"] for m in series[s] if rejetee(m)}
if len(ms) < 2:
print(f"{s:9}{len(ms):5} (moins de 2 valeurs)")
continue
ref = [x for x in ms if not hors or not hors[0] <= x[0] <= hors[1]]
if len(ref) < 2:
print(f"{s:9}{len(ms):5} (moins de 2 valeurs hors --hors)")
continue
niv = statistics.median(v for _, v in ref)
sauts = [(b[1] - a_[1], a_[0], b[0]) for a_, b in zip(ms, ms[1:])]
bruit = bruit_propre([v for _, v in ref])
d, t1, t2 = max(sauts, key=lambda x: abs(x[0]))
bru = f"{d / bruit:9.1f}" if bruit else f"{'-':>9}"
marque = [iso(x) for x in (t1, t2) if x in rej]
print(f"{s:9}{len(ms):5}{niv:10.4g}{bruit:10.3g}{d:+10.4g}{bru} {iso(t1)} et {iso(t2)}"
+ (f" (rejetée : {', '.join(marque)})" if marque else ""))
if not bruit:
continue
lv = locaux(ms, o.glissant) if o.glissant is not None else [niv] * len(ms)
groupe = [] # (t, écart au niveau)
for (t, v), n in zip(ms + [(None, 0)], lv + [None]):
if t is not None and n is not None and abs(v - n) > o.k * bruit:
if groupe and (t - groupe[-1][0] > o.fusion or (v > n) != (groupe[-1][1] > 0)): # même sens seul
plages.append((s, groupe, bruit, rej))
groupe = []
groupe.append((t, v - n))
par_instant.setdefault(iso(t), {}).setdefault(s, []).append(f"{ABREGE.get(col, col)}{(v - n) / bruit:+.0f}")
elif t is None and groupe:
plages.append((s, groupe, bruit, rej))
toutes = len(plages)
plages = [p for p in plages if len(p[1]) >= o.min_mesures]
print(f"\n{col} : plages à plus de {o.k:g} bruits du niveau{' glissant ±' + glis + ' (médiane des voisines)' if glis else ''}{' (niveau hors ' + o.hors + ')' if hors and not glis else ''}{' (bruit hors ' + o.hors + ')' if hors and glis else ''} (fusion {int(o.fusion.total_seconds() // 60)} min)"
+ (f" ; {len(plages)} d'au moins {o.min_mesures} mesures sur {toutes}" if o.min_mesures > 1 else ""))
print(f"{'capteur':9}{'début':<18}{'fin':<18}{'mesures':>8}{'extrême':>10}{'(bruits)':>9} {'à':<17}{'rejetées':>9}")
for s, g, bruit, rej in plages:
t, d = max(g, key=lambda x: abs(x[1]))
print(f"{s:9}{iso(g[0][0]):<18}{iso(g[-1][0]):<18}{len(g):8}{d:+10.4g}{d / bruit:9.1f} {iso(t):<17}"
f"{sum(x[0] in rej for x in g):9}{' extrême rejetée' if t in rej else ''}")
if not plages:
print("(aucune)")
if o.instants is not None:
refs = sorted(s for s in par if est_reference(par[s]))
rej_t, ref_t = {}, {s: {} for s in refs}
for s in set(noms) | set(refs):
for m in filtrer(par[s], o.debut, o.fin):
if rejetee(m) and s in noms:
rej_t.setdefault(iso(m["t"]), []).append(s)
if s in ref_t:
ref_t[s][iso(m["t"])] = m
f = lambda v, fmt: "-" if v is None else format(v, fmt)
lignes = [(t, c) for t, c in sorted(par_instant.items()) if len(c) >= o.instants]
print(f"\nInstants où au moins {o.instants} capteurs sont à plus de {o.k:g} bruits du niveau{' glissant ±' + glis if glis else ''} "
f"({', '.join(colonnes)}) : {len(lignes)} sur {len(par_instant)} instants avec au moins un capteur hors niveau")
print("instant n capteurs hors niveau (écart en bruits) | rejetés | références : error_value (r rejetée) / inlier")
for t, c in lignes:
r = " ".join(f"{s} {f(ref_t[s][t]['error_value'], '.3g')}{'r' if rejetee(ref_t[s][t]) else ''}/{f(ref_t[s][t]['inlier'], 'g')}"
for s in refs if t in ref_t[s])
print(f"{t} {len(c):3} {' '.join(s + '.' + ','.join(v) for s, v in c.items())} | {','.join(rej_t.get(t, [])) or '-'} | {r or '-'}")
if __name__ == "__main__":
main()
+93
View File
@@ -0,0 +1,93 @@
"""PNG plot of the series of an "advanced" export, for the diagnostic agent to look at.
Usage: .venv/bin/python tools/scripts/plot.py data/PS_DES_COUDRES/
[--sensors SENSOR_A,SENSOR_B] [--columns error_value,corr_x,corr_y] [--start ...] [--end ...]
[--zone START/END ...] [--out FOLDER]
One panel per column, one curve per sensor, UTC times; each --zone is greyed. Columns: error_value, corr_x, corr_y,
relative_x, relative_y, x, y, sigma0, reproj_err, and the six image quality metrics (peak to snr, raw values; against a calm reference: quality.py).
Prints the path of the PNG (out/diagnostic/, or --out: the report's folder, for an evidence figure).
"""
import argparse
import os
import sys
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402
from data import filter_window, is_reference, load, parse_time # noqa: E402
from summary import QUALITY # noqa: E402
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
def zone(text):
"""'2026-10-13T01:00Z/2026-10-13T03:00Z' -> (start, end)."""
start, sep, end = text.partition("/")
if not sep:
raise argparse.ArgumentTypeError(f"invalid zone: {text!r} (e.g. 2026-10-13T01:00Z/2026-10-13T03:00Z)")
return parse_time(start), parse_time(end)
def bound(t, default):
return t.strftime("%Y%m%dT%H%MZ") if t else default
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder")
a.add_argument("--sensors", help="comma-separated list (default: all)")
a.add_argument("--columns", default="error_value,corr_x,corr_y")
a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time)
a.add_argument("--zone", type=zone, action="append", default=[], help="window to grey, START/END (repeatable)")
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG")
o = a.parse_args()
by = load(o.folder)
if not by:
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
columns = o.columns.split(",")
for requested, known, what in ((names, by, "sensors"), (columns, COLUMNS + QUALITY, "columns")):
missing = [x for x in requested if x not in known]
if missing:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
series = {s: filter_window(by[s], o.start, o.end) for s in names}
if not any(series.values()):
sys.exit("No measurement in the requested window.")
empty = [c for c in columns if all(m[c] is None for s in names for m in series[s])]
if len(empty) == len(columns):
sys.exit(f"Nothing to plot: {', '.join(columns)} empty for these sensors in the window.")
if empty:
print(f"Empty panels (no value for these sensors): {', '.join(empty)}")
fig, axes = plt.subplots(len(columns), 1, sharex=True, figsize=(14, 3.2 * len(columns)), squeeze=False)
tab20 = plt.get_cmap("tab20").colors
colors = tab20[::2] + tab20[1::2] # dark shades first: two neighbouring sensors do not share a hue
for ax, col in zip(axes[:, 0], columns):
for i, s in enumerate(names):
ms = series[s] # an empty value becomes NaN: the curve breaks instead of bridging the gap
ax.plot([m["t"] for m in ms], [float("nan") if m[col] is None else m[col] for m in ms], ".-", ms=2, lw=0.7,
color=colors[i % len(colors)], label=s)
for start, end in o.zone:
ax.axvspan(start, end, color="0.85", zorder=0)
ax.set_ylabel(col)
ax.grid(alpha=0.3)
axes[0, 0].legend(loc="upper left", bbox_to_anchor=(1.0, 1.0), fontsize=8)
axes[-1, 0].xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
station = os.path.basename(os.path.normpath(o.folder))
fig.suptitle(f"{station}: {', '.join(columns)} (UTC times)")
os.makedirs(o.out, exist_ok=True)
who = "-".join(names) if o.sensors else "all"
path = os.path.join(o.out, f"{station}_{'-'.join(columns)}_{who}_"
f"{bound(o.start, 'start')}_{bound(o.end, 'end')}.png")
fig.savefig(path, dpi=100, bbox_inches="tight")
print(path)
if __name__ == "__main__":
main()
-417
View File
@@ -1,417 +0,0 @@
"""Les six métriques de qualité d'image (peak, flux, sharpness, fwhm, dominance, snr) par capteur, face à sa référence calme.
Usage : python3 tools/scripts/qualite.py <dossier> [--debut T] [--fin T] [--capteurs A,B] [--calme T1/T2,T3/T4]
[--commun] [--table METRIQUE] [--pas 15min] [--seuil 0.5]
python3 tools/scripts/qualite.py <dossier> --etiquettes labeling/auto/X.jsonl [--calme ...] [--debut T] [--fin T]
[--numeros 3,7] [--seuil 0.5] [--marge 60] [--bords] | [--hors [--min 1] [--sauf "Thermal displacement"]]
.venv/bin/python tools/scripts/qualite.py <dossier> --figure [--calme ...] [--debut T] [--fin T]
[--metriques flux,snr,dominance] [--etiquettes F] [--zone T1/T2 ...] [--sortie D]
python3 tools/scripts/qualite.py --essai (contrôle du calcul du rapport)
Définitions : labeling/connaissances/metriques-de-qualite.md. Colonnes remplies sur les caméras récentes seulement ; sur un dossier où
elles sont vides, l'outil le dit et s'arrête.
Référence d'un capteur : la médiane de chaque métrique sur ses mesures des fenêtres --calme, à la même heure UTC du jour
(24 créneaux ; un créneau sans mesure calme prend la médiane de toutes les heures calmes). Sans --calme : tout le dossier.
Chaque mesure devient un rapport valeur / référence : 1 = comme la référence ; < 1 plus bas ; > 1 plus haut. Pour fwhm et
dominance, c'est la hausse qui dit une image dégradée (tache plus large, second pic plus fort). Mesures rejetées comprises
(la qualité de leur image reste mesurée).
Sans --etiquettes ni --figure :
1. le peak maximal du dossier, puis le résumé par capteur : mesures rejetées, « pk/max » (peak médian du capteur sur la
fenêtre divisé par ce maximum : une tache au plafond de l'image a un peak constant), et pour chaque métrique le rapport médian sur la fenêtre et l'extrême (min pour peak, flux, sharpness, snr ; max pour
fwhm, dominance) avec son instant ; puis les lignes « cibles » et « références » : la médiane des rapports médians ;
2. --commun : par pas de --pas, la médiane des rapports des cibles et celle des références, pour chaque métrique ;
3. --table M : par pas, le rapport de M de chaque capteur (« * » : une mesure du pas rejetée), « méd » (médiane sur les
capteurs) et le nombre de capteurs sous --seuil (au-dessus de 1/--seuil pour fwhm et dominance).
--etiquettes F (un JSON par ligne, debut, fin, categorie), sans --figure : par étiquette (numérotée dans l'ordre du
fichier ; --debut/--fin gardent celles qui y commencent), le rapport médian des cibles et des références sur l'heure
d'avant, la plage et l'heure d'après (--marge minutes) : « cib 0.95|0.40|0.97 » ; puis les capteurs « touchés » sur la
plage (flux ou snr sous --seuil, ou dominance au-dessus de 1/--seuil), avec leur pire rapport de flux, du plus bas au plus
haut (r = référence). --bords : par instant de la plage élargie de --marge, le nombre de capteurs touchés et la médiane
des cibles de flux, snr, dominance (« < » : dans la plage). --hors : à la place, les suites d'instants de --debut à --fin
qu'aucune étiquette ne couvre (hors familles --sauf), où au moins --min capteurs sont touchés (instants à moins de 15 min
groupés), avec les capteurs touchés et leur pire flux.
--figure (matplotlib) : PNG « qualite-commun » : un panneau par métrique, échelle logarithmique, médiane des cibles (noir),
des références (rouge) et bande du 10e au 90e centile des cibles ; PNG « qualite-<métrique> » par --metriques : un petit
graphique par capteur, références en rouge. --etiquettes grise chaque étiquette d'une couleur par famille ; --zone grise
des plages. Trait coupé sur un trou de plus de 30 min. Imprime le chemin de chaque PNG.
Rend des valeurs, pas de verdict. Lancé depuis la racine du dépôt.
"""
import argparse
import bisect
import json
import os
import statistics
import sys
from datetime import timedelta
import donnees
from resume import QUALITE as METRIQUES, duree
HAUT_MAUVAIS = ("fwhm", "dominance") # pour ces deux-là, c'est la hausse qui dit une image dégradée
COULEURS = {"Visibility": "tab:blue", "Obstruction": "tab:orange", "Thermal displacement": "tab:red",
"Displacement": "tab:purple", "Target issue": "tab:brown", "No data": "black"}
def fenetres(texte):
"""'T1/T2,T3/T4' -> [(début, fin)]."""
return [tuple(donnees.instant(b) for b in f.split("/")) for f in texte.split(",")] if texte else []
def dans(t, fen):
return any(a <= t <= b for a, b in fen)
def references(par, calme):
"""{capteur: {métrique: {heure: médiane, None: médiane toutes heures}}} sur les mesures calmes."""
ref = {}
for s, ms in par.items():
ref[s] = {}
for c in METRIQUES:
parh, tout = {}, []
for m in ms:
if m[c] is None or (calme and not dans(m["t"], calme)):
continue
parh.setdefault(m["t"].hour, []).append(m[c])
tout.append(m[c])
r = {h: statistics.median(v) for h, v in parh.items()}
r[None] = statistics.median(tout) if tout else None
ref[s][c] = r
return ref
def rapport(ref, s, c, m):
r = ref[s][c].get(m["t"].hour, ref[s][c][None])
return m[c] / r if m[c] is not None and r else None
def touche(r, c, seuil):
return r is not None and ((c in ("flux", "snr") and r < seuil) or (c == "dominance" and r > 1 / seuil))
def med(v):
return f"{statistics.median(v):.2f}" if v else "-"
def resume(par, ref, noms, refs, pmax):
print(f"peak maximal du dossier : {pmax if pmax is None else f'{pmax:g}'} ; pk/max : peak médian du capteur sur la "
"fenêtre, divisé par ce maximum")
print("Résumé : rapport médian à la référence calme, puis l'extrême et son instant (UTC)")
print(f"{'capteur':>12} {'rej':>4} {'pk/max':>6} " + " ".join(f"{c:>26}" for c in METRIQUES))
meds = {c: {"cibles": [], "références": []} for c in METRIQUES}
for s in noms:
ms = par[s]
if not ms:
continue
cases = []
for c in METRIQUES:
rs = [(rapport(ref, s, c, m), m["t"]) for m in ms]
rs = [x for x in rs if x[0] is not None]
if not rs:
cases.append(f"{'vide':>26}")
continue
md = statistics.median(r for r, _ in rs)
ext = (max if c in HAUT_MAUVAIS else min)(rs, key=lambda x: x[0])
meds[c]["références" if s in refs else "cibles"].append(md)
cases.append(f"{md:6.2f} ({ext[0]:6.2f} {ext[1]:%d/%m %H:%M})")
rej = sum(1 for m in ms if donnees.rejetee(m))
pk = [m["peak"] for m in ms if m["peak"] is not None]
pk = f"{statistics.median(pk) / pmax:6.2f}" if pk and pmax else f"{'-':>6}"
print(f"{s + ('r' if s in refs else ''):>12} {rej:>4} {pk} " + " ".join(cases))
for g in ("cibles", "références"):
print(f"{g:>24} " + " ".join(f"{statistics.median(meds[c][g]):26.2f}" if meds[c][g] else f"{'-':>26}"
for c in METRIQUES))
def par_pas(par, noms, pas):
pas = duree(pas)
debut = min(m["t"] for s in noms for m in par[s]).replace(hour=0, minute=0, second=0, microsecond=0)
return lambda t: debut + (t - debut) // pas * pas
def table(par, ref, noms, c, pas, seuil):
cle, cases = par_pas(par, noms, pas), {}
for s in noms:
for m in par[s]:
r = rapport(ref, s, c, m)
if r is not None:
cases.setdefault(cle(m["t"]), {}).setdefault(s, []).append((r, donnees.rejetee(m)))
haut = c in HAUT_MAUVAIS
print(f"\nTable de {c} / référence, pas de {pas} ('*' : une mesure du pas rejetée)")
w = max(7, *map(len, noms))
entete = f"{'instant':16} " + " ".join(f"{s:>{w}}" for s in noms) + f" {'méd':>6} {'n' + ('>' if haut else '<') + str(seuil):>6}"
for i, k in enumerate(sorted(cases)):
if i % 24 == 0:
print(entete)
ligne, vals = [], []
for s in noms:
v = cases[k].get(s)
if not v:
ligne.append(f"{'':>{w}}")
continue
r = statistics.median(x for x, _ in v)
vals.append(r)
ligne.append(f"{r:{w - 1}.2f}" + ("*" if any(j for _, j in v) else " "))
n = sum(1 for r in vals if (r > 1 / seuil if haut else r < seuil))
print(f"{k:%Y-%m-%d %H:%M} " + " ".join(ligne) + f" {statistics.median(vals):6.2f} {n:>6}")
def commun(par, ref, noms, refs, pas):
"""Par pas : médiane des rapports des cibles, puis des références, pour chaque métrique."""
cle, cases = par_pas(par, noms, pas), {}
for s in noms:
for m in par[s]:
d = cases.setdefault(cle(m["t"]), {c: ([], []) for c in METRIQUES})
for c in METRIQUES:
r = rapport(ref, s, c, m)
if r is not None:
d[c][s in refs].append(r)
print(f"\nMédianes des rapports par pas de {pas} : cibles / références")
entete = f"{'instant':16} " + " ".join(f"{c:>13}" for c in METRIQUES)
for i, k in enumerate(sorted(cases)):
if i % 24 == 0:
print(entete)
print(f"{k:%Y-%m-%d %H:%M} " + " ".join(
f"{statistics.median(cases[k][c][0]) if cases[k][c][0] else float('nan'):6.2f}/"
f"{statistics.median(cases[k][c][1]) if cases[k][c][1] else float('nan'):<6.2f}" for c in METRIQUES))
def lire_etiquettes(chemin):
return [json.loads(ligne) for ligne in open(chemin) if ligne.strip()]
def etiquettes(a, tout, ref, refs, noms, deb, fin):
marge = timedelta(minutes=a.marge)
temps = {s: [m["t"] for m in tout[s]] for s in noms}
garder = {int(n) for n in a.numeros.split(",")} if a.numeros else None
print(f"Référence calme : {a.calme or 'tout le dossier'} ; rapports médians avant|plage|après ({a.marge} min)")
for n, e in enumerate(lire_etiquettes(a.etiquettes), 1):
t0, t1 = donnees.instant(e["debut"]), donnees.instant(e["fin"])
if (garder and n not in garder) or (deb and t0 < deb) or (fin and t0 > fin):
continue
print(f"\n{n}. {e['debut']} → {e['fin']}, {e['categorie']}")
win = {s: tout[s][bisect.bisect_left(temps[s], t0 - marge):bisect.bisect_right(temps[s], t1 + marge)] for s in noms}
parties = [(t0 - marge, t0 - timedelta(seconds=1)), (t0, t1), (t1 + timedelta(seconds=1), t1 + marge)]
for c in METRIQUES:
cases = []
for g in (False, True):
vals = []
for a0, a1 in parties:
v = [rapport(ref, s, c, m) for s in win if (s in refs) == g for m in win[s] if a0 <= m["t"] <= a1]
vals.append(med([x for x in v if x is not None]))
cases.append(("ref " if g else "cib ") + "|".join(vals))
print(f" {c:>9} " + " ".join(cases))
pires = []
for s in noms:
ms = [m for m in win[s] if t0 <= m["t"] <= t1]
if any(touche(rapport(ref, s, c, m), c, a.seuil) for m in ms for c in ("flux", "snr", "dominance")):
fl = [x for x in (rapport(ref, s, "flux", m) for m in ms) if x is not None]
pires.append((min(fl) if fl else 1, s + ("r" if s in refs else "")))
pires.sort()
print(f" touchés ({len(pires)}) : " + ", ".join(f"{s} {f:.2f}" for f, s in pires) if pires else " touchés : aucun")
if a.bords:
inst = {}
for s in win:
for m in win[s]:
d = inst.setdefault(m["t"], {"n": 0, "flux": [], "snr": [], "dominance": []})
rs = {c: rapport(ref, s, c, m) for c in ("flux", "snr", "dominance")}
d["n"] += any(touche(rs[c], c, a.seuil) for c in rs)
if s not in refs:
for c in rs:
if rs[c] is not None:
d[c].append(rs[c])
print(" instant n flux snr dom")
for t in sorted(inst):
d = inst[t]
print(f" {t:%Y-%m-%d %H:%M} {d['n']:>3} {med(d['flux']):>5} {med(d['snr']):>5} {med(d['dominance']):>5}"
+ (" <" if t0 <= t <= t1 else ""))
def hors(a, tout, ref, refs, noms, deb, fin):
sauf = set(a.sauf.split(",")) if a.sauf else set()
couvert = [(donnees.instant(e["debut"]), donnees.instant(e["fin"])) for e in lire_etiquettes(a.etiquettes)
if e["categorie"] not in sauf]
inst = {}
for s in noms:
for m in donnees.filtrer(tout[s], deb, fin):
if dans(m["t"], couvert):
continue
if any(touche(rapport(ref, s, c, m), c, a.seuil) for c in ("flux", "snr", "dominance")):
inst.setdefault(m["t"], {})[s + ("r" if s in refs else "")] = rapport(ref, s, "flux", m) or 0
suites = []
for t in sorted(t for t in inst if len(inst[t]) >= a.min):
if suites and t - suites[-1][-1] <= timedelta(minutes=15):
suites[-1].append(t)
else:
suites.append([t])
print(f"Hors des étiquettes (sauf {a.sauf}) : instants où au moins {a.min} capteurs sont touchés ; {len(suites)} suites")
for su in suites:
pire = {}
for t in su:
for s, f in inst[t].items():
pire[s] = min(pire.get(s, 9), f)
print(f"{su[0]:%Y-%m-%d %H:%M} → {su[-1]:%H:%M} ({len(su)}) : "
+ ", ".join(f"{s} {f:.2f}" for s, f in sorted(pire.items(), key=lambda x: x[1])))
def morceaux(pts, trou=timedelta(minutes=30)):
"""[(t, ...)] -> listes coupées là où deux instants sont à plus de `trou` : un trait ne traverse pas un trou de mesures."""
out = []
for i, x in enumerate(pts):
if not out or x[0] - pts[i - 1][0] > trou:
out.append([])
out[-1].append(x)
return out
def figure(a, par, ref, noms, refs, deb, fin):
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
plages = [(x, y, "grey") for x, y in fenetres(",".join(a.zone or []))]
if a.etiquettes:
for e in lire_etiquettes(a.etiquettes):
x, y = donnees.instant(e["debut"]), donnees.instant(e["fin"])
if (deb is None or y >= deb) and (fin is None or x <= fin):
plages.append((x, y, COULEURS.get(e["categorie"], "grey")))
station = os.path.basename(os.path.normpath(a.dossier))
t0 = min(m["t"] for s in noms for m in par[s])
t1 = max(m["t"] for s in noms for m in par[s])
suffixe = f"{t0:%Y%m%dT%H%MZ}_{t1:%Y%m%dT%H%MZ}"
os.makedirs(a.sortie, exist_ok=True)
def axe(ax):
for x, y, couleur in plages:
ax.axvspan(x, y, color=couleur, alpha=0.12, lw=0)
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%M"))
ax.set_yscale("log")
ax.axhline(1, ls=":", c="k", lw=0.8)
ax.set_xlim(t0, t1)
fig, axes = plt.subplots(len(METRIQUES), 1, figsize=(16, 2.4 * len(METRIQUES)), sharex=True)
for ax, c in zip(axes, METRIQUES):
inst = {}
for s in noms:
for m in par[s]:
r = rapport(ref, s, c, m)
if r is not None and r > 0:
inst.setdefault(m["t"], ([], []))[s in refs].append(r)
ts = sorted(inst)
cib = [(t, sorted(inst[t][0])) for t in ts if inst[t][0]]
rfs = [(t, statistics.median(inst[t][1])) for t in ts if inst[t][1]]
for i, mc in enumerate(morceaux(cib)):
ax.fill_between([t for t, _ in mc], [v[len(v) // 10] for _, v in mc], [v[(len(v) * 9) // 10] for _, v in mc],
color="0.8", lw=0, step="mid")
ax.plot([t for t, _ in mc], [statistics.median(v) for _, v in mc], c="k", lw=0.7,
label="cibles (médiane, bande 10-90 %)" if i == 0 else None)
for i, mc in enumerate(morceaux(rfs)):
ax.plot([t for t, _ in mc], [v for _, v in mc], c="tab:red", lw=0.7, label="références (médiane)" if i == 0 else None)
axe(ax)
ax.set_ylabel(c)
axes[0].legend(loc="upper right", fontsize=8)
axes[0].set_title(f"{station} : métriques de qualité / référence calme ({a.calme or 'tout le dossier'}), même heure UTC")
chemin = os.path.join(a.sortie, f"{station}_qualite-commun_{suffixe}.png")
fig.tight_layout()
fig.savefig(chemin, dpi=90)
plt.close(fig)
print(chemin)
for c in a.metriques.split(","):
col = 6
lig = (len(noms) + col - 1) // col
fig, axes = plt.subplots(lig, col, figsize=(3.2 * col, 1.9 * lig), sharex=True, squeeze=False)
for ax, s in zip(axes.flat, noms):
pts = [(m["t"], rapport(ref, s, c, m)) for m in par[s]]
pts = [(t, r) for t, r in pts if r is not None and r > 0]
for mc in morceaux(pts):
ax.plot([t for t, _ in mc], [r for _, r in mc], lw=0.5, c="tab:red" if s in refs else "k")
axe(ax)
ax.set_title(s, fontsize=8, color="tab:red" if s in refs else "k")
ax.tick_params(labelsize=6)
for ax in list(axes.flat)[len(noms):]:
ax.axis("off")
fig.suptitle(f"{station} : {c} / référence calme, par capteur (références en rouge)")
chemin = os.path.join(a.sortie, f"{station}_qualite-{c}_{suffixe}.png")
fig.tight_layout()
fig.savefig(chemin, dpi=80)
plt.close(fig)
print(chemin)
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("dossier")
p.add_argument("--debut", type=donnees.instant)
p.add_argument("--fin", type=donnees.instant)
p.add_argument("--capteurs", help="liste séparée par des virgules (défaut : tous)")
p.add_argument("--calme", help="fenêtres de la référence, T1/T2,T3/T4 (défaut : tout le dossier)")
p.add_argument("--commun", action="store_true", help="médianes des cibles et des références par pas")
p.add_argument("--table", choices=METRIQUES, help="rapport d'une métrique par pas et par capteur")
p.add_argument("--pas", default="15min", type=lambda x: duree(x) and x, help="15min, 1h, 1d")
p.add_argument("--seuil", type=float, default=0.5, help="« touché » : flux ou snr sous seuil, dominance au-dessus de 1/seuil")
p.add_argument("--etiquettes", help="fichier d'étiquettes (JSON par ligne)")
p.add_argument("--numeros", help="numéros d'étiquettes à lire, 3,7")
p.add_argument("--marge", type=int, default=60, help="minutes avant et après chaque étiquette")
p.add_argument("--bords", action="store_true")
p.add_argument("--hors", action="store_true", help="instants touchés qu'aucune étiquette ne couvre")
p.add_argument("--min", type=int, default=1)
p.add_argument("--sauf", default="Thermal displacement", help="familles ignorées par --hors, A,B")
p.add_argument("--figure", action="store_true", help="PNG (matplotlib : .venv/bin/python)")
p.add_argument("--metriques", default="flux,snr,dominance", help="PNG par capteur de --figure")
p.add_argument("--zone", action="append", help="plage à griser sur les PNG, T1/T2 (répétable)")
p.add_argument("--sortie", default=os.path.join("out", "diagnostic"), help="dossier des PNG")
a = p.parse_args()
if a.hors and not a.etiquettes:
p.error("--hors demande --etiquettes")
inconnues = [c for c in a.metriques.split(",") if c not in METRIQUES]
if inconnues:
p.error(f"métriques inconnues : {', '.join(inconnues)} ; connues : {', '.join(METRIQUES)}")
tout = donnees.charger(a.dossier)
if not tout:
sys.exit(f"Aucun export « advanced » dans {a.dossier} (voir inventaire.py).")
if not any(m[c] is not None for ms in tout.values() for m in ms for c in METRIQUES):
sys.exit(f"Aucune métrique de qualité dans {a.dossier} : peak à snr vides (caméra ancienne ; "
"labeling/connaissances/metriques-de-qualite.md).")
noms = a.capteurs.split(",") if a.capteurs else sorted(tout, key=lambda s: (not donnees.est_reference(tout[s]), s))
absents = [s for s in noms if s not in tout]
if absents:
sys.exit(f"Capteurs absents : {', '.join(absents)}. Présents : {', '.join(sorted(tout))}.")
ref = references(tout, fenetres(a.calme))
refs = {s for s in tout if donnees.est_reference(tout[s])}
par = {s: donnees.filtrer(tout[s], a.debut, a.fin) for s in noms}
if not any(par.values()):
sys.exit("Aucune mesure dans la fenêtre demandée.")
if a.figure:
return figure(a, par, ref, noms, refs, a.debut, a.fin)
if a.etiquettes:
return (hors if a.hors else etiquettes)(a, tout, ref, refs, noms, a.debut, a.fin)
pmax = max((m["peak"] for ms in tout.values() for m in ms if m["peak"] is not None), default=None)
print(f"Référence calme : {a.calme or 'tout le dossier'}, par heure UTC du jour (r = référence)")
resume(par, ref, noms, refs, pmax)
if a.commun:
commun(par, ref, noms, refs, a.pas)
if a.table:
table(par, ref, noms, a.table, a.pas, a.seuil)
def essai():
"""Contrôle du rapport à la référence : même heure, repli sur toutes les heures, fenêtre calme respectée."""
from datetime import datetime, timezone
t = lambda j, h: datetime(2026, 10, j, h, tzinfo=timezone.utc) # noqa: E731
vide = dict.fromkeys(METRIQUES)
par = {"A": [dict(vide, t=t(1, 3), flux=10.0), dict(vide, t=t(2, 3), flux=30.0), dict(vide, t=t(2, 5), flux=100.0),
dict(vide, t=t(9, 3), flux=1000.0)]}
ref = references(par, [(t(1, 0), t(3, 0))])
assert ref["A"]["flux"][3] == 20.0 and ref["A"]["flux"][None] == 30.0, ref
assert rapport(ref, "A", "flux", dict(vide, t=t(9, 3), flux=10.0)) == 0.5
assert rapport(ref, "A", "flux", dict(vide, t=t(9, 7), flux=60.0)) == 2.0 # heure sans calme : toutes les heures
assert rapport(ref, "A", "snr", dict(vide, t=t(9, 3), snr=5.0)) is None
assert touche(0.4, "flux", 0.5) and touche(2.5, "dominance", 0.5) and not touche(2.5, "flux", 0.5)
print("essai : ok")
if __name__ == "__main__":
essai() if sys.argv[1:] == ["--essai"] else main()
+417
View File
@@ -0,0 +1,417 @@
"""The six image-quality metrics (peak, flux, sharpness, fwhm, dominance, snr) per sensor, against its calm reference.
Usage: python3 tools/scripts/quality.py <folder> [--start T] [--end T] [--sensors A,B] [--calm T1/T2,T3/T4]
[--common] [--table METRIC] [--step 15min] [--threshold 0.5]
python3 tools/scripts/quality.py <folder> --labels labeling/auto/X.jsonl [--calm ...] [--start T] [--end T]
[--numbers 3,7] [--threshold 0.5] [--margin 60] [--edges] | [--outside [--min 1] [--except "Thermal displacement"]]
.venv/bin/python tools/scripts/quality.py <folder> --figure [--calm ...] [--start T] [--end T]
[--metrics flux,snr,dominance] [--labels F] [--zone T1/T2 ...] [--out D]
python3 tools/scripts/quality.py --self-test (check of the ratio computation)
Definitions: labeling/knowledge/quality-metrics.md. Columns filled on recent cameras only; on a folder where
they are empty, the tool says so and stops.
Reference of a sensor: the median of each metric over its measurements in the --calm windows, at the same UTC hour of the day
(24 slots; a slot without a calm measurement takes the median of all calm hours). Without --calm: the whole folder.
Each measurement becomes a ratio value / reference: 1 = like the reference; < 1 lower; > 1 higher. For fwhm and
dominance, it is the rise that says a degraded image (wider spot, stronger second peak). Rejected measurements included
(the quality of their image is still measured).
Without --labels or --figure:
1. the maximum peak of the folder, then the summary per sensor: rejected measurements, "pk/max" (median peak of the sensor over the
window divided by this maximum: a spot at the ceiling of the image has a constant peak), and for each metric the median ratio over the window and the extreme (min for peak, flux, sharpness, snr; max for
fwhm, dominance) with its time; then the lines "targets" and "references": the median of the median ratios;
2. --common: per step of --step, the median of the ratios of the targets and that of the references, for each metric;
3. --table M: per step, the ratio of M of each sensor ("*": a measurement of the step rejected), "med" (median over the
sensors) and the number of sensors below --threshold (above 1/--threshold for fwhm and dominance).
--labels F (one JSON per line, start, end, family), without --figure: per label (numbered in the order of the
file; --start/--end keep those that start within), the median ratio of the targets and of the references over the hour
before, the range and the hour after (--margin minutes): "tgt 0.95|0.40|0.97"; then the sensors "affected" over the
range (flux or snr below --threshold, or dominance above 1/--threshold), with their worst flux ratio, from lowest to
highest (r = reference). --edges: per time of the range widened by --margin, the number of affected sensors and the median
of the targets of flux, snr, dominance ("<": within the range). --outside: instead, the runs of times from --start to --end
that no label covers (except families --except), where at least --min sensors are affected (times less than 15 min apart
grouped), with the affected sensors and their worst flux.
--figure (matplotlib): PNG "quality-common": one panel per metric, logarithmic scale, median of the targets (black),
of the references (red) and band from the 10th to the 90th percentile of the targets; PNG "quality-<metric>" per --metrics: one small
plot per sensor, references in red. --labels shades each label in one colour per family; --zone shades
ranges. Line broken over a gap of more than 30 min. Prints the path of each PNG.
Gives values, not a verdict. Run from the root of the repository.
"""
import argparse
import bisect
import json
import os
import statistics
import sys
from datetime import timedelta
import data
from summary import QUALITY as METRICS, duration
HIGH_BAD = ("fwhm", "dominance") # for these two, it is the rise that says a degraded image
COLOURS = {"Visibility": "tab:blue", "Obstruction": "tab:orange", "Thermal displacement": "tab:red",
"Displacement": "tab:purple", "Target issue": "tab:brown", "No data": "black"}
def windows(text):
"""'T1/T2,T3/T4' -> [(start, end)]."""
return [tuple(data.parse_time(b) for b in f.split("/")) for f in text.split(",")] if text else []
def within(t, wins):
return any(a <= t <= b for a, b in wins)
def references(by_sensor, calm):
"""{sensor: {metric: {hour: median, None: median over all hours}}} over the calm measurements."""
ref = {}
for s, ms in by_sensor.items():
ref[s] = {}
for c in METRICS:
per_hour, every = {}, []
for m in ms:
if m[c] is None or (calm and not within(m["t"], calm)):
continue
per_hour.setdefault(m["t"].hour, []).append(m[c])
every.append(m[c])
r = {h: statistics.median(v) for h, v in per_hour.items()}
r[None] = statistics.median(every) if every else None
ref[s][c] = r
return ref
def ratio(ref, s, c, m):
r = ref[s][c].get(m["t"].hour, ref[s][c][None])
return m[c] / r if m[c] is not None and r else None
def affected(r, c, threshold):
return r is not None and ((c in ("flux", "snr") and r < threshold) or (c == "dominance" and r > 1 / threshold))
def med(v):
return f"{statistics.median(v):.2f}" if v else "-"
def summary(by_sensor, ref, names, refs, pmax):
print(f"maximum peak of the folder: {pmax if pmax is None else f'{pmax:g}'}; pk/max: median peak of the sensor over the "
"window, divided by this maximum")
print("Summary: median ratio to the calm reference, then the extreme and its time (UTC)")
print(f"{'sensor':>12} {'rej':>4} {'pk/max':>6} " + " ".join(f"{c:>26}" for c in METRICS))
meds = {c: {"targets": [], "references": []} for c in METRICS}
for s in names:
ms = by_sensor[s]
if not ms:
continue
cells = []
for c in METRICS:
rs = [(ratio(ref, s, c, m), m["t"]) for m in ms]
rs = [x for x in rs if x[0] is not None]
if not rs:
cells.append(f"{'empty':>26}")
continue
md = statistics.median(r for r, _ in rs)
ext = (max if c in HIGH_BAD else min)(rs, key=lambda x: x[0])
meds[c]["references" if s in refs else "targets"].append(md)
cells.append(f"{md:6.2f} ({ext[0]:6.2f} {ext[1]:%d/%m %H:%M})")
rej = sum(1 for m in ms if data.rejected(m))
pk = [m["peak"] for m in ms if m["peak"] is not None]
pk = f"{statistics.median(pk) / pmax:6.2f}" if pk and pmax else f"{'-':>6}"
print(f"{s + ('r' if s in refs else ''):>12} {rej:>4} {pk} " + " ".join(cells))
for g in ("targets", "references"):
print(f"{g:>24} " + " ".join(f"{statistics.median(meds[c][g]):26.2f}" if meds[c][g] else f"{'-':>26}"
for c in METRICS))
def per_step(by_sensor, names, step):
step = duration(step)
start = min(m["t"] for s in names for m in by_sensor[s]).replace(hour=0, minute=0, second=0, microsecond=0)
return lambda t: start + (t - start) // step * step
def table(by_sensor, ref, names, c, step, threshold):
key, cells = per_step(by_sensor, names, step), {}
for s in names:
for m in by_sensor[s]:
r = ratio(ref, s, c, m)
if r is not None:
cells.setdefault(key(m["t"]), {}).setdefault(s, []).append((r, data.rejected(m)))
high = c in HIGH_BAD
print(f"\nTable of {c} / reference, step of {step} ('*': a measurement of the step rejected)")
w = max(7, *map(len, names))
header = f"{'time':16} " + " ".join(f"{s:>{w}}" for s in names) + f" {'med':>6} {'n' + ('>' if high else '<') + str(threshold):>6}"
for i, k in enumerate(sorted(cells)):
if i % 24 == 0:
print(header)
line, vals = [], []
for s in names:
v = cells[k].get(s)
if not v:
line.append(f"{'':>{w}}")
continue
r = statistics.median(x for x, _ in v)
vals.append(r)
line.append(f"{r:{w - 1}.2f}" + ("*" if any(j for _, j in v) else " "))
n = sum(1 for r in vals if (r > 1 / threshold if high else r < threshold))
print(f"{k:%Y-%m-%d %H:%M} " + " ".join(line) + f" {statistics.median(vals):6.2f} {n:>6}")
def common(by_sensor, ref, names, refs, step):
"""Per step: median of the ratios of the targets, then of the references, for each metric."""
key, cells = per_step(by_sensor, names, step), {}
for s in names:
for m in by_sensor[s]:
d = cells.setdefault(key(m["t"]), {c: ([], []) for c in METRICS})
for c in METRICS:
r = ratio(ref, s, c, m)
if r is not None:
d[c][s in refs].append(r)
print(f"\nMedians of the ratios per step of {step}: targets / references")
header = f"{'time':16} " + " ".join(f"{c:>13}" for c in METRICS)
for i, k in enumerate(sorted(cells)):
if i % 24 == 0:
print(header)
print(f"{k:%Y-%m-%d %H:%M} " + " ".join(
f"{statistics.median(cells[k][c][0]) if cells[k][c][0] else float('nan'):6.2f}/"
f"{statistics.median(cells[k][c][1]) if cells[k][c][1] else float('nan'):<6.2f}" for c in METRICS))
def read_labels(path):
return [json.loads(line) for line in open(path) if line.strip()]
def labels(a, every, ref, refs, names, start, end):
margin = timedelta(minutes=a.margin)
times = {s: [m["t"] for m in every[s]] for s in names}
keep = {int(n) for n in a.numbers.split(",")} if a.numbers else None
print(f"Calm reference: {a.calm or 'the whole folder'}; median ratios before|range|after ({a.margin} min)")
for n, e in enumerate(read_labels(a.labels), 1):
t0, t1 = data.parse_time(e["start"]), data.parse_time(e["end"])
if (keep and n not in keep) or (start and t0 < start) or (end and t0 > end):
continue
print(f"\n{n}. {e['start']} → {e['end']}, {e['family']}")
win = {s: every[s][bisect.bisect_left(times[s], t0 - margin):bisect.bisect_right(times[s], t1 + margin)] for s in names}
parts = [(t0 - margin, t0 - timedelta(seconds=1)), (t0, t1), (t1 + timedelta(seconds=1), t1 + margin)]
for c in METRICS:
cells = []
for g in (False, True):
vals = []
for a0, a1 in parts:
v = [ratio(ref, s, c, m) for s in win if (s in refs) == g for m in win[s] if a0 <= m["t"] <= a1]
vals.append(med([x for x in v if x is not None]))
cells.append(("ref " if g else "tgt ") + "|".join(vals))
print(f" {c:>9} " + " ".join(cells))
worst = []
for s in names:
ms = [m for m in win[s] if t0 <= m["t"] <= t1]
if any(affected(ratio(ref, s, c, m), c, a.threshold) for m in ms for c in ("flux", "snr", "dominance")):
fl = [x for x in (ratio(ref, s, "flux", m) for m in ms) if x is not None]
worst.append((min(fl) if fl else 1, s + ("r" if s in refs else "")))
worst.sort()
print(f" affected ({len(worst)}): " + ", ".join(f"{s} {f:.2f}" for f, s in worst) if worst else " affected: none")
if a.edges:
inst = {}
for s in win:
for m in win[s]:
d = inst.setdefault(m["t"], {"n": 0, "flux": [], "snr": [], "dominance": []})
rs = {c: ratio(ref, s, c, m) for c in ("flux", "snr", "dominance")}
d["n"] += any(affected(rs[c], c, a.threshold) for c in rs)
if s not in refs:
for c in rs:
if rs[c] is not None:
d[c].append(rs[c])
print(" time n flux snr dom")
for t in sorted(inst):
d = inst[t]
print(f" {t:%Y-%m-%d %H:%M} {d['n']:>3} {med(d['flux']):>5} {med(d['snr']):>5} {med(d['dominance']):>5}"
+ (" <" if t0 <= t <= t1 else ""))
def outside(a, every, ref, refs, names, start, end):
excluded = set(a.except_.split(",")) if a.except_ else set()
covered = [(data.parse_time(e["start"]), data.parse_time(e["end"])) for e in read_labels(a.labels)
if e["family"] not in excluded]
inst = {}
for s in names:
for m in data.filter_window(every[s], start, end):
if within(m["t"], covered):
continue
if any(affected(ratio(ref, s, c, m), c, a.threshold) for c in ("flux", "snr", "dominance")):
inst.setdefault(m["t"], {})[s + ("r" if s in refs else "")] = ratio(ref, s, "flux", m) or 0
runs = []
for t in sorted(t for t in inst if len(inst[t]) >= a.min):
if runs and t - runs[-1][-1] <= timedelta(minutes=15):
runs[-1].append(t)
else:
runs.append([t])
print(f"Outside the labels (except {a.except_}): times when at least {a.min} sensors are affected; {len(runs)} runs")
for ru in runs:
worst = {}
for t in ru:
for s, f in inst[t].items():
worst[s] = min(worst.get(s, 9), f)
print(f"{ru[0]:%Y-%m-%d %H:%M} → {ru[-1]:%H:%M} ({len(ru)}): "
+ ", ".join(f"{s} {f:.2f}" for s, f in sorted(worst.items(), key=lambda x: x[1])))
def pieces(pts, gap=timedelta(minutes=30)):
"""[(t, ...)] -> lists broken where two times are more than `gap` apart: a line does not cross a measurement gap."""
out = []
for i, x in enumerate(pts):
if not out or x[0] - pts[i - 1][0] > gap:
out.append([])
out[-1].append(x)
return out
def figure(a, by_sensor, ref, names, refs, start, end):
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
ranges = [(x, y, "grey") for x, y in windows(",".join(a.zone or []))]
if a.labels:
for e in read_labels(a.labels):
x, y = data.parse_time(e["start"]), data.parse_time(e["end"])
if (start is None or y >= start) and (end is None or x <= end):
ranges.append((x, y, COLOURS.get(e["family"], "grey")))
station = os.path.basename(os.path.normpath(a.folder))
t0 = min(m["t"] for s in names for m in by_sensor[s])
t1 = max(m["t"] for s in names for m in by_sensor[s])
suffix = f"{t0:%Y%m%dT%H%MZ}_{t1:%Y%m%dT%H%MZ}"
os.makedirs(a.out, exist_ok=True)
def axis(ax):
for x, y, colour in ranges:
ax.axvspan(x, y, color=colour, alpha=0.12, lw=0)
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%M"))
ax.set_yscale("log")
ax.axhline(1, ls=":", c="k", lw=0.8)
ax.set_xlim(t0, t1)
fig, axes = plt.subplots(len(METRICS), 1, figsize=(16, 2.4 * len(METRICS)), sharex=True)
for ax, c in zip(axes, METRICS):
inst = {}
for s in names:
for m in by_sensor[s]:
r = ratio(ref, s, c, m)
if r is not None and r > 0:
inst.setdefault(m["t"], ([], []))[s in refs].append(r)
ts = sorted(inst)
tgt = [(t, sorted(inst[t][0])) for t in ts if inst[t][0]]
rfs = [(t, statistics.median(inst[t][1])) for t in ts if inst[t][1]]
for i, mc in enumerate(pieces(tgt)):
ax.fill_between([t for t, _ in mc], [v[len(v) // 10] for _, v in mc], [v[(len(v) * 9) // 10] for _, v in mc],
color="0.8", lw=0, step="mid")
ax.plot([t for t, _ in mc], [statistics.median(v) for _, v in mc], c="k", lw=0.7,
label="targets (median, 10-90 % band)" if i == 0 else None)
for i, mc in enumerate(pieces(rfs)):
ax.plot([t for t, _ in mc], [v for _, v in mc], c="tab:red", lw=0.7, label="references (median)" if i == 0 else None)
axis(ax)
ax.set_ylabel(c)
axes[0].legend(loc="upper right", fontsize=8)
axes[0].set_title(f"{station}: quality metrics / calm reference ({a.calm or 'the whole folder'}), same UTC hour")
path = os.path.join(a.out, f"{station}_quality-common_{suffix}.png")
fig.tight_layout()
fig.savefig(path, dpi=90)
plt.close(fig)
print(path)
for c in a.metrics.split(","):
col = 6
nrows = (len(names) + col - 1) // col
fig, axes = plt.subplots(nrows, col, figsize=(3.2 * col, 1.9 * nrows), sharex=True, squeeze=False)
for ax, s in zip(axes.flat, names):
pts = [(m["t"], ratio(ref, s, c, m)) for m in by_sensor[s]]
pts = [(t, r) for t, r in pts if r is not None and r > 0]
for mc in pieces(pts):
ax.plot([t for t, _ in mc], [r for _, r in mc], lw=0.5, c="tab:red" if s in refs else "k")
axis(ax)
ax.set_title(s, fontsize=8, color="tab:red" if s in refs else "k")
ax.tick_params(labelsize=6)
for ax in list(axes.flat)[len(names):]:
ax.axis("off")
fig.suptitle(f"{station}: {c} / calm reference, per sensor (references in red)")
path = os.path.join(a.out, f"{station}_quality-{c}_{suffix}.png")
fig.tight_layout()
fig.savefig(path, dpi=80)
plt.close(fig)
print(path)
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("folder")
p.add_argument("--start", type=data.parse_time)
p.add_argument("--end", type=data.parse_time)
p.add_argument("--sensors", help="comma-separated list (default: all)")
p.add_argument("--calm", help="windows of the reference, T1/T2,T3/T4 (default: the whole folder)")
p.add_argument("--common", action="store_true", help="medians of the targets and of the references per step")
p.add_argument("--table", choices=METRICS, help="ratio of one metric per step and per sensor")
p.add_argument("--step", default="15min", type=lambda x: duration(x) and x, help="15min, 1h, 1d")
p.add_argument("--threshold", type=float, default=0.5, help="\"affected\": flux or snr below threshold, dominance above 1/threshold")
p.add_argument("--labels", help="label file (one JSON per line)")
p.add_argument("--numbers", help="numbers of the labels to read, 3,7")
p.add_argument("--margin", type=int, default=60, help="minutes before and after each label")
p.add_argument("--edges", action="store_true")
p.add_argument("--outside", action="store_true", help="affected times that no label covers")
p.add_argument("--min", type=int, default=1)
p.add_argument("--except", dest="except_", default="Thermal displacement", help="families ignored by --outside, A,B")
p.add_argument("--figure", action="store_true", help="PNG (matplotlib: .venv/bin/python)")
p.add_argument("--metrics", default="flux,snr,dominance", help="per-sensor PNGs of --figure")
p.add_argument("--zone", action="append", help="range to shade on the PNGs, T1/T2 (repeatable)")
p.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNGs")
a = p.parse_args()
if a.outside and not a.labels:
p.error("--outside requires --labels")
unknown = [c for c in a.metrics.split(",") if c not in METRICS]
if unknown:
p.error(f"unknown metrics: {', '.join(unknown)}; known: {', '.join(METRICS)}")
every = data.load(a.folder)
if not every:
sys.exit(f"No \"advanced\" export in {a.folder} (see inventory.py).")
if not any(m[c] is not None for ms in every.values() for m in ms for c in METRICS):
sys.exit(f"No quality metrics in {a.folder}: peak to snr empty (old camera; "
"labeling/knowledge/quality-metrics.md).")
names = a.sensors.split(",") if a.sensors else sorted(every, key=lambda s: (not data.is_reference(every[s]), s))
missing = [s for s in names if s not in every]
if missing:
sys.exit(f"Missing sensors: {', '.join(missing)}. Present: {', '.join(sorted(every))}.")
ref = references(every, windows(a.calm))
refs = {s for s in every if data.is_reference(every[s])}
by_sensor = {s: data.filter_window(every[s], a.start, a.end) for s in names}
if not any(by_sensor.values()):
sys.exit("No measurement in the requested window.")
if a.figure:
return figure(a, by_sensor, ref, names, refs, a.start, a.end)
if a.labels:
return (outside if a.outside else labels)(a, every, ref, refs, names, a.start, a.end)
pmax = max((m["peak"] for ms in every.values() for m in ms if m["peak"] is not None), default=None)
print(f"Calm reference: {a.calm or 'the whole folder'}, per UTC hour of the day (r = reference)")
summary(by_sensor, ref, names, refs, pmax)
if a.common:
common(by_sensor, ref, names, refs, a.step)
if a.table:
table(by_sensor, ref, names, a.table, a.step, a.threshold)
def self_test():
"""Check of the ratio to the reference: same hour, fallback on all hours, calm window respected."""
from datetime import datetime, timezone
t = lambda j, h: datetime(2026, 10, j, h, tzinfo=timezone.utc) # noqa: E731
empty = dict.fromkeys(METRICS)
by_sensor = {"A": [dict(empty, t=t(1, 3), flux=10.0), dict(empty, t=t(2, 3), flux=30.0), dict(empty, t=t(2, 5), flux=100.0),
dict(empty, t=t(9, 3), flux=1000.0)]}
ref = references(by_sensor, [(t(1, 0), t(3, 0))])
assert ref["A"]["flux"][3] == 20.0 and ref["A"]["flux"][None] == 30.0, ref
assert ratio(ref, "A", "flux", dict(empty, t=t(9, 3), flux=10.0)) == 0.5
assert ratio(ref, "A", "flux", dict(empty, t=t(9, 7), flux=60.0)) == 2.0 # hour without calm: all hours
assert ratio(ref, "A", "snr", dict(empty, t=t(9, 3), snr=5.0)) is None
assert affected(0.4, "flux", 0.5) and affected(2.5, "dominance", 0.5) and not affected(2.5, "flux", 0.5)
print("self-test: ok")
if __name__ == "__main__":
self_test() if sys.argv[1:] == ["--self-test"] else main()
+157
View File
@@ -0,0 +1,157 @@
"""Dated ranges per sensor and per column, at the native step of the measurements, for the diagnostic agent.
Usage: python3 tools/scripts/ranges.py data/PS_DES_COUDRES/ [--columns error_value,corr_x,corr_y]
[--k 5] [--merge 1h] [--min-measurements 1] [--outside T1/T2] [--rolling N|DURATION] [--instants N]
[--sensors A,B] [--start ...] [--end ...]
For each column, each sensor is compared with its own level over the --start/--end window (median of the
non-empty values; with --outside T1/T2, that of the rest of the window, outside the moment being read: useful when a
long episode skews the median) and with its own noise (median of the differences between two consecutive non-empty
measurements, outside --outside).
1. One table per column: level, noise, and the largest jump from one measurement to the next (value, in noises,
the two instants).
2. The ranges where the column deviates from its level by more than --k noises: start and end (first and last
off-level measurement), number of off-level measurements, extreme value (signed deviation from the level, in units
and in noises) and its instant. Two ranges in the same direction less than --merge apart (default 1h) make
only one (a deviation downwards then upwards makes two ranges); rejected measurements (empty column) do not cut
a range. Column "rejected": how many of the measurements of the range are rejected by the station (error_value
above the threshold), and "rejected extreme" if the extreme is; a jump with a rejected end says so.
corr_x, corr_y of a rejected measurement are empty, references included (data.load). A range that touches the edge
of the window is dated at the edge (the level is the window's: a lasting plateau can make the other half the "range").
With --min-measurements N, only the ranges of at least N off-level measurements are printed (their total number too).
With --rolling N (an integer: N non-empty measurements on either side) or --rolling DURATION (6h, 1d: the
measurements less than DURATION before and after), the level of each measurement is the median of its neighbours,
without it (local level): a long drift or a plateau follows the level, only the deviations shorter than the window
remain. The "level" column of table 1 stays the median of the window; the noise does not change.
3. With --instants N: the instants where at least N sensors are off level (on one of the columns), with, for
each, the sensors and the deviation in noises (cx corr_x, cy corr_y, ev error_value, otherwise the column name),
the sensors rejected at that instant (among --sensors: removing the sensors that are always rejected shortens the
list), and each reference of the station, in --sensors or not: error_value ("r" if rejected) / inlier ("-" empty).
Each column is read at the measurements, never averaged per step. Gives values, not a verdict.
"""
import argparse
import bisect
import statistics
import sys
from data import filter_window, is_reference, iso, load, parse_time, rejected
from summary import COLUMNS, duration, own_noise
ABBREV = {"corr_x": "cx", "corr_y": "cy", "error_value": "ev"}
def local_levels(ms, g):
"""Local level of each measurement (t, v): median of the non-empty neighbours, without it; g an integer
(measurements on either side) or a timedelta (measurements less than g away); None without a neighbour."""
ts, vs = [t for t, _ in ms], [v for _, v in ms]
out = []
for i, t in enumerate(ts):
lo, hi = (max(0, i - g), i + g + 1) if isinstance(g, int) else (bisect.bisect_left(ts, t - g), bisect.bisect_right(ts, t + g))
w = vs[lo:i] + vs[i + 1:hi]
out.append(statistics.median(w) if w else None)
return out
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder")
a.add_argument("--columns", default="error_value,corr_x,corr_y")
a.add_argument("--k", type=float, default=5, help="deviation from the level that counts, in the sensor's own noises")
a.add_argument("--merge", default="1h", type=duration, help="two ranges closer than this make only one")
a.add_argument("--min-measurements", type=int, default=1, help="prints only the ranges of at least N off-level measurements")
a.add_argument("--outside", help="T1/T2: level and noise are computed outside this moment (ranges are searched everywhere)")
a.add_argument("--rolling", type=lambda x: int(x) if x.isdigit() else duration(x),
help="N (measurements on either side) or duration (6h, 1d): local level, median of the neighbours without the measurement")
a.add_argument("--instants", type=int, help="N: instants where at least N sensors are off level, with rejections and references")
a.add_argument("--sensors", help="comma-separated list (default: all)")
a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time)
o = a.parse_args()
by = load(o.folder)
if not by:
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
columns = o.columns.split(",")
outside = tuple(parse_time(x) for x in o.outside.split("/")) if o.outside else None
if outside and len(outside) != 2:
sys.exit("--outside expects T1/T2.")
for requested, known, what in ((names, by, "sensors"), (columns, COLUMNS, "columns")):
missing = [x for x in requested if x not in known]
if missing:
sys.exit(f"Unknown {what}: {', '.join(missing)}. Known: {', '.join(sorted(known))}.")
series = {s: filter_window(by[s], o.start, o.end) for s in names}
if not any(series.values()):
sys.exit("No measurement in the requested window.")
roll = (f"{o.rolling} measurements" if isinstance(o.rolling, int) else f"{int(o.rolling.total_seconds() // 60)} min") if o.rolling is not None else ""
by_instant = {} # iso(t) -> ["sensor.col+deviation"] (--instants)
for col in columns:
print(f"\n== {col}: level, noise, largest jump from one measurement to the next ==")
print(f"{'sensor':9}{'n':>5}{'level':>10}{'noise':>10}{'jump':>10}{'(noises)':>9} between")
ranges = []
for s in names:
ms = [(m["t"], m[col]) for m in series[s] if m[col] is not None]
rej = {m["t"] for m in series[s] if rejected(m)}
if len(ms) < 2:
print(f"{s:9}{len(ms):5} (fewer than 2 values)")
continue
ref = [x for x in ms if not outside or not outside[0] <= x[0] <= outside[1]]
if len(ref) < 2:
print(f"{s:9}{len(ms):5} (fewer than 2 values outside --outside)")
continue
lvl = statistics.median(v for _, v in ref)
jumps = [(b[1] - a_[1], a_[0], b[0]) for a_, b in zip(ms, ms[1:])]
noise = own_noise([v for _, v in ref])
d, t1, t2 = max(jumps, key=lambda x: abs(x[0]))
in_noises = f"{d / noise:9.1f}" if noise else f"{'-':>9}"
flagged = [iso(x) for x in (t1, t2) if x in rej]
print(f"{s:9}{len(ms):5}{lvl:10.4g}{noise:10.3g}{d:+10.4g}{in_noises} {iso(t1)} and {iso(t2)}"
+ (f" (rejected: {', '.join(flagged)})" if flagged else ""))
if not noise:
continue
lv = local_levels(ms, o.rolling) if o.rolling is not None else [lvl] * len(ms)
group = [] # (t, deviation from the level)
for (t, v), n in zip(ms + [(None, 0)], lv + [None]):
if t is not None and n is not None and abs(v - n) > o.k * noise:
if group and (t - group[-1][0] > o.merge or (v > n) != (group[-1][1] > 0)): # same direction only
ranges.append((s, group, noise, rej))
group = []
group.append((t, v - n))
by_instant.setdefault(iso(t), {}).setdefault(s, []).append(f"{ABBREV.get(col, col)}{(v - n) / noise:+.0f}")
elif t is None and group:
ranges.append((s, group, noise, rej))
total = len(ranges)
ranges = [p for p in ranges if len(p[1]) >= o.min_measurements]
print(f"\n{col}: ranges more than {o.k:g} noises from the level{' rolling ±' + roll + ' (median of the neighbours)' if roll else ''}{' (level outside ' + o.outside + ')' if outside and not roll else ''}{' (noise outside ' + o.outside + ')' if outside and roll else ''} (merge {int(o.merge.total_seconds() // 60)} min)"
+ (f"; {len(ranges)} of at least {o.min_measurements} measurements out of {total}" if o.min_measurements > 1 else ""))
print(f"{'sensor':9}{'start':<18}{'end':<18}{'measurements':>13}{'extreme':>10}{'(noises)':>9} {'at':<17}{'rejected':>9}")
for s, g, noise, rej in ranges:
t, d = max(g, key=lambda x: abs(x[1]))
print(f"{s:9}{iso(g[0][0]):<18}{iso(g[-1][0]):<18}{len(g):13}{d:+10.4g}{d / noise:9.1f} {iso(t):<17}"
f"{sum(x[0] in rej for x in g):9}{' rejected extreme' if t in rej else ''}")
if not ranges:
print("(none)")
if o.instants is not None:
refs = sorted(s for s in by if is_reference(by[s]))
rej_t, ref_t = {}, {s: {} for s in refs}
for s in set(names) | set(refs):
for m in filter_window(by[s], o.start, o.end):
if rejected(m) and s in names:
rej_t.setdefault(iso(m["t"]), []).append(s)
if s in ref_t:
ref_t[s][iso(m["t"])] = m
f = lambda v, fmt: "-" if v is None else format(v, fmt)
rows = [(t, c) for t, c in sorted(by_instant.items()) if len(c) >= o.instants]
print(f"\nInstants where at least {o.instants} sensors are more than {o.k:g} noises from the level{' rolling ±' + roll if roll else ''} "
f"({', '.join(columns)}): {len(rows)} of {len(by_instant)} instants with at least one sensor off level")
print("instant n sensors off level (deviation in noises) | rejected | references: error_value (r rejected) / inlier")
for t, c in rows:
r = " ".join(f"{s} {f(ref_t[s][t]['error_value'], '.3g')}{'r' if rejected(ref_t[s][t]) else ''}/{f(ref_t[s][t]['inlier'], 'g')}"
for s in refs if t in ref_t[s])
print(f"{t} {len(c):3} {' '.join(s + '.' + ','.join(v) for s, v in c.items())} | {','.join(rej_t.get(t, [])) or '-'} | {r or '-'}")
if __name__ == "__main__":
main()
-124
View File
@@ -1,124 +0,0 @@
"""Résumé chiffré compact des séries d'un export « advanced », pour l'agent de diagnostic.
Usage : python3 tools/scripts/resume.py data/PS_DES_COUDRES/ [--colonne error_value]
[--stat median|min|max] [--pas 1h] [--niveau periode|fenetre] [--capteurs CAPTEUR_A,CAPTEUR_B] [--debut 2026-01-01T00:00Z] [--fin ...]
[--instants T1,T2] [--marque]
1. Niveau habituel de chaque capteur : médiane, MAD (écart absolu médian), min, max, sur toute la période
(défaut) ou, avec --niveau fenetre, sur la fenêtre --debut/--fin seulement (un changement de niveau
dans la fenêtre ne se compare pas à des semaines d'autres mesures).
Colonnes : error_value, corr_x, corr_y, relative_x, relative_y, x, y, sigma0, reproj_err, et les six métriques de qualité d'image (peak à snr, valeurs brutes ; vides sur les
caméras anciennes ; face à une référence calme : qualite.py).
2. Tableau : une ligne par pas de temps (début du pas, UTC), une colonne par capteur, la statistique
choisie sur les mesures du pas. « rej » : toutes les mesures du pas sont rejetées ; « vide » : des mesures,
mais la colonne est vide (ex. reproj_err d'une cible) ; « - » : aucune mesure. L'en-tête des capteurs se répète
toutes les 24 lignes. Avec --instants T1,T2 : seulement les pas qui contiennent ces instants. Avec --marque : « * »
après la valeur quand une mesure du pas est rejetée (x, y, error_value d'une mesure rejetée restent remplis),
« o » quand une mesure du pas a inlier = 0 (référence écartée par la compensation).
Rend des valeurs, pas de verdict.
"""
import argparse
import re
import statistics
import sys
from datetime import timedelta
from donnees import charger, est_reference, filtrer, instant, iso, rejetee
COLONNES = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
QUALITE = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # qualité de l'image : caméras récentes ; lue par qualite.py
STATS = {"median": statistics.median, "min": min, "max": max}
def duree(texte):
"""'15min', '1h', '6h', '1d' -> timedelta."""
m = re.fullmatch(r"(\d+)(min|h|d)", texte)
if not m:
raise argparse.ArgumentTypeError(f"pas invalide : {texte!r} (ex. 15min, 1h, 1d)")
n, unite = int(m.group(1)), m.group(2)
return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unite]
ARRONDI = 1e-9 # en deçà, un écart entre deux mesures est l'arrondi du calcul de la station, pas un bruit
def bruit_propre(vs):
"""Médiane des écarts absolus entre deux valeurs consécutives ; 0 quand elle est sous ARRONDI (valeurs constantes
à l'arrondi près : le corr des références quand la compensation passe exactement par elles, deux références
retenues), pour qu'un écart ne s'écrive pas en milliards de bruits ; 0 aussi avec moins de deux valeurs."""
d = [abs(b - a) for a, b in zip(vs, vs[1:])]
b = statistics.median(d) if d else 0
return b if b >= ARRONDI else 0
def valeurs(ms, col):
return [m[col] for m in ms if m[col] is not None]
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossier")
a.add_argument("--colonne", default="error_value", choices=COLONNES + QUALITE)
a.add_argument("--stat", default="median", choices=STATS)
a.add_argument("--pas", default="1h", type=duree)
a.add_argument("--niveau", default="periode", choices=("periode", "fenetre"),
help="période sur laquelle médiane et MAD sont calculées (défaut : toutes les mesures)")
a.add_argument("--capteurs", help="liste séparée par des virgules (défaut : tous)")
a.add_argument("--instants", help="T1,T2 : seulement les pas qui contiennent ces instants")
a.add_argument("--marque", action="store_true", help="* : une mesure du pas rejetée ; o : une mesure du pas avec inlier = 0")
a.add_argument("--debut", type=instant)
a.add_argument("--fin", type=instant)
o = a.parse_args()
par = charger(o.dossier)
if not par:
sys.exit(f"Aucun export « advanced » dans {o.dossier} (voir inventaire.py).")
noms = o.capteurs.split(",") if o.capteurs else sorted(par, key=lambda s: (not est_reference(par[s]), s))
inconnus = [s for s in noms if s not in par]
if inconnus:
sys.exit(f"Capteurs absents : {', '.join(inconnus)}. Présents : {', '.join(sorted(par))}.")
t0 = min(m["t"] for s in noms for m in par[s]).replace(hour=0, minute=0, second=0, microsecond=0)
cases = {}
for s in noms:
for m in filtrer(par[s], o.debut, o.fin):
debut_du_pas = t0 + ((m["t"] - t0) // o.pas) * o.pas
cases.setdefault(debut_du_pas, {}).setdefault(s, []).append(m)
if not cases:
sys.exit("Aucune mesure dans la fenêtre demandée.")
sur = "la fenêtre --debut/--fin" if o.niveau == "fenetre" else "toute la période"
print(f"Colonne : {o.colonne}. Niveau habituel sur {sur} (mesures non vides) :")
print(f"{'capteur':10} {'n':>5} {'médiane':>10} {'MAD':>10} {'min':>10} {'max':>10}")
for s in noms:
v = valeurs(filtrer(par[s], o.debut, o.fin) if o.niveau == "fenetre" else par[s], o.colonne)
if not v:
print(f"{s:10} {0:5} (aucune valeur)")
continue
med = statistics.median(v)
mad = statistics.median(abs(x - med) for x in v)
print(f"{s:10} {len(v):5} {med:10.5g} {mad:10.5g} {min(v):10.5g} {max(v):10.5g}")
print(f"\n{o.stat} de {o.colonne} par pas de {o.pas} (début du pas, UTC) :")
print("rej : mesures du pas toutes rejetées ; vide : colonne vide ; - : aucune mesure")
if o.marque:
print("* : une mesure du pas au moins est rejetée ; o : une mesure du pas au moins a inlier = 0")
entete = f"{'début':17} " + " ".join(f"{s:>9}" for s in noms)
voulus = {t0 + ((instant(x) - t0) // o.pas) * o.pas for x in o.instants.split(",")} if o.instants else None
for i, k in enumerate(k for k in sorted(cases) if voulus is None or k in voulus):
if i % 24 == 0:
print(entete)
cellules = []
for s in noms:
ms = cases[k].get(s, [])
v = valeurs(ms, o.colonne)
if v:
marque = ("*" if any(map(rejetee, ms)) else "") + ("o" if any(m["inlier"] == 0 for m in ms) else "") if o.marque else ""
cellules.append(f"{STATS[o.stat](v):.5g}{marque}".rjust(9))
else:
cellules.append(f"{'-' if not ms else 'rej' if all(map(rejetee, ms)) else 'vide':>9}")
print(f"{iso(k):17} " + " ".join(cellules))
if __name__ == "__main__":
main()
+127 -127
View File
@@ -1,26 +1,26 @@
"""Plusieurs stations sur une même ligne de temps : température du boîtier, pose de la caméra, corr des cibles, rejets, trous.
"""Several stations on one timeline: enclosure temperature, camera pose, corr of the targets, rejections, gaps.
Usage : .venv/bin/python tools/scripts/stations.py data/BREITHORN data/MICHIGAN data/NEBRASKA
[--colonnes temperature,cam_tilt,cam_rotation,cam_scale,cib_corr_x,cib_corr_y] [--pas 1h] [--stat median|min|max]
[--table T1/T2|non] [--debut ...] [--fin ...] [--sortie DOSSIER]
Usage: .venv/bin/python tools/scripts/stations.py data/BREITHORN data/MICHIGAN data/NEBRASKA
[--columns temperature,cam_tilt,cam_rotation,cam_scale,tgt_corr_x,tgt_corr_y] [--step 1h] [--stat median|min|max]
[--table T1/T2|no] [--start ...] [--end ...] [--out FOLDER]
Pour chaque station (un dossier de station chacune), sur la fenêtre --debut/--fin :
- colonnes possibles : temperature (médiane des composants sensor_*_deg_c, comme metriques.py), cam_pan, cam_tilt,
cam_rotation, cam_scale, cam_sigma0 (pose brute de la caméra), cib_corr_x, cib_corr_y (médiane sur les cibles de
corr moins le niveau propre de chaque cible, mesures acceptées, comme metriques.py), rej_cibles, rej_refs (nombre de
cibles, de références rejetées : le plus grand des instants de mesure du pas) ;
- imprime, par station, la période des mesures, la cadence (pas médian) et le nombre d'instants ; puis les **trous** :
les suites de pas de --pas sans mesure ou sans température, avec les stations où elles manquent (un trou commun à
plusieurs stations y est sur une seule ligne) ; puis la table, une ligne par pas (début du pas, UTC ; --stat des
valeurs du pas ; « - » : aucune valeur), une colonne par station et par colonne, séparées par des tabulations
(--table T1/T2 : seulement entre T1 et T2, « non » : pas du tout ; le reste porte sur toute la fenêtre) ; puis, pour
chaque colonne et chaque paire de stations, la corrélation r des valeurs et celle des pas (différence d'une valeur à
la suivante, sur les pas où les deux stations ont une valeur : un mouvement commun se voit sur les pas, une tendance
commune sur les valeurs), avec le nombre de pas ;
- un PNG « stations » : un panneau par colonne, un trait par station (température brute ; les autres colonnes moins
leur médiane sur la fenêtre, pour se superposer), coupé sur un trou ; trous de mesure grisés sous l'axe, une bande
par station.
Jamais d'identifiant de metrics/ (SIM_CCID, ip*) : seules les températures y sont lues. Rend des valeurs, pas de verdict.
For each station (one station folder each), over the window --start/--end:
- possible columns: temperature (median of the sensor_*_deg_c components, as in metrics.py), cam_pan, cam_tilt,
cam_rotation, cam_scale, cam_sigma0 (raw camera pose), tgt_corr_x, tgt_corr_y (median over the targets of
corr minus each target's own level, accepted measurements, as in metrics.py), rej_targets, rej_refs (number of
rejected targets, of rejected references: the largest over the measurement times of the step);
- prints, per station, the period of the measurements, the cadence (median step) and the number of times; then the **gaps**:
the runs of steps of --step without a measurement or without temperature, with the stations where they are missing (a gap shared by
several stations is on a single line); then the table, one line per step (start of the step, UTC; --stat of the
values of the step; "-": no value), one column per station and per column, tab-separated
(--table T1/T2: only between T1 and T2, "no": not at all; the rest covers the whole window); then, for
each column and each pair of stations, the correlation r of the values and that of the steps (difference from one value to
the next, over the steps where both stations have a value: a common movement shows on the steps, a common
trend on the values), with the number of steps;
- one PNG "stations": one panel per column, one line per station (raw temperature; the other columns minus
their median over the window, so that they overlap), broken over a gap; measurement gaps shaded below the axis, one band
per station.
Never an identifier from metrics/ (SIM_CCID, ip*): only the temperatures are read there. Gives values, not a verdict.
"""
import argparse
import os
@@ -33,31 +33,31 @@ matplotlib.use("Agg")
import matplotlib.dates as mdates # noqa: E402
import matplotlib.pyplot as plt # noqa: E402
from donnees import POSE, charger, charger_camera, charger_temperature, est_reference, filtrer, instant, iso, rejetee # noqa: E402
from graphique import borne # noqa: E402
from metriques import cibles, coupe, suites # noqa: E402
from resume import STATS, duree # noqa: E402
from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time, rejected # noqa: E402
from metrics import cut, runs, targets # noqa: E402
from plot import bound # noqa: E402
from summary import STATS, duration # noqa: E402
COLONNES = ("temperature",) + tuple("cam_" + c for c in POSE) + ("cib_corr_x", "cib_corr_y", "rej_cibles", "rej_refs")
COLUMNS = ("temperature",) + tuple("cam_" + c for c in POSE) + ("tgt_corr_x", "tgt_corr_y", "rej_targets", "rej_refs")
def lire(dossier, debut, fin):
"""Dossier de station -> ({colonne: [{'t', 'v'}]}, [instants de mesure])."""
par = charger(dossier)
series = {"temperature": filtrer([{"t": t, "v": v} for t, v in charger_temperature(dossier)], debut, fin)}
for p in filtrer(charger_camera(dossier), debut, fin):
def read(folder, start, end):
"""Station folder -> ({column: [{'t', 'v'}]}, [measurement times])."""
by_sensor = load(folder)
series = {"temperature": filter_window([{"t": t, "v": v} for t, v in load_temperature(folder)], start, end)}
for p in filter_window(load_camera(folder), start, end):
for c in POSE:
if p[c] is not None:
series.setdefault("cam_" + c, []).append({"t": p["t"], "v": p[c]})
for c in ("corr_x", "corr_y"):
series["cib_" + c] = cibles(par, c, debut, fin)
rej = {} # t -> [cibles rejetées, références rejetées]
for s, ms in par.items():
ref = est_reference(ms)
for m in filtrer(ms, debut, fin):
series["tgt_" + c] = targets(by_sensor, c, start, end)
rej = {} # t -> [rejected targets, rejected references]
for s, ms in by_sensor.items():
ref = is_reference(ms)
for m in filter_window(ms, start, end):
n = rej.setdefault(m["t"], [0, 0])
n[ref] += rejetee(m)
series["rej_cibles"] = [{"t": t, "v": float(n[0])} for t, n in sorted(rej.items())]
n[ref] += rejected(m)
series["rej_targets"] = [{"t": t, "v": float(n[0])} for t, n in sorted(rej.items())]
series["rej_refs"] = [{"t": t, "v": float(n[1])} for t, n in sorted(rej.items())]
return series, sorted(rej)
@@ -71,123 +71,123 @@ def correlation(a, b):
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossiers", nargs="+", help="dossiers de station, au moins deux")
a.add_argument("--colonnes", default="temperature,cam_tilt,cam_rotation,cam_scale,cib_corr_x,cib_corr_y")
a.add_argument("--pas", default="1h", type=duree)
a.add_argument("folders", nargs="+", help="station folders, at least two")
a.add_argument("--columns", default="temperature,cam_tilt,cam_rotation,cam_scale,tgt_corr_x,tgt_corr_y")
a.add_argument("--step", default="1h", type=duration)
a.add_argument("--stat", default="median", choices=STATS)
a.add_argument("--table", default="", metavar="T1/T2|non", help="table imprimée seulement entre T1 et T2 (« non » : pas du tout)")
a.add_argument("--debut", type=instant)
a.add_argument("--fin", type=instant)
a.add_argument("--sortie", default=os.path.join("out", "diagnostic"), help="dossier du PNG")
a.add_argument("--table", default="", metavar="T1/T2|no", help="table printed only between T1 and T2 (\"no\": not at all)")
a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time)
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG")
o = a.parse_args()
colonnes = o.colonnes.split(",")
if absents := [c for c in colonnes if c not in COLONNES]:
sys.exit(f"Colonnes inconnues : {', '.join(absents)}. Connues : {', '.join(COLONNES)}.")
if o.table in ("", "non"):
dans_table = lambda t: o.table == "" # noqa: E731
columns = o.columns.split(",")
if missing := [c for c in columns if c not in COLUMNS]:
sys.exit(f"Unknown columns: {', '.join(missing)}. Known: {', '.join(COLUMNS)}.")
if o.table in ("", "no"):
in_table = lambda t: o.table == "" # noqa: E731
else:
bornes = [instant(x) for x in o.table.split("/")]
if len(bornes) != 2:
sys.exit("--table attend T1/T2 ou non.")
dans_table = lambda t: bornes[0] <= t <= bornes[1] # noqa: E731
limits = [parse_time(x) for x in o.table.split("/")]
if len(limits) != 2:
sys.exit("--table expects T1/T2 or no.")
in_table = lambda t: limits[0] <= t <= limits[1] # noqa: E731
noms = [os.path.basename(os.path.normpath(d)) for d in o.dossiers]
donnees, mesures = {}, {}
for n, d in zip(noms, o.dossiers):
donnees[n], mesures[n] = lire(d, o.debut, o.fin)
tous = [p["t"] for s in donnees.values() for c in s.values() for p in c]
if not tous:
sys.exit("Aucune valeur dans la fenêtre demandée.")
t0 = (o.debut or min(tous)).replace(minute=0, second=0, microsecond=0)
t1 = o.fin or max(tous)
grille = [t0 + i * o.pas for i in range(int((t1 - t0) / o.pas) + 1)]
case = lambda t: t0 + ((t - t0) // o.pas) * o.pas # noqa: E731
names = [os.path.basename(os.path.normpath(d)) for d in o.folders]
by_station, measured = {}, {}
for n, d in zip(names, o.folders):
by_station[n], measured[n] = read(d, o.start, o.end)
times = [p["t"] for s in by_station.values() for c in s.values() for p in c]
if not times:
sys.exit("No value in the requested window.")
t0 = (o.start or min(times)).replace(minute=0, second=0, microsecond=0)
t1 = o.end or max(times)
grid = [t0 + i * o.step for i in range(int((t1 - t0) / o.step) + 1)]
cell = lambda t: t0 + ((t - t0) // o.step) * o.step # noqa: E731
print("Stations :")
for n in noms:
ts = mesures[n]
print("Stations:")
for n in names:
ts = measured[n]
if not ts:
print(f" {n} : aucune mesure dans la fenêtre")
print(f" {n}: no measurement in the window")
continue
cad = statistics.median(b - a_ for a_, b in zip(ts, ts[1:])) if len(ts) > 1 else None
print(f" {n} : mesures {iso(ts[0])} → {iso(ts[-1])}, {len(ts)} instants, cadence {round(cad.total_seconds() / 60) if cad else '-'} min ; "
+ ", ".join(f"{c} {len(donnees[n].get(c, []))}" for c in colonnes))
print(f" {n}: measurements {iso(ts[0])} → {iso(ts[-1])}, {len(ts)} times, cadence {round(cad.total_seconds() / 60) if cad else '-'} min; "
+ ", ".join(f"{c} {len(by_station[n].get(c, []))}" for c in columns))
# trous : pas sans mesure, sans température, et les stations concernées
avec = {n: {case(t) for t in mesures[n]} for n in noms}
avec_t = {n: {case(p["t"]) for p in donnees[n]["temperature"]} for n in noms}
print(f"\nTrous (pas de {int(o.pas.total_seconds() // 60)} min sans mesure ou sans température), avec les stations où ils manquent :")
cles = {}
for k in grille:
m = tuple(n for n in noms if k not in avec[n])
t = tuple(n for n in noms if k not in avec_t[n])
# gaps: steps without a measurement, without temperature, and the stations concerned
with_meas = {n: {cell(t) for t in measured[n]} for n in names}
with_temp = {n: {cell(p["t"]) for p in by_station[n]["temperature"]} for n in names}
print(f"\nGaps (steps of {int(o.step.total_seconds() // 60)} min without a measurement or without temperature), with the stations where they are missing:")
keys = {}
for k in grid:
m = tuple(n for n in names if k not in with_meas[n])
t = tuple(n for n in names if k not in with_temp[n])
if m or t:
cles.setdefault((m, t), []).append(k)
lignes = []
for (m, t), ks in cles.items():
for d, f, nb in suites(ks, o.pas):
lignes.append((d, f, nb, m, t))
for d, f, nb, m, t in sorted(lignes):
print(f" {iso(d)} → {iso(f)} ({nb} pas) : sans mesure {','.join(m) or '-'} ; sans température {','.join(t) or '-'}")
if not lignes:
print(" (aucun)")
keys.setdefault((m, t), []).append(k)
lines = []
for (m, t), ks in keys.items():
for d, f, nb in runs(ks, o.step):
lines.append((d, f, nb, m, t))
for d, f, nb, m, t in sorted(lines):
print(f" {iso(d)} → {iso(f)} ({nb} steps): no measurement {','.join(m) or '-'}; no temperature {','.join(t) or '-'}")
if not lines:
print(" (none)")
stat = STATS[o.stat]
table = {} # (station, colonne) -> {pas: valeur}
for n in noms:
for c in colonnes:
cases = {}
for p in donnees[n].get(c, []):
cases.setdefault(case(p["t"]), []).append(p["v"])
table[n, c] = {k: stat(v) for k, v in cases.items()}
entete = [f"{n}:{c}" for n in noms for c in colonnes]
print(f"\n{o.stat} par pas de {int(o.pas.total_seconds() // 60)} min (début du pas, UTC) ; « - » : aucune valeur"
+ (f" (table : {o.table})" if o.table else "") + " :")
if o.table != "non":
print("\t".join(["heure"] + entete))
for k in grille:
if dans_table(k):
v = [table[n, c].get(k) for n in noms for c in colonnes]
table = {} # (station, column) -> {step: value}
for n in names:
for c in columns:
cells = {}
for p in by_station[n].get(c, []):
cells.setdefault(cell(p["t"]), []).append(p["v"])
table[n, c] = {k: stat(v) for k, v in cells.items()}
header = [f"{n}:{c}" for n in names for c in columns]
print(f"\n{o.stat} per step of {int(o.step.total_seconds() // 60)} min (start of the step, UTC); \"-\": no value"
+ (f" (table: {o.table})" if o.table else "") + ":")
if o.table != "no":
print("\t".join(["time"] + header))
for k in grid:
if in_table(k):
v = [table[n, c].get(k) for n in names for c in columns]
if any(x is not None for x in v):
print("\t".join([iso(k)] + ["-" if x is None else f"{x:.6g}" for x in v]))
print("\nCorrélation entre stations, r des valeurs, puis r des pas d'une valeur à la suivante (n : pas où les deux ont une valeur) :")
for c in colonnes:
for i, s1 in enumerate(noms):
for s2 in noms[i + 1:]:
print("\nCorrelation between stations, r of the values, then r of the steps from one value to the next (n: steps where both have a value):")
for c in columns:
for i, s1 in enumerate(names):
for s2 in names[i + 1:]:
ks = sorted(set(table[s1, c]) & set(table[s2, c]))
v1, v2 = [table[s1, c][k] for k in ks], [table[s2, c][k] for k in ks]
d1, d2 = [b - a_ for a_, b in zip(v1, v1[1:])], [b - a_ for a_, b in zip(v2, v2[1:])]
print(f" {c:12} {s1}-{s2} : r valeurs {correlation(v1, v2)}, r pas {correlation(d1, d2)} (n = {len(ks)})")
print(f" {c:12} {s1}-{s2}: r values {correlation(v1, v2)}, r steps {correlation(d1, d2)} (n = {len(ks)})")
fig, axes = plt.subplots(len(colonnes) + 1, 1, sharex=True, figsize=(14, 2.4 * (len(colonnes) + 1)), squeeze=False,
layout="constrained", gridspec_kw={"height_ratios": [3] * len(colonnes) + [0.3 * len(noms)]})
for ax, c in zip(axes[:-1, 0], colonnes):
for i, n in enumerate(noms):
s = donnees[n].get(c, [])
fig, axes = plt.subplots(len(columns) + 1, 1, sharex=True, figsize=(14, 2.4 * (len(columns) + 1)), squeeze=False,
layout="constrained", gridspec_kw={"height_ratios": [3] * len(columns) + [0.3 * len(names)]})
for ax, c in zip(axes[:-1, 0], columns):
for i, n in enumerate(names):
s = by_station[n].get(c, [])
if not s:
continue
med = 0 if c == "temperature" else statistics.median(p["v"] for p in s)
ts, vs = coupe([{"t": p["t"], "v": p["v"] - med} for p in s])
ts, vs = cut([{"t": p["t"], "v": p["v"] - med} for p in s])
ax.plot(ts, vs, ".-", ms=2, lw=0.8, color=f"C{i % 10}", label=n)
ax.set_ylabel(c if c == "temperature" else f"{c}\nmoins sa médiane", fontsize=8)
ax.set_ylabel(c if c == "temperature" else f"{c}\nminus its median", fontsize=8)
ax.grid(alpha=0.3)
ax.legend(loc="upper left", bbox_to_anchor=(1.0, 1.0), fontsize=8)
ax = axes[-1, 0]
for i, n in enumerate(noms):
for k in grille:
if k not in avec[n]:
ax.axvspan(k, k + o.pas, ymin=i / len(noms), ymax=(i + 1) / len(noms), color=f"C{i % 10}", lw=0)
ax.set_yticks([(i + 0.5) / len(noms) for i in range(len(noms))], noms, fontsize=7)
for i, n in enumerate(names):
for k in grid:
if k not in with_meas[n]:
ax.axvspan(k, k + o.step, ymin=i / len(names), ymax=(i + 1) / len(names), color=f"C{i % 10}", lw=0)
ax.set_yticks([(i + 0.5) / len(names) for i in range(len(names))], names, fontsize=7)
ax.set_ylim(0, 1)
ax.set_ylabel("sans mesure", fontsize=8)
ax.set_xlim(grille[0], grille[-1] + o.pas)
ax.set_ylabel("no measurement", fontsize=8)
ax.set_xlim(grid[0], grid[-1] + o.step)
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
fig.suptitle(f"{', '.join(noms)} : même ligne de temps (heures UTC)")
os.makedirs(o.sortie, exist_ok=True)
chemin = os.path.join(o.sortie, f"{'-'.join(noms)}_stations_{borne(o.debut, 'debut')}_{borne(o.fin, 'fin')}.png")
fig.savefig(chemin, dpi=100, bbox_inches="tight")
print(f"\n{chemin}")
fig.suptitle(f"{', '.join(names)}: same timeline (UTC times)")
os.makedirs(o.out, exist_ok=True)
path = os.path.join(o.out, f"{'-'.join(names)}_stations_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png")
fig.savefig(path, dpi=100, bbox_inches="tight")
print(f"\n{path}")
if __name__ == "__main__":
+124
View File
@@ -0,0 +1,124 @@
"""Compact numeric summary of the series of an "advanced" export, for the diagnostic agent.
Usage: python3 tools/scripts/summary.py data/PS_DES_COUDRES/ [--column error_value]
[--stat median|min|max] [--step 1h] [--level period|window] [--sensors SENSOR_A,SENSOR_B] [--start 2026-01-01T00:00Z] [--end ...]
[--instants T1,T2] [--mark]
1. Usual level of each sensor: median, MAD (median absolute deviation), min, max, over the whole period
(default) or, with --level window, over the --start/--end window only (a level change
in the window is not compared with weeks of other measurements).
Columns: error_value, corr_x, corr_y, relative_x, relative_y, x, y, sigma0, reproj_err, and the six image quality metrics (peak to snr, raw values; empty on
old cameras; against a calm reference: quality.py).
2. Table: one row per time step (step start, UTC), one column per sensor, the chosen statistic
over the measurements of the step. "rej": all measurements of the step are rejected; "empty": measurements,
but the column is empty (e.g. reproj_err of a target); "-": no measurement. The sensor header is repeated
every 24 rows. With --instants T1,T2: only the steps that contain these instants. With --mark: "*"
after the value when a measurement of the step is rejected (x, y, error_value of a rejected measurement stay filled),
"o" when a measurement of the step has inlier = 0 (reference set aside by the compensation).
Gives values, not a verdict.
"""
import argparse
import re
import statistics
import sys
from datetime import timedelta
from data import filter_window, is_reference, iso, load, parse_time, rejected
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
STATS = {"median": statistics.median, "min": min, "max": max}
def duration(text):
"""'15min', '1h', '6h', '1d' -> timedelta."""
m = re.fullmatch(r"(\d+)(min|h|d)", text)
if not m:
raise argparse.ArgumentTypeError(f"invalid step: {text!r} (e.g. 15min, 1h, 1d)")
n, unit = int(m.group(1)), m.group(2)
return {"min": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n)}[unit]
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
def own_noise(vs):
"""Median of the absolute differences between two consecutive values; 0 when it is below ROUNDING (values constant
up to rounding: the corr of the references when the compensation goes exactly through them, two references
retained), so that a deviation is not written as billions of noises; 0 also with fewer than two values."""
d = [abs(b - a) for a, b in zip(vs, vs[1:])]
b = statistics.median(d) if d else 0
return b if b >= ROUNDING else 0
def values(ms, col):
return [m[col] for m in ms if m[col] is not None]
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder")
a.add_argument("--column", default="error_value", choices=COLUMNS + QUALITY)
a.add_argument("--stat", default="median", choices=STATS)
a.add_argument("--step", default="1h", type=duration)
a.add_argument("--level", default="period", choices=("period", "window"),
help="period over which median and MAD are computed (default: all measurements)")
a.add_argument("--sensors", help="comma-separated list (default: all)")
a.add_argument("--instants", help="T1,T2: only the steps that contain these instants")
a.add_argument("--mark", action="store_true", help="*: a measurement of the step rejected; o: a measurement of the step with inlier = 0")
a.add_argument("--start", type=parse_time)
a.add_argument("--end", type=parse_time)
o = a.parse_args()
by = load(o.folder)
if not by:
sys.exit(f'No "advanced" export in {o.folder} (see inventory.py).')
names = o.sensors.split(",") if o.sensors else sorted(by, key=lambda s: (not is_reference(by[s]), s))
unknown = [s for s in names if s not in by]
if unknown:
sys.exit(f"Missing sensors: {', '.join(unknown)}. Present: {', '.join(sorted(by))}.")
t0 = min(m["t"] for s in names for m in by[s]).replace(hour=0, minute=0, second=0, microsecond=0)
steps = {}
for s in names:
for m in filter_window(by[s], o.start, o.end):
step_start = t0 + ((m["t"] - t0) // o.step) * o.step
steps.setdefault(step_start, {}).setdefault(s, []).append(m)
if not steps:
sys.exit("No measurement in the requested window.")
over = "the --start/--end window" if o.level == "window" else "the whole period"
print(f"Column: {o.column}. Usual level over {over} (non-empty measurements):")
print(f"{'sensor':10} {'n':>5} {'median':>10} {'MAD':>10} {'min':>10} {'max':>10}")
for s in names:
v = values(filter_window(by[s], o.start, o.end) if o.level == "window" else by[s], o.column)
if not v:
print(f"{s:10} {0:5} (no value)")
continue
med = statistics.median(v)
mad = statistics.median(abs(x - med) for x in v)
print(f"{s:10} {len(v):5} {med:10.5g} {mad:10.5g} {min(v):10.5g} {max(v):10.5g}")
print(f"\n{o.stat} of {o.column} per step of {o.step} (step start, UTC):")
print("rej: all measurements of the step rejected; empty: empty column; -: no measurement")
if o.mark:
print("*: at least one measurement of the step is rejected; o: at least one measurement of the step has inlier = 0")
header = f"{'start':17} " + " ".join(f"{s:>9}" for s in names)
wanted = {t0 + ((parse_time(x) - t0) // o.step) * o.step for x in o.instants.split(",")} if o.instants else None
for i, k in enumerate(k for k in sorted(steps) if wanted is None or k in wanted):
if i % 24 == 0:
print(header)
cells = []
for s in names:
ms = steps[k].get(s, [])
v = values(ms, o.column)
if v:
mark = ("*" if any(map(rejected, ms)) else "") + ("o" if any(m["inlier"] == 0 for m in ms) else "") if o.mark else ""
cells.append(f"{STATS[o.stat](v):.5g}{mark}".rjust(9))
else:
cells.append(f"{'-' if not ms else 'rej' if all(map(rejected, ms)) else 'empty':>9}")
print(f"{iso(k):17} " + " ".join(cells))
if __name__ == "__main__":
main()
+171
View File
@@ -0,0 +1,171 @@
"""Thumbnails of the camera images around sensors, at full resolution, for the diagnostic agent.
Usage: .venv/bin/python tools/scripts/thumbnails.py data/PS_DES_COUDRES/
--sensors REF_L,REF_R --times 2026-10-13T22:00Z,2026-10-14T02:00Z [--radius 100] [--contrast [MIN,MAX]] [--out FOLDER]
One row per sensor, one column per time. When there is no image at the requested time, the nearest one is
taken: the gap is printed and the actual time of the image is in the thumbnail's title. Each thumbnail is centred on the raw position (x, y) of the sensor
in the "advanced" export, read at the measurement nearest to the image; the pixels are not retouched, so the brightness
can be compared from one thumbnail to another. Prints the positions used, the background (median of the pixels of the
thumbnail), the 99th percentile and the peak (maximum) of the grey level of each thumbnail (0 to 255), then the path of the PNG.
--contrast: all thumbnails in grey levels, with the same stretch for the whole sheet, so that a faint
spot shows and can be compared from one thumbnail to another. Without a value, the stretch goes from the 1st percentile to the maximum
of the sheet (a saturated pixel somewhere crushes it: then give --contrast MIN,MAX again, with the help of the background and the
peak printed for each thumbnail). A smaller --radius (30 to 50) magnifies. The bounds are printed.
The frame centre (px) is written under each thumbnail: when the x, y of the export jump, the frame moves
by as much. --fixed keeps, on each row, the frame centred on the position at the first requested time, and marks
the x, y of the export at each time with a red cross: a spot that enters or leaves the frame is not confused with
a spot that appears. A frame cut by the edge of the image is flagged (image size): on some
exports, x, y are not pixels (values close to 0), and the frame falls in the corner of the image.
--zone X,Y,W,H: the same zone of the image (pixels of the original image: top left corner, width, height, as
gif.py) at each time, side by side (up to 4 per row, fewer when the zone is wide), same stretch with --contrast; --sensors becomes optional:
those given are marked with a red cross and their name at their x, y at each time. For the scene around
the targets (foreground, object, lighting), which a large --radius would make too small.
Each thumbnail is rendered at its size in pixels (one image pixel for one PNG pixel) from 260 to 1000 px a side;
Read shrinks a PNG that is too wide: with a large --radius or a large --zone, few times per sheet.
"""
import argparse
import os
import sys
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from data import find_files, image_time, iso, load, parse_time # noqa: E402
def percentile(grey, p):
"""Percentile p (0 to 100) of the pixels of a thumbnail, without interpolation."""
v = sorted(grey.ravel().tolist())
return v[min(int(p / 100 * len(v)), len(v) - 1)]
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("folder")
a.add_argument("--sensors", default="", help="comma-separated list (optional with --zone)")
a.add_argument("--times", required=True, help="times of the images, comma-separated (UTC)")
a.add_argument("--radius", type=int, default=100, help="half side of the thumbnail in pixels (default 100)")
a.add_argument("--contrast", nargs="?", const="auto", metavar="MIN,MAX", help="grey levels, same stretch for the whole sheet: MIN,MAX (0 to 255) or, without a value, from the 1st percentile to the maximum of the sheet")
a.add_argument("--fixed", action="store_true", help="one fixed frame per sensor (position at the first time), x, y of each time marked with a cross")
a.add_argument("--zone", metavar="X,Y,W,H", help="the same zone of the image at each time (pixels of the original image: top left corner, width, height)")
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG")
o = a.parse_args()
by = load(o.folder)
images = {image_time(p): p for p in find_files(o.folder)["images"]}
names, times = [s for s in o.sensors.split(",") if s], [parse_time(h) for h in o.times.split(",")]
zone = tuple(map(int, o.zone.split(","))) if o.zone else None
if not names and not zone:
sys.exit("Give --sensors, or --zone X,Y,W,H.")
missing = [s for s in names if s not in by]
if missing:
sys.exit(f"Unknown sensors: {', '.join(missing)}. Known: {', '.join(sorted(by))}.")
known = sorted(h for h in images if h)
if not known:
sys.exit(f"No image in {o.folder}.")
nearest = {h: min(known, key=lambda i: abs(i - h)) for h in times} # image at the requested time, otherwise the nearest
for h in times:
if nearest[h] != h:
print(f"No image at {iso(h)}: the nearest is the one of {iso(nearest[h])} (gap {abs(nearest[h] - h)}).")
def levels(c):
grey = c[..., :3].mean(axis=-1) if c.ndim == 3 else c
return grey * 255 if grey.max() <= 1 and c.dtype.kind == "f" else grey
def measurement(s, t):
return min((m for m in by[s] if m["x"] is not None), key=lambda m: abs(m["t"] - t))
r = o.radius
crops, centres, crosses = {}, {}, {}
for j, h in enumerate(times):
img = plt.imread(images[nearest[h]])
height, width = img.shape[:2]
if zone:
X, Y, W, H = zone
c = img[max(Y, 0):Y + H, max(X, 0):X + W]
if c.size == 0:
sys.exit(f"--zone {o.zone} outside the image ({width} × {height} px).")
grey = levels(c)
crops[0, j] = (c, grey)
crosses[0, j] = [(s, m["x"] - max(X, 0), m["y"] - max(Y, 0)) for s in names for m in [measurement(s, nearest[h])]]
print(f"zone {o.zone} {iso(h)}: {os.path.basename(images[nearest[h]])}, background {percentile(grey, 50):.0f}, "
f"p99 {percentile(grey, 99):.0f}, peak {grey.max():.0f}"
+ ("" if c.shape[:2] == (H, W) else f", cut by the image edge ({width} × {height} px)")
+ "".join(f"; {s} x={x + max(X, 0):.1f} y={y + max(Y, 0):.1f}" for s, x, y in crosses[0, j]))
continue
for i, s in enumerate(names):
m = measurement(s, nearest[h])
x, y = centres[i] if o.fixed and j else (round(m["x"]), round(m["y"]))
if j == 0:
centres[i] = (x, y)
c = img[max(y - r, 0):y + r, max(x - r, 0):x + r]
if c.size == 0:
sys.exit(f"{s} {iso(h)}: x={m['x']:.1f} y={m['y']:.1f} outside the image ({width} × {height} px); "
"--zone X,Y,W,H looks at a fixed zone.")
crosses[i, j] = (m["x"] - max(x - r, 0), m["y"] - max(y - r, 0), x, y)
grey = levels(c)
crops[i, j] = (c, grey)
cut = x - r < 0 or y - r < 0 or x + r > width or y + r > height
print(f"{s} {iso(h)}: x={m['x']:.1f} y={m['y']:.1f} (measurement of {iso(m['t'])}), {os.path.basename(images[nearest[h]])}, "
f"background {percentile(grey, 50):.0f}, p99 {percentile(grey, 99):.0f}, peak {grey.max():.0f}"
+ (f", frame cut by the image edge ({width} × {height} px)" if cut else ""))
bounds = {}
if o.contrast:
if o.contrast == "auto":
bounds = dict(vmin=min(percentile(g, 1) for _, g in crops.values()), vmax=max(g.max() for _, g in crops.values()))
else:
bounds = dict(zip(("vmin", "vmax"), map(float, o.contrast.split(","))))
print(f"Common stretch: {bounds['vmin']:.0f} to {bounds['vmax']:.0f} (grey levels, 0 to 255)")
# A thumbnail at its size in pixels (at 100 dpi) once it exceeds 2.6 inches: a large --radius is no longer shrunk.
side = max(c.shape[1] for c, _ in crops.values()) / 100, max(c.shape[0] for c, _ in crops.values()) / 100
side = tuple(v * min(1, 10 / max(side)) for v in side) # at most 1000 px a side per thumbnail
if zone:
nc = max(1, min(4, len(times), int(20 // max(2.6, side[0])))) # sheet at most ~2000 px wide: Read shrinks it beyond
nl = -(-len(times) // nc)
fig, axes = plt.subplots(nl, nc, figsize=(max(2.6, side[0]) * nc, (max(2.6, side[1]) + 0.4) * nl), squeeze=False)
for ax in axes.ravel()[len(times):]:
ax.axis("off")
else:
fig, axes = plt.subplots(len(names), len(times), figsize=(max(2.6, side[0]) * len(times), (max(2.6, side[1]) + 0.2) * len(names)), squeeze=False)
for (i, j), (c, grey) in crops.items():
ax = axes[j // nc, j % nc] if zone else axes[i, j]
if o.contrast:
ax.imshow(grey, cmap="gray", interpolation="nearest", **bounds)
else:
ax.imshow(c, interpolation="nearest")
ax.set_title(f"{'zone' if zone else names[i]} {nearest[times[j]].strftime('%d/%m %H:%MZ')}" + ("" if nearest[times[j]] == times[j] else " (requested " + times[j].strftime('%H:%MZ') + ")"), fontsize=9)
if zone:
for s, cx, cy in crosses[i, j]:
ax.plot(cx, cy, "+", color="r", ms=10, mew=1)
ax.annotate(s, (cx, cy), xytext=(4, 4), textcoords="offset points", color="r", fontsize=7)
ax.set_xlim(-0.5, c.shape[1] - 0.5)
ax.set_ylim(c.shape[0] - 0.5, -0.5)
ax.set_xlabel(f"zone {o.zone}", fontsize=7)
ax.set_xticks([])
ax.set_yticks([])
continue
cx, cy, x, y = crosses[i, j]
if o.fixed:
ax.plot(cx, cy, "+", color="r", ms=10, mew=1)
ax.set_xlabel(f"frame centre {x}, {y}" + (f"\nx, y {cx + max(x - r, 0):.1f}, {cy + max(y - r, 0):.1f}" if o.fixed else ""), fontsize=7)
ax.set_xticks([])
ax.set_yticks([])
station = os.path.basename(os.path.normpath(o.folder))
fig.suptitle(f"{station}: zone {o.zone} (x, y, width, height in px; UTC times)" if zone else
f"{station}: {2 * r} px thumbnails around each sensor (UTC times)", fontsize=10)
os.makedirs(o.out, exist_ok=True)
what = f"zone{o.zone.replace(',', '-')}" + ("_" + "-".join(names) if names else "") if zone else "-".join(names)
path = os.path.join(o.out, f"{station}_thumbnails_{what}_"
f"{'-'.join(h.strftime('%d%H%M') for h in times)}{'_contrast' if o.contrast else ''}{'_fixed' if o.fixed else ''}.png")
fig.savefig(path, dpi=100, bbox_inches="tight")
print(path)
if __name__ == "__main__":
main()
-171
View File
@@ -1,171 +0,0 @@
"""Vignettes des images de la caméra autour de capteurs, à pleine résolution, pour l'agent de diagnostic.
Usage : .venv/bin/python tools/scripts/vignettes.py data/PS_DES_COUDRES/
--capteurs REF_G,REF_D --heures 2026-10-13T22:00Z,2026-10-14T02:00Z [--rayon 100] [--contraste [MIN,MAX]] [--sortie DOSSIER]
Une ligne par capteur, une colonne par heure. Quand il n'y a pas d'image à l'heure demandée, la plus proche est
prise : l'écart est imprimé et l'heure réelle de l'image est dans le titre de la vignette. Chaque vignette est centrée sur la position brute (x, y) du capteur
dans l'export « advanced », lue à la mesure la plus proche de l'image ; les pixels ne sont pas retouchés, l'éclat
se compare donc d'une vignette à l'autre. Imprime les positions utilisées, le fond (médiane des pixels de la
vignette), le 99e centile et le pic (maximum) du niveau de gris de chaque vignette (0 à 255), puis le chemin du PNG.
--contraste : toutes les vignettes en niveaux de gris, avec le même étirement pour toute la planche, pour qu'une
tache faible se voie et se compare d'une vignette à l'autre. Sans valeur, l'étirement va du 1er centile au maximum
de la planche (un pixel saturé quelque part l'écrase : redonner alors --contraste MIN,MAX, en s'aidant du fond et du
pic imprimés pour chaque vignette). Un --rayon plus petit (30 à 50) grossit. Les bornes sont imprimées.
Le centre du cadre (px) est écrit sous chaque vignette : quand les x, y de l'export sautent, le cadre se déplace
d'autant. --fixe garde, sur chaque ligne, le cadre centré sur la position de la première heure demandée, et marque
d'une croix rouge les x, y de l'export à chaque heure : une tache qui entre dans le cadre ou en sort ne s'y confond
pas avec une tache qui apparaît. Un cadre que le bord de l'image coupe est signalé (taille de l'image) : sur certains
exports, x, y ne sont pas des pixels (valeurs proches de 0), et le cadre tombe dans le coin de l'image.
--zone X,Y,L,H : la même zone de l'image (pixels de l'image d'origine : coin haut gauche, largeur, hauteur, comme
gif.py) à chaque heure, côte à côte (jusqu'à 4 par ligne, moins quand la zone est large), même étirement avec --contraste ; --capteurs devient facultatif :
ceux qui sont donnés sont marqués d'une croix rouge et de leur nom à leurs x, y de chaque heure. Pour la scène autour
des cibles (premier plan, objet, éclairage), qu'un grand --rayon rendrait trop petite.
Chaque vignette est rendue à sa taille en pixels (un pixel de l'image pour un pixel du PNG) de 260 à 1000 px de côté ;
Read réduit un PNG trop large : avec un grand --rayon ou une grande --zone, peu d'heures par planche.
"""
import argparse
import os
import sys
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from donnees import charger, heure_image, instant, iso, trouver # noqa: E402
def centile(gris, p):
"""Centile p (0 à 100) des pixels d'une vignette, sans interpolation."""
v = sorted(gris.ravel().tolist())
return v[min(int(p / 100 * len(v)), len(v) - 1)]
def main():
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
a.add_argument("dossier")
a.add_argument("--capteurs", default="", help="liste séparée par des virgules (facultative avec --zone)")
a.add_argument("--heures", required=True, help="heures des images, séparées par des virgules (UTC)")
a.add_argument("--rayon", type=int, default=100, help="demi-côté de la vignette en pixels (défaut 100)")
a.add_argument("--contraste", nargs="?", const="auto", metavar="MIN,MAX", help="niveaux de gris, même étirement pour toute la planche : MIN,MAX (0 à 255) ou, sans valeur, du 1er centile au maximum de la planche")
a.add_argument("--fixe", action="store_true", help="un cadre fixe par capteur (position à la première heure), x, y de chaque heure marqués d'une croix")
a.add_argument("--zone", metavar="X,Y,L,H", help="la même zone de l'image à chaque heure (pixels de l'image d'origine : coin haut gauche, largeur, hauteur)")
a.add_argument("--sortie", default=os.path.join("out", "diagnostic"), help="dossier du PNG")
o = a.parse_args()
par = charger(o.dossier)
images = {heure_image(p): p for p in trouver(o.dossier)["images"]}
noms, heures = [s for s in o.capteurs.split(",") if s], [instant(h) for h in o.heures.split(",")]
zone = tuple(map(int, o.zone.split(","))) if o.zone else None
if not noms and not zone:
sys.exit("Donne --capteurs, ou --zone X,Y,L,H.")
absents = [s for s in noms if s not in par]
if absents:
sys.exit(f"Capteurs inconnus : {', '.join(absents)}. Connus : {', '.join(sorted(par))}.")
connues = sorted(h for h in images if h)
if not connues:
sys.exit(f"Aucune image dans {o.dossier}.")
proche = {h: min(connues, key=lambda i: abs(i - h)) for h in heures} # image à l'heure demandée, sinon la plus proche
for h in heures:
if proche[h] != h:
print(f"Pas d'image à {iso(h)} : la plus proche est celle de {iso(proche[h])} (écart {abs(proche[h] - h)}).")
def niveaux(c):
gris = c[..., :3].mean(axis=-1) if c.ndim == 3 else c
return gris * 255 if gris.max() <= 1 and c.dtype.kind == "f" else gris
def mesure(s, t):
return min((m for m in par[s] if m["x"] is not None), key=lambda m: abs(m["t"] - t))
r = o.rayon
coupes, centres, croix = {}, {}, {}
for j, h in enumerate(heures):
img = plt.imread(images[proche[h]])
haut, larg = img.shape[:2]
if zone:
X, Y, L, H = zone
c = img[max(Y, 0):Y + H, max(X, 0):X + L]
if c.size == 0:
sys.exit(f"--zone {o.zone} hors de l'image ({larg} × {haut} px).")
gris = niveaux(c)
coupes[0, j] = (c, gris)
croix[0, j] = [(s, m["x"] - max(X, 0), m["y"] - max(Y, 0)) for s in noms for m in [mesure(s, proche[h])]]
print(f"zone {o.zone} {iso(h)} : {os.path.basename(images[proche[h]])}, fond {centile(gris, 50):.0f}, "
f"p99 {centile(gris, 99):.0f}, pic {gris.max():.0f}"
+ ("" if c.shape[:2] == (H, L) else f", coupée par le bord de l'image ({larg} × {haut} px)")
+ "".join(f" ; {s} x={x + max(X, 0):.1f} y={y + max(Y, 0):.1f}" for s, x, y in croix[0, j]))
continue
for i, s in enumerate(noms):
m = mesure(s, proche[h])
x, y = centres[i] if o.fixe and j else (round(m["x"]), round(m["y"]))
if j == 0:
centres[i] = (x, y)
c = img[max(y - r, 0):y + r, max(x - r, 0):x + r]
if c.size == 0:
sys.exit(f"{s} {iso(h)} : x={m['x']:.1f} y={m['y']:.1f} hors de l'image ({larg} × {haut} px) ; "
"--zone X,Y,L,H regarde une zone fixe.")
croix[i, j] = (m["x"] - max(x - r, 0), m["y"] - max(y - r, 0), x, y)
gris = niveaux(c)
coupes[i, j] = (c, gris)
coupe = x - r < 0 or y - r < 0 or x + r > larg or y + r > haut
print(f"{s} {iso(h)} : x={m['x']:.1f} y={m['y']:.1f} (mesure de {iso(m['t'])}), {os.path.basename(images[proche[h]])}, "
f"fond {centile(gris, 50):.0f}, p99 {centile(gris, 99):.0f}, pic {gris.max():.0f}"
+ (f", cadre coupé par le bord de l'image ({larg} × {haut} px)" if coupe else ""))
bornes = {}
if o.contraste:
if o.contraste == "auto":
bornes = dict(vmin=min(centile(g, 1) for _, g in coupes.values()), vmax=max(g.max() for _, g in coupes.values()))
else:
bornes = dict(zip(("vmin", "vmax"), map(float, o.contraste.split(","))))
print(f"Étirement commun : {bornes['vmin']:.0f} à {bornes['vmax']:.0f} (niveaux de gris, 0 à 255)")
# Une vignette à sa taille en pixels (à 100 dpi) dès qu'elle dépasse 2,6 pouces : un grand --rayon n'est plus réduit.
cote = max(c.shape[1] for c, _ in coupes.values()) / 100, max(c.shape[0] for c, _ in coupes.values()) / 100
cote = tuple(v * min(1, 10 / max(cote)) for v in cote) # au plus 1000 px de côté par vignette
if zone:
nc = max(1, min(4, len(heures), int(20 // max(2.6, cote[0])))) # planche d'au plus ~2000 px de large : Read la réduit au-delà
nl = -(-len(heures) // nc)
fig, axes = plt.subplots(nl, nc, figsize=(max(2.6, cote[0]) * nc, (max(2.6, cote[1]) + 0.4) * nl), squeeze=False)
for ax in axes.ravel()[len(heures):]:
ax.axis("off")
else:
fig, axes = plt.subplots(len(noms), len(heures), figsize=(max(2.6, cote[0]) * len(heures), (max(2.6, cote[1]) + 0.2) * len(noms)), squeeze=False)
for (i, j), (c, gris) in coupes.items():
ax = axes[j // nc, j % nc] if zone else axes[i, j]
if o.contraste:
ax.imshow(gris, cmap="gray", interpolation="nearest", **bornes)
else:
ax.imshow(c, interpolation="nearest")
ax.set_title(f"{'zone' if zone else noms[i]} {proche[heures[j]].strftime('%d/%m %H:%MZ')}" + ("" if proche[heures[j]] == heures[j] else " (demandée " + heures[j].strftime('%H:%MZ') + ")"), fontsize=9)
if zone:
for s, cx, cy in croix[i, j]:
ax.plot(cx, cy, "+", color="r", ms=10, mew=1)
ax.annotate(s, (cx, cy), xytext=(4, 4), textcoords="offset points", color="r", fontsize=7)
ax.set_xlim(-0.5, c.shape[1] - 0.5)
ax.set_ylim(c.shape[0] - 0.5, -0.5)
ax.set_xlabel(f"zone {o.zone}", fontsize=7)
ax.set_xticks([])
ax.set_yticks([])
continue
cx, cy, x, y = croix[i, j]
if o.fixe:
ax.plot(cx, cy, "+", color="r", ms=10, mew=1)
ax.set_xlabel(f"centre du cadre {x}, {y}" + (f"\nx, y {cx + max(x - r, 0):.1f}, {cy + max(y - r, 0):.1f}" if o.fixe else ""), fontsize=7)
ax.set_xticks([])
ax.set_yticks([])
station = os.path.basename(os.path.normpath(o.dossier))
fig.suptitle(f"{station} : zone {o.zone} (x, y, largeur, hauteur en px ; heures UTC)" if zone else
f"{station} : vignettes de {2 * r} px autour de chaque capteur (heures UTC)", fontsize=10)
os.makedirs(o.sortie, exist_ok=True)
quoi = f"zone{o.zone.replace(',', '-')}" + ("_" + "-".join(noms) if noms else "") if zone else "-".join(noms)
sortie = os.path.join(o.sortie, f"{station}_vignettes_{quoi}_"
f"{'-'.join(h.strftime('%d%H%M') for h in heures)}{'_contraste' if o.contraste else ''}{'_fixe' if o.fixe else ''}.png")
fig.savefig(sortie, dpi=100, bbox_inches="tight")
print(sortie)
if __name__ == "__main__":
main()