English: tools renamed and translated (same numbers as before), options and printed names in English; the guide translated
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
67c88625c5
commit
f28bd19e04
25 files changed
+2764
-2762
No files matched your search
+127
-127
@@ -1,26 +1,26 @@
|
||||
"""Plusieurs stations sur une même ligne de temps : température du boîtier, pose de la caméra, corr des cibles, rejets, trous.
|
||||
"""Several stations on one timeline: enclosure temperature, camera pose, corr of the targets, rejections, gaps.
|
||||
|
||||
Usage : .venv/bin/python tools/scripts/stations.py data/BREITHORN data/MICHIGAN data/NEBRASKA
|
||||
[--colonnes temperature,cam_tilt,cam_rotation,cam_scale,cib_corr_x,cib_corr_y] [--pas 1h] [--stat median|min|max]
|
||||
[--table T1/T2|non] [--debut ...] [--fin ...] [--sortie DOSSIER]
|
||||
Usage: .venv/bin/python tools/scripts/stations.py data/BREITHORN data/MICHIGAN data/NEBRASKA
|
||||
[--columns temperature,cam_tilt,cam_rotation,cam_scale,tgt_corr_x,tgt_corr_y] [--step 1h] [--stat median|min|max]
|
||||
[--table T1/T2|no] [--start ...] [--end ...] [--out FOLDER]
|
||||
|
||||
Pour chaque station (un dossier de station chacune), sur la fenêtre --debut/--fin :
|
||||
- colonnes possibles : temperature (médiane des composants sensor_*_deg_c, comme metriques.py), cam_pan, cam_tilt,
|
||||
cam_rotation, cam_scale, cam_sigma0 (pose brute de la caméra), cib_corr_x, cib_corr_y (médiane sur les cibles de
|
||||
corr moins le niveau propre de chaque cible, mesures acceptées, comme metriques.py), rej_cibles, rej_refs (nombre de
|
||||
cibles, de références rejetées : le plus grand des instants de mesure du pas) ;
|
||||
- imprime, par station, la période des mesures, la cadence (pas médian) et le nombre d'instants ; puis les **trous** :
|
||||
les suites de pas de --pas sans mesure ou sans température, avec les stations où elles manquent (un trou commun à
|
||||
plusieurs stations y est sur une seule ligne) ; puis la table, une ligne par pas (début du pas, UTC ; --stat des
|
||||
valeurs du pas ; « - » : aucune valeur), une colonne par station et par colonne, séparées par des tabulations
|
||||
(--table T1/T2 : seulement entre T1 et T2, « non » : pas du tout ; le reste porte sur toute la fenêtre) ; puis, pour
|
||||
chaque colonne et chaque paire de stations, la corrélation r des valeurs et celle des pas (différence d'une valeur à
|
||||
la suivante, sur les pas où les deux stations ont une valeur : un mouvement commun se voit sur les pas, une tendance
|
||||
commune sur les valeurs), avec le nombre de pas ;
|
||||
- un PNG « stations » : un panneau par colonne, un trait par station (température brute ; les autres colonnes moins
|
||||
leur médiane sur la fenêtre, pour se superposer), coupé sur un trou ; trous de mesure grisés sous l'axe, une bande
|
||||
par station.
|
||||
Jamais d'identifiant de metrics/ (SIM_CCID, ip*) : seules les températures y sont lues. Rend des valeurs, pas de verdict.
|
||||
For each station (one station folder each), over the window --start/--end:
|
||||
- possible columns: temperature (median of the sensor_*_deg_c components, as in metrics.py), cam_pan, cam_tilt,
|
||||
cam_rotation, cam_scale, cam_sigma0 (raw camera pose), tgt_corr_x, tgt_corr_y (median over the targets of
|
||||
corr minus each target's own level, accepted measurements, as in metrics.py), rej_targets, rej_refs (number of
|
||||
rejected targets, of rejected references: the largest over the measurement times of the step);
|
||||
- prints, per station, the period of the measurements, the cadence (median step) and the number of times; then the **gaps**:
|
||||
the runs of steps of --step without a measurement or without temperature, with the stations where they are missing (a gap shared by
|
||||
several stations is on a single line); then the table, one line per step (start of the step, UTC; --stat of the
|
||||
values of the step; "-": no value), one column per station and per column, tab-separated
|
||||
(--table T1/T2: only between T1 and T2, "no": not at all; the rest covers the whole window); then, for
|
||||
each column and each pair of stations, the correlation r of the values and that of the steps (difference from one value to
|
||||
the next, over the steps where both stations have a value: a common movement shows on the steps, a common
|
||||
trend on the values), with the number of steps;
|
||||
- one PNG "stations": one panel per column, one line per station (raw temperature; the other columns minus
|
||||
their median over the window, so that they overlap), broken over a gap; measurement gaps shaded below the axis, one band
|
||||
per station.
|
||||
Never an identifier from metrics/ (SIM_CCID, ip*): only the temperatures are read there. Gives values, not a verdict.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
@@ -33,31 +33,31 @@ matplotlib.use("Agg")
|
||||
import matplotlib.dates as mdates # noqa: E402
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
from donnees import POSE, charger, charger_camera, charger_temperature, est_reference, filtrer, instant, iso, rejetee # noqa: E402
|
||||
from graphique import borne # noqa: E402
|
||||
from metriques import cibles, coupe, suites # noqa: E402
|
||||
from resume import STATS, duree # noqa: E402
|
||||
from data import POSE, filter_window, is_reference, iso, load, load_camera, load_temperature, parse_time, rejected # noqa: E402
|
||||
from metrics import cut, runs, targets # noqa: E402
|
||||
from plot import bound # noqa: E402
|
||||
from summary import STATS, duration # noqa: E402
|
||||
|
||||
COLONNES = ("temperature",) + tuple("cam_" + c for c in POSE) + ("cib_corr_x", "cib_corr_y", "rej_cibles", "rej_refs")
|
||||
COLUMNS = ("temperature",) + tuple("cam_" + c for c in POSE) + ("tgt_corr_x", "tgt_corr_y", "rej_targets", "rej_refs")
|
||||
|
||||
|
||||
def lire(dossier, debut, fin):
|
||||
"""Dossier de station -> ({colonne: [{'t', 'v'}]}, [instants de mesure])."""
|
||||
par = charger(dossier)
|
||||
series = {"temperature": filtrer([{"t": t, "v": v} for t, v in charger_temperature(dossier)], debut, fin)}
|
||||
for p in filtrer(charger_camera(dossier), debut, fin):
|
||||
def read(folder, start, end):
|
||||
"""Station folder -> ({column: [{'t', 'v'}]}, [measurement times])."""
|
||||
by_sensor = load(folder)
|
||||
series = {"temperature": filter_window([{"t": t, "v": v} for t, v in load_temperature(folder)], start, end)}
|
||||
for p in filter_window(load_camera(folder), start, end):
|
||||
for c in POSE:
|
||||
if p[c] is not None:
|
||||
series.setdefault("cam_" + c, []).append({"t": p["t"], "v": p[c]})
|
||||
for c in ("corr_x", "corr_y"):
|
||||
series["cib_" + c] = cibles(par, c, debut, fin)
|
||||
rej = {} # t -> [cibles rejetées, références rejetées]
|
||||
for s, ms in par.items():
|
||||
ref = est_reference(ms)
|
||||
for m in filtrer(ms, debut, fin):
|
||||
series["tgt_" + c] = targets(by_sensor, c, start, end)
|
||||
rej = {} # t -> [rejected targets, rejected references]
|
||||
for s, ms in by_sensor.items():
|
||||
ref = is_reference(ms)
|
||||
for m in filter_window(ms, start, end):
|
||||
n = rej.setdefault(m["t"], [0, 0])
|
||||
n[ref] += rejetee(m)
|
||||
series["rej_cibles"] = [{"t": t, "v": float(n[0])} for t, n in sorted(rej.items())]
|
||||
n[ref] += rejected(m)
|
||||
series["rej_targets"] = [{"t": t, "v": float(n[0])} for t, n in sorted(rej.items())]
|
||||
series["rej_refs"] = [{"t": t, "v": float(n[1])} for t, n in sorted(rej.items())]
|
||||
return series, sorted(rej)
|
||||
|
||||
@@ -71,123 +71,123 @@ def correlation(a, b):
|
||||
|
||||
def main():
|
||||
a = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
a.add_argument("dossiers", nargs="+", help="dossiers de station, au moins deux")
|
||||
a.add_argument("--colonnes", default="temperature,cam_tilt,cam_rotation,cam_scale,cib_corr_x,cib_corr_y")
|
||||
a.add_argument("--pas", default="1h", type=duree)
|
||||
a.add_argument("folders", nargs="+", help="station folders, at least two")
|
||||
a.add_argument("--columns", default="temperature,cam_tilt,cam_rotation,cam_scale,tgt_corr_x,tgt_corr_y")
|
||||
a.add_argument("--step", default="1h", type=duration)
|
||||
a.add_argument("--stat", default="median", choices=STATS)
|
||||
a.add_argument("--table", default="", metavar="T1/T2|non", help="table imprimée seulement entre T1 et T2 (« non » : pas du tout)")
|
||||
a.add_argument("--debut", type=instant)
|
||||
a.add_argument("--fin", type=instant)
|
||||
a.add_argument("--sortie", default=os.path.join("out", "diagnostic"), help="dossier du PNG")
|
||||
a.add_argument("--table", default="", metavar="T1/T2|no", help="table printed only between T1 and T2 (\"no\": not at all)")
|
||||
a.add_argument("--start", type=parse_time)
|
||||
a.add_argument("--end", type=parse_time)
|
||||
a.add_argument("--out", default=os.path.join("out", "diagnostic"), help="folder of the PNG")
|
||||
o = a.parse_args()
|
||||
colonnes = o.colonnes.split(",")
|
||||
if absents := [c for c in colonnes if c not in COLONNES]:
|
||||
sys.exit(f"Colonnes inconnues : {', '.join(absents)}. Connues : {', '.join(COLONNES)}.")
|
||||
if o.table in ("", "non"):
|
||||
dans_table = lambda t: o.table == "" # noqa: E731
|
||||
columns = o.columns.split(",")
|
||||
if missing := [c for c in columns if c not in COLUMNS]:
|
||||
sys.exit(f"Unknown columns: {', '.join(missing)}. Known: {', '.join(COLUMNS)}.")
|
||||
if o.table in ("", "no"):
|
||||
in_table = lambda t: o.table == "" # noqa: E731
|
||||
else:
|
||||
bornes = [instant(x) for x in o.table.split("/")]
|
||||
if len(bornes) != 2:
|
||||
sys.exit("--table attend T1/T2 ou non.")
|
||||
dans_table = lambda t: bornes[0] <= t <= bornes[1] # noqa: E731
|
||||
limits = [parse_time(x) for x in o.table.split("/")]
|
||||
if len(limits) != 2:
|
||||
sys.exit("--table expects T1/T2 or no.")
|
||||
in_table = lambda t: limits[0] <= t <= limits[1] # noqa: E731
|
||||
|
||||
noms = [os.path.basename(os.path.normpath(d)) for d in o.dossiers]
|
||||
donnees, mesures = {}, {}
|
||||
for n, d in zip(noms, o.dossiers):
|
||||
donnees[n], mesures[n] = lire(d, o.debut, o.fin)
|
||||
tous = [p["t"] for s in donnees.values() for c in s.values() for p in c]
|
||||
if not tous:
|
||||
sys.exit("Aucune valeur dans la fenêtre demandée.")
|
||||
t0 = (o.debut or min(tous)).replace(minute=0, second=0, microsecond=0)
|
||||
t1 = o.fin or max(tous)
|
||||
grille = [t0 + i * o.pas for i in range(int((t1 - t0) / o.pas) + 1)]
|
||||
case = lambda t: t0 + ((t - t0) // o.pas) * o.pas # noqa: E731
|
||||
names = [os.path.basename(os.path.normpath(d)) for d in o.folders]
|
||||
by_station, measured = {}, {}
|
||||
for n, d in zip(names, o.folders):
|
||||
by_station[n], measured[n] = read(d, o.start, o.end)
|
||||
times = [p["t"] for s in by_station.values() for c in s.values() for p in c]
|
||||
if not times:
|
||||
sys.exit("No value in the requested window.")
|
||||
t0 = (o.start or min(times)).replace(minute=0, second=0, microsecond=0)
|
||||
t1 = o.end or max(times)
|
||||
grid = [t0 + i * o.step for i in range(int((t1 - t0) / o.step) + 1)]
|
||||
cell = lambda t: t0 + ((t - t0) // o.step) * o.step # noqa: E731
|
||||
|
||||
print("Stations :")
|
||||
for n in noms:
|
||||
ts = mesures[n]
|
||||
print("Stations:")
|
||||
for n in names:
|
||||
ts = measured[n]
|
||||
if not ts:
|
||||
print(f" {n} : aucune mesure dans la fenêtre")
|
||||
print(f" {n}: no measurement in the window")
|
||||
continue
|
||||
cad = statistics.median(b - a_ for a_, b in zip(ts, ts[1:])) if len(ts) > 1 else None
|
||||
print(f" {n} : mesures {iso(ts[0])} → {iso(ts[-1])}, {len(ts)} instants, cadence {round(cad.total_seconds() / 60) if cad else '-'} min ; "
|
||||
+ ", ".join(f"{c} {len(donnees[n].get(c, []))}" for c in colonnes))
|
||||
print(f" {n}: measurements {iso(ts[0])} → {iso(ts[-1])}, {len(ts)} times, cadence {round(cad.total_seconds() / 60) if cad else '-'} min; "
|
||||
+ ", ".join(f"{c} {len(by_station[n].get(c, []))}" for c in columns))
|
||||
|
||||
# trous : pas sans mesure, sans température, et les stations concernées
|
||||
avec = {n: {case(t) for t in mesures[n]} for n in noms}
|
||||
avec_t = {n: {case(p["t"]) for p in donnees[n]["temperature"]} for n in noms}
|
||||
print(f"\nTrous (pas de {int(o.pas.total_seconds() // 60)} min sans mesure ou sans température), avec les stations où ils manquent :")
|
||||
cles = {}
|
||||
for k in grille:
|
||||
m = tuple(n for n in noms if k not in avec[n])
|
||||
t = tuple(n for n in noms if k not in avec_t[n])
|
||||
# gaps: steps without a measurement, without temperature, and the stations concerned
|
||||
with_meas = {n: {cell(t) for t in measured[n]} for n in names}
|
||||
with_temp = {n: {cell(p["t"]) for p in by_station[n]["temperature"]} for n in names}
|
||||
print(f"\nGaps (steps of {int(o.step.total_seconds() // 60)} min without a measurement or without temperature), with the stations where they are missing:")
|
||||
keys = {}
|
||||
for k in grid:
|
||||
m = tuple(n for n in names if k not in with_meas[n])
|
||||
t = tuple(n for n in names if k not in with_temp[n])
|
||||
if m or t:
|
||||
cles.setdefault((m, t), []).append(k)
|
||||
lignes = []
|
||||
for (m, t), ks in cles.items():
|
||||
for d, f, nb in suites(ks, o.pas):
|
||||
lignes.append((d, f, nb, m, t))
|
||||
for d, f, nb, m, t in sorted(lignes):
|
||||
print(f" {iso(d)} → {iso(f)} ({nb} pas) : sans mesure {','.join(m) or '-'} ; sans température {','.join(t) or '-'}")
|
||||
if not lignes:
|
||||
print(" (aucun)")
|
||||
keys.setdefault((m, t), []).append(k)
|
||||
lines = []
|
||||
for (m, t), ks in keys.items():
|
||||
for d, f, nb in runs(ks, o.step):
|
||||
lines.append((d, f, nb, m, t))
|
||||
for d, f, nb, m, t in sorted(lines):
|
||||
print(f" {iso(d)} → {iso(f)} ({nb} steps): no measurement {','.join(m) or '-'}; no temperature {','.join(t) or '-'}")
|
||||
if not lines:
|
||||
print(" (none)")
|
||||
|
||||
stat = STATS[o.stat]
|
||||
table = {} # (station, colonne) -> {pas: valeur}
|
||||
for n in noms:
|
||||
for c in colonnes:
|
||||
cases = {}
|
||||
for p in donnees[n].get(c, []):
|
||||
cases.setdefault(case(p["t"]), []).append(p["v"])
|
||||
table[n, c] = {k: stat(v) for k, v in cases.items()}
|
||||
entete = [f"{n}:{c}" for n in noms for c in colonnes]
|
||||
print(f"\n{o.stat} par pas de {int(o.pas.total_seconds() // 60)} min (début du pas, UTC) ; « - » : aucune valeur"
|
||||
+ (f" (table : {o.table})" if o.table else "") + " :")
|
||||
if o.table != "non":
|
||||
print("\t".join(["heure"] + entete))
|
||||
for k in grille:
|
||||
if dans_table(k):
|
||||
v = [table[n, c].get(k) for n in noms for c in colonnes]
|
||||
table = {} # (station, column) -> {step: value}
|
||||
for n in names:
|
||||
for c in columns:
|
||||
cells = {}
|
||||
for p in by_station[n].get(c, []):
|
||||
cells.setdefault(cell(p["t"]), []).append(p["v"])
|
||||
table[n, c] = {k: stat(v) for k, v in cells.items()}
|
||||
header = [f"{n}:{c}" for n in names for c in columns]
|
||||
print(f"\n{o.stat} per step of {int(o.step.total_seconds() // 60)} min (start of the step, UTC); \"-\": no value"
|
||||
+ (f" (table: {o.table})" if o.table else "") + ":")
|
||||
if o.table != "no":
|
||||
print("\t".join(["time"] + header))
|
||||
for k in grid:
|
||||
if in_table(k):
|
||||
v = [table[n, c].get(k) for n in names for c in columns]
|
||||
if any(x is not None for x in v):
|
||||
print("\t".join([iso(k)] + ["-" if x is None else f"{x:.6g}" for x in v]))
|
||||
|
||||
print("\nCorrélation entre stations, r des valeurs, puis r des pas d'une valeur à la suivante (n : pas où les deux ont une valeur) :")
|
||||
for c in colonnes:
|
||||
for i, s1 in enumerate(noms):
|
||||
for s2 in noms[i + 1:]:
|
||||
print("\nCorrelation between stations, r of the values, then r of the steps from one value to the next (n: steps where both have a value):")
|
||||
for c in columns:
|
||||
for i, s1 in enumerate(names):
|
||||
for s2 in names[i + 1:]:
|
||||
ks = sorted(set(table[s1, c]) & set(table[s2, c]))
|
||||
v1, v2 = [table[s1, c][k] for k in ks], [table[s2, c][k] for k in ks]
|
||||
d1, d2 = [b - a_ for a_, b in zip(v1, v1[1:])], [b - a_ for a_, b in zip(v2, v2[1:])]
|
||||
print(f" {c:12} {s1}-{s2} : r valeurs {correlation(v1, v2)}, r pas {correlation(d1, d2)} (n = {len(ks)})")
|
||||
print(f" {c:12} {s1}-{s2}: r values {correlation(v1, v2)}, r steps {correlation(d1, d2)} (n = {len(ks)})")
|
||||
|
||||
fig, axes = plt.subplots(len(colonnes) + 1, 1, sharex=True, figsize=(14, 2.4 * (len(colonnes) + 1)), squeeze=False,
|
||||
layout="constrained", gridspec_kw={"height_ratios": [3] * len(colonnes) + [0.3 * len(noms)]})
|
||||
for ax, c in zip(axes[:-1, 0], colonnes):
|
||||
for i, n in enumerate(noms):
|
||||
s = donnees[n].get(c, [])
|
||||
fig, axes = plt.subplots(len(columns) + 1, 1, sharex=True, figsize=(14, 2.4 * (len(columns) + 1)), squeeze=False,
|
||||
layout="constrained", gridspec_kw={"height_ratios": [3] * len(columns) + [0.3 * len(names)]})
|
||||
for ax, c in zip(axes[:-1, 0], columns):
|
||||
for i, n in enumerate(names):
|
||||
s = by_station[n].get(c, [])
|
||||
if not s:
|
||||
continue
|
||||
med = 0 if c == "temperature" else statistics.median(p["v"] for p in s)
|
||||
ts, vs = coupe([{"t": p["t"], "v": p["v"] - med} for p in s])
|
||||
ts, vs = cut([{"t": p["t"], "v": p["v"] - med} for p in s])
|
||||
ax.plot(ts, vs, ".-", ms=2, lw=0.8, color=f"C{i % 10}", label=n)
|
||||
ax.set_ylabel(c if c == "temperature" else f"{c}\nmoins sa médiane", fontsize=8)
|
||||
ax.set_ylabel(c if c == "temperature" else f"{c}\nminus its median", fontsize=8)
|
||||
ax.grid(alpha=0.3)
|
||||
ax.legend(loc="upper left", bbox_to_anchor=(1.0, 1.0), fontsize=8)
|
||||
ax = axes[-1, 0]
|
||||
for i, n in enumerate(noms):
|
||||
for k in grille:
|
||||
if k not in avec[n]:
|
||||
ax.axvspan(k, k + o.pas, ymin=i / len(noms), ymax=(i + 1) / len(noms), color=f"C{i % 10}", lw=0)
|
||||
ax.set_yticks([(i + 0.5) / len(noms) for i in range(len(noms))], noms, fontsize=7)
|
||||
for i, n in enumerate(names):
|
||||
for k in grid:
|
||||
if k not in with_meas[n]:
|
||||
ax.axvspan(k, k + o.step, ymin=i / len(names), ymax=(i + 1) / len(names), color=f"C{i % 10}", lw=0)
|
||||
ax.set_yticks([(i + 0.5) / len(names) for i in range(len(names))], names, fontsize=7)
|
||||
ax.set_ylim(0, 1)
|
||||
ax.set_ylabel("sans mesure", fontsize=8)
|
||||
ax.set_xlim(grille[0], grille[-1] + o.pas)
|
||||
ax.set_ylabel("no measurement", fontsize=8)
|
||||
ax.set_xlim(grid[0], grid[-1] + o.step)
|
||||
ax.xaxis.set_major_formatter(mdates.DateFormatter("%d/%m\n%H:%MZ"))
|
||||
fig.suptitle(f"{', '.join(noms)} : même ligne de temps (heures UTC)")
|
||||
os.makedirs(o.sortie, exist_ok=True)
|
||||
chemin = os.path.join(o.sortie, f"{'-'.join(noms)}_stations_{borne(o.debut, 'debut')}_{borne(o.fin, 'fin')}.png")
|
||||
fig.savefig(chemin, dpi=100, bbox_inches="tight")
|
||||
print(f"\n{chemin}")
|
||||
fig.suptitle(f"{', '.join(names)}: same timeline (UTC times)")
|
||||
os.makedirs(o.out, exist_ok=True)
|
||||
path = os.path.join(o.out, f"{'-'.join(names)}_stations_{bound(o.start, 'start')}_{bound(o.end, 'end')}.png")
|
||||
fig.savefig(path, dpi=100, bbox_inches="tight")
|
||||
print(f"\n{path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user