Add the diagnostic agent's data reader and inventory tool (D-046)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7892fd9126
commit
80d54a688f
2 files changed
+166
No files matched your search
@@ -0,0 +1,89 @@
|
|||||||
|
"""Lecture des données d'une station pour l'agent de diagnostic (bibliothèque standard).
|
||||||
|
|
||||||
|
Formats : docs/domaine/colonnes-des-deplacements.md. Toute heure est en UTC.
|
||||||
|
"""
|
||||||
|
import csv
|
||||||
|
import glob
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
# Colonnes numériques de l'export « advanced » ; une case vide devient None.
|
||||||
|
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
|
||||||
|
"corr_x", "corr_y", "reproj_err", "sigma0")
|
||||||
|
IMAGE_RE = re.compile(r"^(\d{8}T\d{4})Z_")
|
||||||
|
|
||||||
|
|
||||||
|
def _entete(chemin):
|
||||||
|
with open(chemin, newline="") as f:
|
||||||
|
return next(csv.reader(f), [])
|
||||||
|
|
||||||
|
|
||||||
|
def trouver(dossier):
|
||||||
|
"""Dossier de station -> {'advanced', 'horaire', 'images', 'autres'} : chemins triés, sous-dossiers compris."""
|
||||||
|
out = {"advanced": [], "horaire": [], "images": [], "autres": []}
|
||||||
|
for p in sorted(glob.glob(os.path.join(dossier, "**", "*"), recursive=True)):
|
||||||
|
if os.path.isdir(p):
|
||||||
|
continue
|
||||||
|
entete = _entete(p) if p.endswith(".csv") else []
|
||||||
|
if p.lower().endswith((".jpg", ".jpeg", ".png")):
|
||||||
|
out["images"].append(p)
|
||||||
|
elif "sensor_name" in entete:
|
||||||
|
out["advanced"].append(p)
|
||||||
|
elif "timestamp" in entete:
|
||||||
|
out["horaire"].append(p)
|
||||||
|
else:
|
||||||
|
out["autres"].append(p)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _nombre(v):
|
||||||
|
v = v.strip()
|
||||||
|
return float(v) if v else None
|
||||||
|
|
||||||
|
|
||||||
|
def charger(dossier):
|
||||||
|
"""Exports « advanced » du dossier -> {capteur: [mesure]}, triées par t, un seul relevé par instant.
|
||||||
|
|
||||||
|
Mesure : dict avec t (datetime UTC), pattern_type, et les colonnes de NUM (float ou None).
|
||||||
|
"""
|
||||||
|
par = {}
|
||||||
|
for chemin in trouver(dossier)["advanced"]:
|
||||||
|
with open(chemin, newline="") as f:
|
||||||
|
for r in csv.DictReader(f):
|
||||||
|
m = {c: _nombre(r[c]) for c in NUM}
|
||||||
|
m["t"] = datetime.fromisoformat(r["time_utc"]).astimezone(timezone.utc)
|
||||||
|
m["pattern_type"] = r["pattern_type"]
|
||||||
|
par.setdefault(r["sensor_name"], {})[m["t"]] = m
|
||||||
|
return {s: [d[t] for t in sorted(d)] for s, d in par.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def est_reference(mesures):
|
||||||
|
"""Référence du réseau de compensation (pattern_type global_ref), sinon cible."""
|
||||||
|
return mesures[0]["pattern_type"] == "global_ref"
|
||||||
|
|
||||||
|
|
||||||
|
def rejetee(m):
|
||||||
|
"""Mesure rejetée par la station (error_value au-dessus du seuil du capteur) : corr_x et corr_y vides."""
|
||||||
|
return m["corr_x"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def filtrer(mesures, debut, fin):
|
||||||
|
"""Mesures entre debut et fin inclus ; None = pas de borne."""
|
||||||
|
return [m for m in mesures if (debut is None or m["t"] >= debut) and (fin is None or m["t"] <= fin)]
|
||||||
|
|
||||||
|
|
||||||
|
def heure_image(chemin):
|
||||||
|
"""'20261013T1200Z_10167931.jpg' -> datetime UTC ; None si le nom ne suit pas ce format (D-045)."""
|
||||||
|
m = IMAGE_RE.match(os.path.basename(chemin))
|
||||||
|
return datetime.strptime(m.group(1), "%Y%m%dT%H%M").replace(tzinfo=timezone.utc) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def instant(texte):
|
||||||
|
"""'2026-10-13T23:00Z' ou '2026-10-13 23:00' -> datetime UTC ; une heure sans fuseau est lue en UTC."""
|
||||||
|
t = datetime.fromisoformat(texte)
|
||||||
|
return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def iso(t):
|
||||||
|
return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
"""Inventaire d'un dossier de station : ce qui est disponible, ce qui manque.
|
||||||
|
|
||||||
|
Usage : python3 agents/diagnostic/outils/inventaire.py data/PS_DES_COUDRES/
|
||||||
|
Rend des valeurs, pas de verdict.
|
||||||
|
"""
|
||||||
|
import statistics
|
||||||
|
import sys
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
from donnees import charger, est_reference, heure_image, iso, rejetee, trouver
|
||||||
|
|
||||||
|
|
||||||
|
def fichiers(f):
|
||||||
|
print(f"Exports « advanced » (source des mesures) : {len(f['advanced'])}")
|
||||||
|
for p in f["advanced"]:
|
||||||
|
print(f" {p}")
|
||||||
|
print(f"Exports horaires : {len(f['horaire'])} (non lus par les outils : moyennes horaires, "
|
||||||
|
"heure locale sans fuseau ; voir docs/domaine/colonnes-des-deplacements.md)")
|
||||||
|
for p in f["horaire"]:
|
||||||
|
print(f" {p}")
|
||||||
|
print(f"Autres fichiers : {len(f['autres'])}")
|
||||||
|
for p in f["autres"]:
|
||||||
|
print(f" {p}")
|
||||||
|
|
||||||
|
|
||||||
|
def images(chemins):
|
||||||
|
heures = sorted(h for h in map(heure_image, chemins) if h)
|
||||||
|
print(f"Images : {len(chemins)}" + (f", du {iso(heures[0])} au {iso(heures[-1])}" if heures else ""))
|
||||||
|
inconnues = [p for p in chemins if heure_image(p) is None]
|
||||||
|
if inconnues:
|
||||||
|
print(f" noms sans heure reconnue : {len(inconnues)}")
|
||||||
|
if heures:
|
||||||
|
manquantes = []
|
||||||
|
for a, b in zip(heures, heures[1:]):
|
||||||
|
h = a + timedelta(hours=1)
|
||||||
|
while h < b:
|
||||||
|
manquantes.append(iso(h))
|
||||||
|
h += timedelta(hours=1)
|
||||||
|
print(f" heures pleines sans image entre la première et la dernière : {', '.join(manquantes) or 'aucune'}")
|
||||||
|
|
||||||
|
|
||||||
|
def mesures(par):
|
||||||
|
if not par:
|
||||||
|
print("\nAucune mesure : pas d'export « advanced » dans ce dossier.")
|
||||||
|
return
|
||||||
|
instants = sorted({m["t"] for ms in par.values() for m in ms})
|
||||||
|
cadence = statistics.median(b - a for a, b in zip(instants, instants[1:]))
|
||||||
|
print(f"\nPériode des mesures : {iso(instants[0])} → {iso(instants[-1])}, {len(instants)} instants")
|
||||||
|
print(f"Cadence (pas médian) : {cadence.total_seconds() / 60:.0f} min")
|
||||||
|
trous = [(a, b, round((b - a) / cadence) - 1) for a, b in zip(instants, instants[1:])]
|
||||||
|
trous = [x for x in trous if x[2] > 0]
|
||||||
|
print(f"Mesures manquantes (instants absents de la cadence) : {sum(n for *_, n in trous)}")
|
||||||
|
for a, b, n in trous:
|
||||||
|
print(f" entre {iso(a)} et {iso(b)} : {n}")
|
||||||
|
noms = sorted(par, key=lambda s: (not est_reference(par[s]), s))
|
||||||
|
refs = [s for s in noms if est_reference(par[s])]
|
||||||
|
print(f"\nCapteurs : {len(par)} ({len(refs)} références, {len(par) - len(refs)} cibles)")
|
||||||
|
print(f"{'capteur':10} {'type':10} {'motif':11} {'seuil':>5} {'mesures':>7} {'rejetées':>8}")
|
||||||
|
for s in noms:
|
||||||
|
ms = par[s]
|
||||||
|
seuil = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms}))
|
||||||
|
print(f"{s:10} {'référence' if est_reference(ms) else 'cible':10} {ms[0]['pattern_type']:11} "
|
||||||
|
f"{seuil:>5} {len(ms):7} {sum(map(rejetee, ms)):8}")
|
||||||
|
|
||||||
|
|
||||||
|
def main(dossier):
|
||||||
|
print(f"Dossier : {dossier}")
|
||||||
|
f = trouver(dossier)
|
||||||
|
fichiers(f)
|
||||||
|
images(f["images"])
|
||||||
|
mesures(charger(dossier))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) != 2:
|
||||||
|
sys.exit(__doc__)
|
||||||
|
main(sys.argv[1])
|
||||||
Reference in new issue
Block a user