218 lines
11 KiB
Python
218 lines
11 KiB
Python
"""Chiffres de l'analyse des métriques de qualité, LENZSPITZE, lot 2026-10-16 → 2026-11-05T11:00Z.
|
||
Usage (racine du dépôt) : python3 support/2026-11-12_LENZSPITZE_metrique-qualite/analyste/outils/stats.py [section]
|
||
Sections : points (défaut), episode DEBUT FIN, correl, jours, intact, plage DEBUT FIN, matins, instant DEBUT FIN. Lecture seule de data/LENZSPITZE/."""
|
||
import csv, glob, json, math, statistics, sys
|
||
from collections import defaultdict
|
||
from datetime import datetime
|
||
|
||
DOSSIER = "data/LENZSPITZE"
|
||
DEBUT, FIN = datetime.fromisoformat("2026-10-16T00:00+00:00"), datetime.fromisoformat("2026-11-05T11:00+00:00")
|
||
CALME = (datetime.fromisoformat("2026-10-28T00:00+00:00"), datetime.fromisoformat("2026-11-01T00:00+00:00"))
|
||
METRIQUES = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"]
|
||
|
||
|
||
def charger():
|
||
"""Une liste de dicts par capteur, triée par temps ; valeurs float (None si vide)."""
|
||
serie = defaultdict(list)
|
||
for f in sorted(glob.glob(f"{DOSSIER}/displacement/*_advanced_*.csv")):
|
||
for r in csv.DictReader(open(f)):
|
||
t = datetime.fromisoformat(r["time_utc"])
|
||
if not DEBUT <= t <= FIN:
|
||
continue
|
||
d = {"t": t}
|
||
for k in ["error_value", "error_threshold", "reproj_err", "x", "y"] + METRIQUES:
|
||
d[k] = float(r[k]) if r[k] else None
|
||
d["motif"] = r["pattern_type"]
|
||
serie[r["sensor_name"]].append(d)
|
||
for s in serie.values():
|
||
s.sort(key=lambda d: d["t"])
|
||
return serie
|
||
|
||
|
||
def motifs():
|
||
d = json.load(open(f"{DOSSIER}/lentzpitze_patterns.json"))
|
||
out = {}
|
||
for v in d.values():
|
||
if isinstance(v, dict) and "sensor_name" in v:
|
||
c = v["corners"]
|
||
out[str(v["sensor_name"])] = {"dist_m": v["distance_mm"] / 1000, "taille_px": (c[1][0] - c[0][0] + c[1][1] - c[0][1]) / 2,
|
||
"motif": v["pattern_type"]}
|
||
return out
|
||
|
||
|
||
def reference_calme(mesures, m):
|
||
"""Médiane de la métrique m par heure UTC du jour sur les jours calmes."""
|
||
par_h = defaultdict(list)
|
||
for d in mesures:
|
||
if CALME[0] <= d["t"] < CALME[1] and d[m] is not None:
|
||
par_h[d["t"].hour].append(d[m])
|
||
return {h: statistics.median(v) for h, v in par_h.items()}
|
||
|
||
|
||
def rang(v):
|
||
o = sorted(range(len(v)), key=v.__getitem__)
|
||
r = [0] * len(v)
|
||
for i, j in enumerate(o):
|
||
r[j] = i
|
||
return r
|
||
|
||
|
||
def spearman(a, b):
|
||
ra, rb = rang(a), rang(b)
|
||
return statistics.correlation(ra, rb) if len(a) > 2 else float("nan")
|
||
|
||
|
||
def points(serie, meta):
|
||
print("point type motif dist_m taille ev/seuil_med p95 %rej flux_med snr_med sharp_med fwhm_med dom_med pk/max")
|
||
pkmax = max(d["peak"] for s in serie.values() for d in s if d["peak"] is not None)
|
||
lignes = []
|
||
for n, s in serie.items():
|
||
glob_ref = any(d["reproj_err"] is not None for d in s)
|
||
ev = [d["error_value"] / d["error_threshold"] for d in s]
|
||
rej = sum(e > 1 for e in ev) / len(ev) * 100
|
||
med = {m: statistics.median(d[m] for d in s if d[m] is not None) for m in METRIQUES}
|
||
q = statistics.quantiles(ev, n=20)[-1]
|
||
mm = meta.get(n, {})
|
||
lignes.append((n, "réf" if glob_ref else "cib", s[0]["motif"] or "-", mm.get("dist_m"), mm.get("taille_px"),
|
||
statistics.median(ev), q, rej, med, med["peak"] / pkmax))
|
||
lignes.sort(key=lambda l: -l[5])
|
||
for n, ty, mo, di, ta, e, q, rj, med, pk in lignes:
|
||
print(f"{n:6} {ty:4} {mo:10} {di if di else '-':>6} {ta if ta else '-':>6} {e:12.2f} {q:5.2f} {rj:5.1f} {med['flux']:9.0f} {med['snr']:8.0f}"
|
||
f" {med['sharpness']:9.3f} {med['fwhm']:8.2f} {med['dominance']:8.3f} {pk:6.2f}")
|
||
# niveau habituel face à la distance, la taille, le snr (rangs, tous points avec motif)
|
||
av = [l for l in lignes if l[3] is not None]
|
||
for nom, idx in (("distance", 3), ("taille", 4)):
|
||
print(f"Spearman ev/seuil médian ~ {nom} (n={len(av)}) : {spearman([l[5] for l in av], [l[idx] for l in av]):+.2f}")
|
||
for m in ("snr", "flux", "sharpness", "dominance", "fwhm"):
|
||
print(f"Spearman ev/seuil médian ~ {m} médian (n={len(lignes)}) : {spearman([l[5] for l in lignes], [l[8][m] for l in lignes]):+.2f}")
|
||
|
||
|
||
def correl(serie):
|
||
"""Par point, sur le lot, Spearman entre error_value et chaque métrique (rapport à la référence calme)."""
|
||
print("Spearman, par point, error_value ~ métrique (sur toutes les mesures du lot) ; médiane sur les points, puis par point")
|
||
tous = defaultdict(list)
|
||
for n, s in sorted(serie.items()):
|
||
ligne = []
|
||
for m in METRIQUES:
|
||
v = [(d["error_value"], d[m]) for d in s if d[m] is not None]
|
||
r = spearman([a for a, _ in v], [b for _, b in v])
|
||
tous[m].append(r)
|
||
ligne.append(f"{m} {r:+.2f}")
|
||
print(f" {n:6} " + " ".join(ligne))
|
||
print("médiane : " + " ".join(f"{m} {statistics.median(tous[m]):+.2f}" for m in METRIQUES))
|
||
|
||
|
||
def episode(serie, meta, debut, fin):
|
||
"""Par point : error_value/seuil max, rapport flux min / snr min / sharpness min / dominance max à la réf. calme."""
|
||
t0, t1 = datetime.fromisoformat(debut), datetime.fromisoformat(fin)
|
||
print(f"Épisode {debut} → {fin} : par point, extrêmes sur la plage (rapports à la référence calme {CALME[0]:%d.%m}–{CALME[1]:%d.%m}, même heure)")
|
||
print("point dist_m ev/seuil_max n>seuil flux_min snr_min sharp_min fwhm_max dom_max")
|
||
rows = []
|
||
for n, s in serie.items():
|
||
ref = {m: reference_calme(s, m) for m in ("flux", "snr", "sharpness", "fwhm", "dominance")}
|
||
pl = [d for d in s if t0 <= d["t"] <= t1]
|
||
if not pl:
|
||
continue
|
||
def rap(m, f):
|
||
v = [d[m] / ref[m][d["t"].hour] for d in pl if d[m] is not None and ref[m].get(d["t"].hour)]
|
||
return f(v) if v else float("nan")
|
||
ev = [d["error_value"] / d["error_threshold"] for d in pl]
|
||
rows.append((n, meta.get(n, {}).get("dist_m"), max(ev), sum(e > 1 for e in ev), rap("flux", min), rap("snr", min),
|
||
rap("sharpness", min), rap("fwhm", max), rap("dominance", max)))
|
||
rows.sort(key=lambda r: -(r[1] or 0))
|
||
for r in rows:
|
||
print(f"{r[0]:6} {r[1] if r[1] else '-':>6} {r[2]:12.2f} {r[3]:8d} {r[4]:9.3f} {r[5]:8.3f} {r[6]:9.3f} {r[7]:9.2f} {r[8]:8.1f}")
|
||
|
||
|
||
def jours(serie):
|
||
"""Par jour : part des mesures au-dessus du seuil, cibles / références globales."""
|
||
c = defaultdict(lambda: [0, 0, 0, 0])
|
||
for n, s in serie.items():
|
||
glob_ref = any(d["reproj_err"] is not None for d in s)
|
||
for d in s:
|
||
k = c[d["t"].date()]
|
||
k[2 * glob_ref] += 1
|
||
k[2 * glob_ref + 1] += d["error_value"] > d["error_threshold"]
|
||
print("jour cibles %>seuil réf. %>seuil")
|
||
for j in sorted(c):
|
||
a = c[j]
|
||
print(f"{j} {100 * a[1] / a[0]:6.2f} {100 * a[3] / max(a[2], 1):6.2f}")
|
||
|
||
|
||
def intact(serie, seuil_flux=0.6):
|
||
"""Mesures au-dessus du seuil dont le flux reste entier (rapport >= seuil_flux) : ce qui les explique, sinon la lumière."""
|
||
print(f"Mesures au-dessus du seuil avec flux >= {seuil_flux} x habituel ; rapports à la référence calme")
|
||
print("instant point ev/seuil flux snr sharp fwhm dom peak")
|
||
for n, s in sorted(serie.items()):
|
||
ref = {m: reference_calme(s, m) for m in METRIQUES}
|
||
for d in s:
|
||
h = d["t"].hour
|
||
if d["error_value"] <= d["error_threshold"] or not all(ref[m].get(h) for m in METRIQUES):
|
||
continue
|
||
r = {m: d[m] / ref[m][h] for m in METRIQUES}
|
||
if r["flux"] >= seuil_flux:
|
||
print(f"{d['t']:%Y-%m-%d %H:%M} {n:6} {d['error_value'] / d['error_threshold']:7.2f} {r['flux']:5.2f} {r['snr']:5.2f}"
|
||
f" {r['sharpness']:5.2f} {r['fwhm']:5.2f} {r['dominance']:5.1f} {r['peak']:5.2f}")
|
||
|
||
|
||
def plage(serie, debut, fin):
|
||
"""Par point, sur la plage : snr et flux (rapports à la référence calme), min et médiane ; error_value / seuil max."""
|
||
t0, t1 = datetime.fromisoformat(debut), datetime.fromisoformat(fin)
|
||
print(f"Plage {debut} → {fin}")
|
||
for n, s in sorted(serie.items()):
|
||
ref = {m: reference_calme(s, m) for m in ("flux", "snr")}
|
||
v = [(d["snr"] / ref["snr"][d["t"].hour], d["flux"] / ref["flux"][d["t"].hour], d["error_value"] / d["error_threshold"])
|
||
for d in s if t0 <= d["t"] <= t1]
|
||
print(f"{n:6} snr min {min(a for a, _, _ in v):.2f} méd {statistics.median(a for a, _, _ in v):.2f}"
|
||
f" flux min {min(b for _, b, _ in v):.2f} méd {statistics.median(b for _, b, _ in v):.2f} ev/seuil max {max(c for *_, c in v):.2f}")
|
||
|
||
|
||
def matins(serie, noms=("202H", "101B")):
|
||
"""Par jour, entre 06 et 08 h UTC, la mesure où le snr (rapport) est le plus bas, avec flux, dominance, error_value / seuil."""
|
||
for n in noms:
|
||
ref = {m: reference_calme(serie[n], m) for m in ("flux", "snr", "dominance")}
|
||
jour = {}
|
||
for d in serie[n]:
|
||
h = d["t"].hour
|
||
if 6 <= h <= 8:
|
||
r = d["snr"] / ref["snr"][h]
|
||
if d["t"].date() not in jour or r < jour[d["t"].date()][1]:
|
||
jour[d["t"].date()] = (d["t"], r, d["flux"] / ref["flux"][h], d["dominance"] / ref["dominance"][h],
|
||
d["error_value"] / d["error_threshold"])
|
||
for v in sorted(jour.values()):
|
||
print(f"{n} {v[0]:%d.%m %H:%M} snr {v[1]:.2f} flux {v[2]:.2f} dom {v[3]:.1f} ev/seuil {v[4]:.2f}")
|
||
|
||
|
||
def instant(serie, debut, fin):
|
||
"""Par point et par mesure entre debut et fin : error_value / seuil et rapports des six métriques à la référence calme."""
|
||
t0, t1 = datetime.fromisoformat(debut), datetime.fromisoformat(fin)
|
||
print("point instant ev/seuil " + " ".join(METRIQUES))
|
||
for n, s in sorted(serie.items()):
|
||
ref = {m: reference_calme(s, m) for m in METRIQUES}
|
||
for d in s:
|
||
if t0 <= d["t"] <= t1:
|
||
h = d["t"].hour
|
||
print(f"{n:6} {d['t']:%d.%m %H:%M} {d['error_value'] / d['error_threshold']:6.2f} "
|
||
+ " ".join(f"{d[m] / ref[m][h]:5.2f}" for m in METRIQUES))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sec = sys.argv[1] if len(sys.argv) > 1 else "points"
|
||
serie = charger()
|
||
if sec == "points":
|
||
points(serie, motifs())
|
||
elif sec == "correl":
|
||
correl(serie)
|
||
elif sec == "jours":
|
||
jours(serie)
|
||
elif sec == "intact":
|
||
intact(serie)
|
||
elif sec == "plage":
|
||
plage(serie, sys.argv[2], sys.argv[3])
|
||
elif sec == "instant":
|
||
instant(serie, sys.argv[2], sys.argv[3])
|
||
elif sec == "matins":
|
||
matins(serie)
|
||
elif sec == "episode":
|
||
episode(serie, motifs(), sys.argv[2], sys.argv[3])
|