From 53b4f7e47ca05d200aee7d40895fcaa68cd3f75f Mon Sep 17 00:00:00 2001 From: maxime Date: Fri, 9 Oct 2026 19:37:56 +0200 Subject: [PATCH] Harden MA7D_DROP inputs and report UNKNOWN stretches - NaN/inf values are treated as missing instead of poisoning the running sum (which produced a false "not observed"). - Unreadable CSV files are skipped with a message naming the file; a station whose analysis fails is skipped with its name and the scan continues. - The summary now states hours without verdict and the longest UNKNOWN stretch (5 months on MISSISSIPPI were silent). Co-Authored-By: Claude Opus 5.5 --- src/energy_check.py | 78 +++++++++++++++++++++++++++----------- tests/test_energy_check.py | 35 +++++++++++++++++ 2 files changed, 91 insertions(+), 22 deletions(-) diff --git a/src/energy_check.py b/src/energy_check.py index dd93ef6..66f0814 100644 --- a/src/energy_check.py +++ b/src/energy_check.py @@ -7,6 +7,7 @@ import argparse import csv import glob import json +import math import os import re import sys @@ -43,10 +44,13 @@ def parse_name(path): def load_csv(path): """CSV ',time_utc,metric' -> {datetime UTC: float ou None (pas de mesure)}.""" - with open(path, newline="") as f: - rows = csv.reader(f) - next(rows) - return {datetime.fromisoformat(r[1]): float(r[2]) if r[2] else None for r in rows} + try: + with open(path, newline="") as f: + rows = csv.reader(f) + next(rows) + return {datetime.fromisoformat(r[1]): float(r[2]) if r[2] else None for r in rows} + except (StopIteration, IndexError, ValueError) as e: + raise ValueError(f"{path} : CSV illisible ({e!r})") from e def load_stations(paths): @@ -66,7 +70,10 @@ def load_stations(paths): if key in seen: raise ValueError(f"doublon {key[0]}/{key[1]} : {seen[key]} et {path}") seen[key] = real - stations.setdefault(key[0], {})[key[1]] = load_csv(path) + try: + stations.setdefault(key[0], {})[key[1]] = load_csv(path) + except ValueError as e: + print(f"ignoré : {e}", file=sys.stderr) return stations @@ -79,7 +86,8 @@ def run(series): if any((t - t0) % HOUR for t in v): raise ValueError("BusVoltage : horodatages hors de la grille horaire") grid = [t0 + i * HOUR for i in range((max(v) - t0) // HOUR + 1)] - vals = [v.get(t) for t in grid] + # NaN/inf (ex. export pandas) = pas de mesure : sinon un NaN pollue la somme courante. + vals = [x if x is not None and math.isfinite(x) else None for x in (v.get(t) for t in grid)] # Moyenne glissante sur WINDOW_H créneaux (somme courante), définie si couverture suffisante. ma, cov = [None] * len(grid), [0.0] * len(grid) @@ -150,25 +158,47 @@ def _v(x): return f"{x:.2f}".replace(".", ",") +def _unknown_line(station, points): + """Heures sans verdict et plus longue plage : l'absence de verdict n'est ni une anomalie ni un état sain.""" + best = cur = None + for p in points: + if p["value"] is None: + cur = (cur[0], p["t"], cur[2] + 1) if cur else (p["t"], p["t"], 1) + if best is None or cur[2] > best[2]: + best = cur + else: + cur = None + n = sum(p["value"] is None for p in points) + line = f"{station} : sans verdict (UNKNOWN) {n} h sur {len(points)}" + if best: + line += f" ; plus longue plage du {_d(best[0])} au {_d(best[1])}" + return line + "." + + def summary(station, result, per): - """Résumé texte généré à partir des faits calculés (une ligne par période).""" + """Résumé texte généré à partir des faits calculés (une ligne par période + heures sans verdict).""" rule = f"Règle {RULE['id']} (seuil {_v(THRESHOLD_V)} V, source : cas de régression)." - defined = [p for p in result["points"] if p["value"] is not None] + points = result["points"] + defined = [p for p in points if p["value"] is not None] if not defined: - return f"{station} : analyse impossible (insufficient evidence) : historique trop court ou vide. {rule}" - if not per: + lines = [f"{station} : analyse impossible (insufficient evidence) : aucun point avec verdict " + f"(historique trop court, vide ou trop lacunaire). {rule}"] + elif not per: w = min(defined, key=lambda p: p["value"]) - return (f"{station} : aucune anomalie énergétique (not observed) sur la période analysée. " - f"Pire écart {_v(w['value'])} V le {_d(w['t'])}. {rule}") - lines = [] - for p in per: - w = next(q for q in result["points"] if q["t"] == p["worst_t"]) - f = w["facts"] - until = "la fin de l'export" if p["end"] is None else _d(p["end"]) - lines.append( - f"{station} : anomalie énergétique (consistent with) du {_d(p['start'])} à {until}. " - f"Pire écart le {_d(w['t'])} : moyenne sur 7 jours {_v(f['ma_v'])} V, soit {_v(-w['value'])} V " - f"sous son maximum du {_d(f['ref_t'])} ({_v(f['ref_v'])} V). {rule}") + lines = [f"{station} : aucune anomalie énergétique (not observed) sur les points avec verdict. " + f"Pire écart {_v(w['value'])} V le {_d(w['t'])}. {rule}"] + else: + lines = [] + for p in per: + w = next(q for q in points if q["t"] == p["worst_t"]) + f = w["facts"] + until = "la fin de l'export" if p["end"] is None else _d(p["end"]) + lines.append( + f"{station} : anomalie énergétique (consistent with) du {_d(p['start'])} à {until}. " + f"Pire écart le {_d(w['t'])} : moyenne sur 7 jours {_v(f['ma_v'])} V, soit {_v(-w['value'])} V " + f"sous son maximum du {_d(f['ref_t'])} ({_v(f['ref_v'])} V). {rule}") + if points: + lines.append(_unknown_line(station, points)) return "\n".join(lines) @@ -182,7 +212,11 @@ def main(argv=None): if "BusVoltage" not in series: print(f"{station} : ignorée, pas de BusVoltage") continue - result = run(series) + try: + result = run(series) + except ValueError as e: + print(f"{station} : ignorée, {e}", file=sys.stderr) + continue per = periods(result["points"]) with open(os.path.join(a.out, f"{station}_energy.json"), "w") as f: json.dump({"station": station, **result, "periods": per}, f, indent=1, diff --git a/tests/test_energy_check.py b/tests/test_energy_check.py index 43a32af..f089d1d 100644 --- a/tests/test_energy_check.py +++ b/tests/test_energy_check.py @@ -73,6 +73,17 @@ class Loading(unittest.TestCase): self.assertEqual(list(st), ["JUNGFRAU"]) self.assertIn("export.csv", err.getvalue()) + def test_empty_csv_file_is_skipped_with_message(self): + with tempfile.TemporaryDirectory() as d: + shutil.copy(JUNGFRAU_V, d) + empty = Path(d) / "FLORIDA_BusVoltage_2026_04_13_14_00 to 2026_10_09_14_00.csv" + empty.touch() + err = io.StringIO() + with redirect_stderr(err): + st = ec.load_stations([d]) + self.assertEqual(list(st), ["JUNGFRAU"]) + self.assertIn(empty.name, err.getvalue()) + class Ma7dDrop(unittest.TestCase): def test_florida_never_anomalous(self): @@ -127,6 +138,11 @@ class Ma7dDrop(unittest.TestCase): with self.assertRaises(ValueError): ec.run({"BusVoltage": v}) + def test_nan_value_is_treated_as_missing(self): + v = dict(stations()["VERMONT"]["BusVoltage"]) + v[at(2026, 6, 1)] = float("nan") + self.assertEqual(first_anomaly(ec.run({"BusVoltage": v})), at(2026, 9, 24, 7)) + class Reporting(unittest.TestCase): def test_vermont_period_is_ongoing(self): @@ -166,6 +182,25 @@ class Reporting(unittest.TestCase): ec.main([str(VERMONT_THERMAL), "--out", d]) self.assertIn("ignorée, pas de BusVoltage", out.getvalue()) + def test_cli_continues_after_station_error(self): + out, err = io.StringIO(), io.StringIO() + with tempfile.TemporaryDirectory() as d: + shutil.copy(JUNGFRAU_V, d) + florida = Path(d) / "FLORIDA_BusVoltage_2026_04_13_14_00 to 2026_10_09_14_00.csv" + shutil.copy(DATA / florida.name, florida) + with open(florida, "a") as f: + f.write(",2026-10-09 14:30:00+00:00,\n") # horodatage hors grille, sans mesure + with redirect_stdout(out), redirect_stderr(err): + ec.main([d, "--out", d]) + self.assertIn("FLORIDA : ignorée", err.getvalue()) + self.assertIn("JUNGFRAU : aucune anomalie", out.getvalue()) + + def test_summary_reports_longest_unknown_stretch(self): + res = result("MISSISSIPPI") + text = ec.summary("MISSISSIPPI", res, ec.periods(res["points"])) + self.assertIn("sans verdict (UNKNOWN)", text) + self.assertIn("du 2025-12-26 23:00 au 2026-06-07 16:00", text) + if __name__ == "__main__": unittest.main()