Harden MA7D_DROP inputs and report UNKNOWN stretches
- NaN/inf values are treated as missing instead of poisoning the running sum (which produced a false "not observed"). - Unreadable CSV files are skipped with a message naming the file; a station whose analysis fails is skipped with its name and the scan continues. - The summary now states hours without verdict and the longest UNKNOWN stretch (5 months on MISSISSIPPI were silent). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
79bd98c9cb
commit
53b4f7e47c
2 files changed
+91
-22
No files matched your search
+56
-22
@@ -7,6 +7,7 @@ import argparse
|
|||||||
import csv
|
import csv
|
||||||
import glob
|
import glob
|
||||||
import json
|
import json
|
||||||
|
import math
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
@@ -43,10 +44,13 @@ def parse_name(path):
|
|||||||
|
|
||||||
def load_csv(path):
|
def load_csv(path):
|
||||||
"""CSV ',time_utc,metric' -> {datetime UTC: float ou None (pas de mesure)}."""
|
"""CSV ',time_utc,metric' -> {datetime UTC: float ou None (pas de mesure)}."""
|
||||||
with open(path, newline="") as f:
|
try:
|
||||||
rows = csv.reader(f)
|
with open(path, newline="") as f:
|
||||||
next(rows)
|
rows = csv.reader(f)
|
||||||
return {datetime.fromisoformat(r[1]): float(r[2]) if r[2] else None for r in rows}
|
next(rows)
|
||||||
|
return {datetime.fromisoformat(r[1]): float(r[2]) if r[2] else None for r in rows}
|
||||||
|
except (StopIteration, IndexError, ValueError) as e:
|
||||||
|
raise ValueError(f"{path} : CSV illisible ({e!r})") from e
|
||||||
|
|
||||||
|
|
||||||
def load_stations(paths):
|
def load_stations(paths):
|
||||||
@@ -66,7 +70,10 @@ def load_stations(paths):
|
|||||||
if key in seen:
|
if key in seen:
|
||||||
raise ValueError(f"doublon {key[0]}/{key[1]} : {seen[key]} et {path}")
|
raise ValueError(f"doublon {key[0]}/{key[1]} : {seen[key]} et {path}")
|
||||||
seen[key] = real
|
seen[key] = real
|
||||||
stations.setdefault(key[0], {})[key[1]] = load_csv(path)
|
try:
|
||||||
|
stations.setdefault(key[0], {})[key[1]] = load_csv(path)
|
||||||
|
except ValueError as e:
|
||||||
|
print(f"ignoré : {e}", file=sys.stderr)
|
||||||
return stations
|
return stations
|
||||||
|
|
||||||
|
|
||||||
@@ -79,7 +86,8 @@ def run(series):
|
|||||||
if any((t - t0) % HOUR for t in v):
|
if any((t - t0) % HOUR for t in v):
|
||||||
raise ValueError("BusVoltage : horodatages hors de la grille horaire")
|
raise ValueError("BusVoltage : horodatages hors de la grille horaire")
|
||||||
grid = [t0 + i * HOUR for i in range((max(v) - t0) // HOUR + 1)]
|
grid = [t0 + i * HOUR for i in range((max(v) - t0) // HOUR + 1)]
|
||||||
vals = [v.get(t) for t in grid]
|
# NaN/inf (ex. export pandas) = pas de mesure : sinon un NaN pollue la somme courante.
|
||||||
|
vals = [x if x is not None and math.isfinite(x) else None for x in (v.get(t) for t in grid)]
|
||||||
|
|
||||||
# Moyenne glissante sur WINDOW_H créneaux (somme courante), définie si couverture suffisante.
|
# Moyenne glissante sur WINDOW_H créneaux (somme courante), définie si couverture suffisante.
|
||||||
ma, cov = [None] * len(grid), [0.0] * len(grid)
|
ma, cov = [None] * len(grid), [0.0] * len(grid)
|
||||||
@@ -150,25 +158,47 @@ def _v(x):
|
|||||||
return f"{x:.2f}".replace(".", ",")
|
return f"{x:.2f}".replace(".", ",")
|
||||||
|
|
||||||
|
|
||||||
|
def _unknown_line(station, points):
|
||||||
|
"""Heures sans verdict et plus longue plage : l'absence de verdict n'est ni une anomalie ni un état sain."""
|
||||||
|
best = cur = None
|
||||||
|
for p in points:
|
||||||
|
if p["value"] is None:
|
||||||
|
cur = (cur[0], p["t"], cur[2] + 1) if cur else (p["t"], p["t"], 1)
|
||||||
|
if best is None or cur[2] > best[2]:
|
||||||
|
best = cur
|
||||||
|
else:
|
||||||
|
cur = None
|
||||||
|
n = sum(p["value"] is None for p in points)
|
||||||
|
line = f"{station} : sans verdict (UNKNOWN) {n} h sur {len(points)}"
|
||||||
|
if best:
|
||||||
|
line += f" ; plus longue plage du {_d(best[0])} au {_d(best[1])}"
|
||||||
|
return line + "."
|
||||||
|
|
||||||
|
|
||||||
def summary(station, result, per):
|
def summary(station, result, per):
|
||||||
"""Résumé texte généré à partir des faits calculés (une ligne par période)."""
|
"""Résumé texte généré à partir des faits calculés (une ligne par période + heures sans verdict)."""
|
||||||
rule = f"Règle {RULE['id']} (seuil {_v(THRESHOLD_V)} V, source : cas de régression)."
|
rule = f"Règle {RULE['id']} (seuil {_v(THRESHOLD_V)} V, source : cas de régression)."
|
||||||
defined = [p for p in result["points"] if p["value"] is not None]
|
points = result["points"]
|
||||||
|
defined = [p for p in points if p["value"] is not None]
|
||||||
if not defined:
|
if not defined:
|
||||||
return f"{station} : analyse impossible (insufficient evidence) : historique trop court ou vide. {rule}"
|
lines = [f"{station} : analyse impossible (insufficient evidence) : aucun point avec verdict "
|
||||||
if not per:
|
f"(historique trop court, vide ou trop lacunaire). {rule}"]
|
||||||
|
elif not per:
|
||||||
w = min(defined, key=lambda p: p["value"])
|
w = min(defined, key=lambda p: p["value"])
|
||||||
return (f"{station} : aucune anomalie énergétique (not observed) sur la période analysée. "
|
lines = [f"{station} : aucune anomalie énergétique (not observed) sur les points avec verdict. "
|
||||||
f"Pire écart {_v(w['value'])} V le {_d(w['t'])}. {rule}")
|
f"Pire écart {_v(w['value'])} V le {_d(w['t'])}. {rule}"]
|
||||||
lines = []
|
else:
|
||||||
for p in per:
|
lines = []
|
||||||
w = next(q for q in result["points"] if q["t"] == p["worst_t"])
|
for p in per:
|
||||||
f = w["facts"]
|
w = next(q for q in points if q["t"] == p["worst_t"])
|
||||||
until = "la fin de l'export" if p["end"] is None else _d(p["end"])
|
f = w["facts"]
|
||||||
lines.append(
|
until = "la fin de l'export" if p["end"] is None else _d(p["end"])
|
||||||
f"{station} : anomalie énergétique (consistent with) du {_d(p['start'])} à {until}. "
|
lines.append(
|
||||||
f"Pire écart le {_d(w['t'])} : moyenne sur 7 jours {_v(f['ma_v'])} V, soit {_v(-w['value'])} V "
|
f"{station} : anomalie énergétique (consistent with) du {_d(p['start'])} à {until}. "
|
||||||
f"sous son maximum du {_d(f['ref_t'])} ({_v(f['ref_v'])} V). {rule}")
|
f"Pire écart le {_d(w['t'])} : moyenne sur 7 jours {_v(f['ma_v'])} V, soit {_v(-w['value'])} V "
|
||||||
|
f"sous son maximum du {_d(f['ref_t'])} ({_v(f['ref_v'])} V). {rule}")
|
||||||
|
if points:
|
||||||
|
lines.append(_unknown_line(station, points))
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
@@ -182,7 +212,11 @@ def main(argv=None):
|
|||||||
if "BusVoltage" not in series:
|
if "BusVoltage" not in series:
|
||||||
print(f"{station} : ignorée, pas de BusVoltage")
|
print(f"{station} : ignorée, pas de BusVoltage")
|
||||||
continue
|
continue
|
||||||
result = run(series)
|
try:
|
||||||
|
result = run(series)
|
||||||
|
except ValueError as e:
|
||||||
|
print(f"{station} : ignorée, {e}", file=sys.stderr)
|
||||||
|
continue
|
||||||
per = periods(result["points"])
|
per = periods(result["points"])
|
||||||
with open(os.path.join(a.out, f"{station}_energy.json"), "w") as f:
|
with open(os.path.join(a.out, f"{station}_energy.json"), "w") as f:
|
||||||
json.dump({"station": station, **result, "periods": per}, f, indent=1,
|
json.dump({"station": station, **result, "periods": per}, f, indent=1,
|
||||||
|
|||||||
@@ -73,6 +73,17 @@ class Loading(unittest.TestCase):
|
|||||||
self.assertEqual(list(st), ["JUNGFRAU"])
|
self.assertEqual(list(st), ["JUNGFRAU"])
|
||||||
self.assertIn("export.csv", err.getvalue())
|
self.assertIn("export.csv", err.getvalue())
|
||||||
|
|
||||||
|
def test_empty_csv_file_is_skipped_with_message(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
shutil.copy(JUNGFRAU_V, d)
|
||||||
|
empty = Path(d) / "FLORIDA_BusVoltage_2026_04_13_14_00 to 2026_10_09_14_00.csv"
|
||||||
|
empty.touch()
|
||||||
|
err = io.StringIO()
|
||||||
|
with redirect_stderr(err):
|
||||||
|
st = ec.load_stations([d])
|
||||||
|
self.assertEqual(list(st), ["JUNGFRAU"])
|
||||||
|
self.assertIn(empty.name, err.getvalue())
|
||||||
|
|
||||||
|
|
||||||
class Ma7dDrop(unittest.TestCase):
|
class Ma7dDrop(unittest.TestCase):
|
||||||
def test_florida_never_anomalous(self):
|
def test_florida_never_anomalous(self):
|
||||||
@@ -127,6 +138,11 @@ class Ma7dDrop(unittest.TestCase):
|
|||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
ec.run({"BusVoltage": v})
|
ec.run({"BusVoltage": v})
|
||||||
|
|
||||||
|
def test_nan_value_is_treated_as_missing(self):
|
||||||
|
v = dict(stations()["VERMONT"]["BusVoltage"])
|
||||||
|
v[at(2026, 6, 1)] = float("nan")
|
||||||
|
self.assertEqual(first_anomaly(ec.run({"BusVoltage": v})), at(2026, 9, 24, 7))
|
||||||
|
|
||||||
|
|
||||||
class Reporting(unittest.TestCase):
|
class Reporting(unittest.TestCase):
|
||||||
def test_vermont_period_is_ongoing(self):
|
def test_vermont_period_is_ongoing(self):
|
||||||
@@ -166,6 +182,25 @@ class Reporting(unittest.TestCase):
|
|||||||
ec.main([str(VERMONT_THERMAL), "--out", d])
|
ec.main([str(VERMONT_THERMAL), "--out", d])
|
||||||
self.assertIn("ignorée, pas de BusVoltage", out.getvalue())
|
self.assertIn("ignorée, pas de BusVoltage", out.getvalue())
|
||||||
|
|
||||||
|
def test_cli_continues_after_station_error(self):
|
||||||
|
out, err = io.StringIO(), io.StringIO()
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
shutil.copy(JUNGFRAU_V, d)
|
||||||
|
florida = Path(d) / "FLORIDA_BusVoltage_2026_04_13_14_00 to 2026_10_09_14_00.csv"
|
||||||
|
shutil.copy(DATA / florida.name, florida)
|
||||||
|
with open(florida, "a") as f:
|
||||||
|
f.write(",2026-10-09 14:30:00+00:00,\n") # horodatage hors grille, sans mesure
|
||||||
|
with redirect_stdout(out), redirect_stderr(err):
|
||||||
|
ec.main([d, "--out", d])
|
||||||
|
self.assertIn("FLORIDA : ignorée", err.getvalue())
|
||||||
|
self.assertIn("JUNGFRAU : aucune anomalie", out.getvalue())
|
||||||
|
|
||||||
|
def test_summary_reports_longest_unknown_stretch(self):
|
||||||
|
res = result("MISSISSIPPI")
|
||||||
|
text = ec.summary("MISSISSIPPI", res, ec.periods(res["points"]))
|
||||||
|
self.assertIn("sans verdict (UNKNOWN)", text)
|
||||||
|
self.assertIn("du 2025-12-26 23:00 au 2026-06-07 16:00", text)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
Reference in new issue
Block a user