Pump system metrics and resume day by day (point 43)

pomper.py now also pumps every system metric of a station, in the format of
the System page download (hourly mean, text lists, empty hour), one CSV per
metric and per day; displacements are written one file per day, today is
never pumped, and a rerun only fetches missing days. Fix image resume: an
image already on disk now counts for its time slot. load_stations merges
several files of the same metric instead of failing on a duplicate.

Not run against the database: first run by the user, with point 27.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-10-24 19:14:59 +02:00
1 parent 6ed1264ec8
commit a81cc4c380
7 files changed
+152 -68

No files matched your search

+12 -8
View File
@@ -38,27 +38,31 @@ def load_csv(path):
def load_stations(paths):
"""Fichiers et/ou dossiers -> {station: {metric: série}}."""
"""Fichiers et/ou dossiers -> {station: {metric: série}}.
Plusieurs fichiers d'une même métrique (un par jour du pompage, exports qui se chevauchent, copies « (1) ») sont
fusionnés en une série triée ; à heure égale, le dernier fichier par ordre de nom l'emporte.
"""
files = []
for p in paths:
files += sorted(glob.glob(os.path.join(p, "*.csv"))) if os.path.isdir(p) else [p]
stations, seen = {}, {}
stations, seen = {}, set()
for path in files:
real = os.path.realpath(path)
if real in seen.values():
if real in seen:
continue
seen.add(real)
key = parse_name(path)
if key is None:
print(f"ignoré (nom non reconnu) : {path}", file=sys.stderr)
continue
if key in seen:
raise ValueError(f"doublon {key[0]}/{key[1]} : {seen[key]} et {path}")
seen[key] = real
try:
stations.setdefault(key[0], {})[key[1]] = load_csv(path)
serie = load_csv(path)
except ValueError as e:
print(f"ignoré : {e}", file=sys.stderr)
return stations
continue
stations.setdefault(key[0], {}).setdefault(key[1], {}).update(serie)
return {s: {m: dict(sorted(v.items())) for m, v in ms.items()} for s, ms in stations.items()}
def gaps(v):