Pump system metrics and resume day by day (point 43)
pomper.py now also pumps every system metric of a station, in the format of the System page download (hourly mean, text lists, empty hour), one CSV per metric and per day; displacements are written one file per day, today is never pumped, and a rerun only fetches missing days. Fix image resume: an image already on disk now counts for its time slot. load_stations merges several files of the same metric instead of failing on a duplicate. Not run against the database: first run by the user, with point 27. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6ed1264ec8
commit
a81cc4c380
7 files changed
+152
-68
No files matched your search
+12
-8
@@ -38,27 +38,31 @@ def load_csv(path):
|
||||
|
||||
|
||||
def load_stations(paths):
|
||||
"""Fichiers et/ou dossiers -> {station: {metric: série}}."""
|
||||
"""Fichiers et/ou dossiers -> {station: {metric: série}}.
|
||||
|
||||
Plusieurs fichiers d'une même métrique (un par jour du pompage, exports qui se chevauchent, copies « (1) ») sont
|
||||
fusionnés en une série triée ; à heure égale, le dernier fichier par ordre de nom l'emporte.
|
||||
"""
|
||||
files = []
|
||||
for p in paths:
|
||||
files += sorted(glob.glob(os.path.join(p, "*.csv"))) if os.path.isdir(p) else [p]
|
||||
stations, seen = {}, {}
|
||||
stations, seen = {}, set()
|
||||
for path in files:
|
||||
real = os.path.realpath(path)
|
||||
if real in seen.values():
|
||||
if real in seen:
|
||||
continue
|
||||
seen.add(real)
|
||||
key = parse_name(path)
|
||||
if key is None:
|
||||
print(f"ignoré (nom non reconnu) : {path}", file=sys.stderr)
|
||||
continue
|
||||
if key in seen:
|
||||
raise ValueError(f"doublon {key[0]}/{key[1]} : {seen[key]} et {path}")
|
||||
seen[key] = real
|
||||
try:
|
||||
stations.setdefault(key[0], {})[key[1]] = load_csv(path)
|
||||
serie = load_csv(path)
|
||||
except ValueError as e:
|
||||
print(f"ignoré : {e}", file=sys.stderr)
|
||||
return stations
|
||||
continue
|
||||
stations.setdefault(key[0], {}).setdefault(key[1], {}).update(serie)
|
||||
return {s: {m: dict(sorted(v.items())) for m, v in ms.items()} for s, ms in stations.items()}
|
||||
|
||||
|
||||
def gaps(v):
|
||||
|
||||
Reference in new issue
Block a user