Pumping: with --from, the last day pumped of each station is pumped again (pumped during the day, it was incomplete), as without --from; a range from the app no longer stops the current day at the hour of an earlier pumping

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-18 14:18:57 +01:00
1 parent ea0228a73a
commit 1b918facd9
1 file changed
+30 -5
+30 -5
View File
@@ -20,8 +20,9 @@ Output (default: the repository's `data/`, next to this folder):
<STATION>/camera/<STATION>_camera_<YYYYMMDD>.csv camera movements, per day
<STATION>/images/<YYYYMMDD>T<HHMM>Z_<id>.<ext> OPTICAL_SHIFT-IRON images (--images)
<STATION>/images_diff/… differential images (--diff)
Resumable: with `--from`, a file already written is not read again, whatever the dates asked; without `--from`, the
days of the range are replaced (images excepted). Days in UTC, `--to` excluded; the current day is pumped as it is,
Resumable: with `--from`, a file already written is not read again, whatever the dates asked, except from the last
day pumped of the station (pumped during the day, it was incomplete); without `--from`, the days of the range are
replaced (images excepted). Days in UTC, `--to` excluded; the current day is pumped as it is,
its metrics up to the last completed hour. To redo a day, delete its files.
No dependency on the app (~/monitoring-dashboard): its reads are copied here, in SQL, as they are (branch `explo`,
@@ -86,13 +87,27 @@ def acquisition_unit(conn, system_id):
return pick[0].id if pick else None
def pumped_days(folder):
"""The days of displacements already pumped."""
return {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date()
for p in (folder / "displacement").glob(f"{folder.name}_advanced_*.csv")}
def given_ranges(folder, start, end):
"""With --from, without --replace: (start, end, replace) to pump. The missing days of the range, and from the last
day pumped (pumped during the day, it was incomplete) to the end of the range pumped again, as update_ranges."""
last = max(pumped_days(folder), default=None)
if last is None or not start <= last < end:
return [(start, end, False)]
return ([(start, last, False)] if start < last else []) + [(last, end, True)]
def update_ranges(folder, end, tomorrow):
"""Without --from: (start, end, replace) to pump, from the days of displacements already pumped. The missing days
between the first and the last (ranges pumped separately, failed day) are filled without touching the others; the
last day is pumped again (pumped during the day, it was incomplete) up to now; a station with no pumped
displacements gets the last 30 days."""
days = {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date()
for p in (folder / "displacement").glob(f"{folder.name}_advanced_*.csv")}
days = pumped_days(folder)
if not days:
return [(tomorrow - timedelta(days=NEW_STATION_DAYS + 1), end, True)]
# a failed metrics day has no file at all (one query for all of them): it is a gap too; a station without any
@@ -443,7 +458,8 @@ def main():
if system_id is None:
print(f"{station}: unknown system")
continue
ranges = [(start, end, a.replace)] if start else update_ranges(a.out / station, end, tomorrow)
ranges = ([(start, end, True)] if start and a.replace else given_ranges(a.out / station, start, end)
if start else update_ranges(a.out / station, end, tomorrow))
unit = acquisition_unit(conn, system_id)
conn.rollback()
seen = {}
@@ -491,6 +507,15 @@ def check():
t = write_targets(Path(d), props, {"T1": 12, "REF": 13})
assert t.values.tolist() == [["OLD", "target", 1, 1.0], ["REF", "global_ref", 50000, 18.0],
["T1", "target", 20000, 2.0]], t
with tempfile.TemporaryDirectory() as d: # given_ranges: the last day pumped and after, pumped again
f, day = Path(d) / "S", lambda n: datetime(2026, 11, n).date()
assert given_ranges(f, day(1), day(19)) == [(day(1), day(19), False)] # nothing pumped
(f / "displacement").mkdir(parents=True)
for n in (2, 18):
(f / "displacement" / f"S_advanced_202611{n:02d}.csv").touch()
assert given_ranges(f, day(1), day(19)) == [(day(1), day(18), False), (day(18), day(19), True)]
assert given_ranges(f, day(18), day(19)) == [(day(18), day(19), True)]
assert given_ranges(f, day(1), day(10)) == [(day(1), day(10), False)] # the last day after the range
print("ok")