From 1b918facd90c4f8a374f590e19ef564b4ad368ed Mon Sep 17 00:00:00 2001 From: maxime Date: Wed, 18 Nov 2026 14:18:57 +0100 Subject: [PATCH] Pumping: with --from, the last day pumped of each station is pumped again (pumped during the day, it was incomplete), as without --from; a range from the app no longer stops the current day at the hour of an earlier pumping Co-Authored-By: Claude Opus 5.5 --- extraction/pump.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/extraction/pump.py b/extraction/pump.py index 965889f..51d7991 100644 --- a/extraction/pump.py +++ b/extraction/pump.py @@ -20,8 +20,9 @@ Output (default: the repository's `data/`, next to this folder): /camera/_camera_.csv camera movements, per day /images/TZ_. OPTICAL_SHIFT-IRON images (--images) /images_diff/… differential images (--diff) -Resumable: with `--from`, a file already written is not read again, whatever the dates asked; without `--from`, the -days of the range are replaced (images excepted). Days in UTC, `--to` excluded; the current day is pumped as it is, +Resumable: with `--from`, a file already written is not read again, whatever the dates asked, except from the last +day pumped of the station (pumped during the day, it was incomplete); without `--from`, the days of the range are +replaced (images excepted). Days in UTC, `--to` excluded; the current day is pumped as it is, its metrics up to the last completed hour. To redo a day, delete its files. No dependency on the app (~/monitoring-dashboard): its reads are copied here, in SQL, as they are (branch `explo`, @@ -86,13 +87,27 @@ def acquisition_unit(conn, system_id): return pick[0].id if pick else None +def pumped_days(folder): + """The days of displacements already pumped.""" + return {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date() + for p in (folder / "displacement").glob(f"{folder.name}_advanced_*.csv")} + + +def given_ranges(folder, start, end): + """With --from, without --replace: (start, end, replace) to pump. The missing days of the range, and from the last + day pumped (pumped during the day, it was incomplete) to the end of the range pumped again, as update_ranges.""" + last = max(pumped_days(folder), default=None) + if last is None or not start <= last < end: + return [(start, end, False)] + return ([(start, last, False)] if start < last else []) + [(last, end, True)] + + def update_ranges(folder, end, tomorrow): """Without --from: (start, end, replace) to pump, from the days of displacements already pumped. The missing days between the first and the last (ranges pumped separately, failed day) are filled without touching the others; the last day is pumped again (pumped during the day, it was incomplete) up to now; a station with no pumped displacements gets the last 30 days.""" - days = {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date() - for p in (folder / "displacement").glob(f"{folder.name}_advanced_*.csv")} + days = pumped_days(folder) if not days: return [(tomorrow - timedelta(days=NEW_STATION_DAYS + 1), end, True)] # a failed metrics day has no file at all (one query for all of them): it is a gap too; a station without any @@ -443,7 +458,8 @@ def main(): if system_id is None: print(f"{station}: unknown system") continue - ranges = [(start, end, a.replace)] if start else update_ranges(a.out / station, end, tomorrow) + ranges = ([(start, end, True)] if start and a.replace else given_ranges(a.out / station, start, end) + if start else update_ranges(a.out / station, end, tomorrow)) unit = acquisition_unit(conn, system_id) conn.rollback() seen = {} @@ -491,6 +507,15 @@ def check(): t = write_targets(Path(d), props, {"T1": 12, "REF": 13}) assert t.values.tolist() == [["OLD", "target", 1, 1.0], ["REF", "global_ref", 50000, 18.0], ["T1", "target", 20000, 2.0]], t + with tempfile.TemporaryDirectory() as d: # given_ranges: the last day pumped and after, pumped again + f, day = Path(d) / "S", lambda n: datetime(2026, 11, n).date() + assert given_ranges(f, day(1), day(19)) == [(day(1), day(19), False)] # nothing pumped + (f / "displacement").mkdir(parents=True) + for n in (2, 18): + (f / "displacement" / f"S_advanced_202611{n:02d}.csv").touch() + assert given_ranges(f, day(1), day(19)) == [(day(1), day(18), False), (day(18), day(19), True)] + assert given_ranges(f, day(18), day(19)) == [(day(18), day(19), True)] + assert given_ranges(f, day(1), day(10)) == [(day(1), day(10), False)] # the last day after the range print("ok")