Pumping: with --from, the last day pumped of each station is pumped again (pumped during the day, it was incomplete), as without --from; a range from the app no longer stops the current day at the hour of an earlier pumping
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ea0228a73a
commit
1b918facd9
1 file changed
+30
-5
+30
-5
@@ -20,8 +20,9 @@ Output (default: the repository's `data/`, next to this folder):
|
||||
<STATION>/camera/<STATION>_camera_<YYYYMMDD>.csv camera movements, per day
|
||||
<STATION>/images/<YYYYMMDD>T<HHMM>Z_<id>.<ext> OPTICAL_SHIFT-IRON images (--images)
|
||||
<STATION>/images_diff/… differential images (--diff)
|
||||
Resumable: with `--from`, a file already written is not read again, whatever the dates asked; without `--from`, the
|
||||
days of the range are replaced (images excepted). Days in UTC, `--to` excluded; the current day is pumped as it is,
|
||||
Resumable: with `--from`, a file already written is not read again, whatever the dates asked, except from the last
|
||||
day pumped of the station (pumped during the day, it was incomplete); without `--from`, the days of the range are
|
||||
replaced (images excepted). Days in UTC, `--to` excluded; the current day is pumped as it is,
|
||||
its metrics up to the last completed hour. To redo a day, delete its files.
|
||||
|
||||
No dependency on the app (~/monitoring-dashboard): its reads are copied here, in SQL, as they are (branch `explo`,
|
||||
@@ -86,13 +87,27 @@ def acquisition_unit(conn, system_id):
|
||||
return pick[0].id if pick else None
|
||||
|
||||
|
||||
def pumped_days(folder):
|
||||
"""The days of displacements already pumped."""
|
||||
return {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date()
|
||||
for p in (folder / "displacement").glob(f"{folder.name}_advanced_*.csv")}
|
||||
|
||||
|
||||
def given_ranges(folder, start, end):
|
||||
"""With --from, without --replace: (start, end, replace) to pump. The missing days of the range, and from the last
|
||||
day pumped (pumped during the day, it was incomplete) to the end of the range pumped again, as update_ranges."""
|
||||
last = max(pumped_days(folder), default=None)
|
||||
if last is None or not start <= last < end:
|
||||
return [(start, end, False)]
|
||||
return ([(start, last, False)] if start < last else []) + [(last, end, True)]
|
||||
|
||||
|
||||
def update_ranges(folder, end, tomorrow):
|
||||
"""Without --from: (start, end, replace) to pump, from the days of displacements already pumped. The missing days
|
||||
between the first and the last (ranges pumped separately, failed day) are filled without touching the others; the
|
||||
last day is pumped again (pumped during the day, it was incomplete) up to now; a station with no pumped
|
||||
displacements gets the last 30 days."""
|
||||
days = {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date()
|
||||
for p in (folder / "displacement").glob(f"{folder.name}_advanced_*.csv")}
|
||||
days = pumped_days(folder)
|
||||
if not days:
|
||||
return [(tomorrow - timedelta(days=NEW_STATION_DAYS + 1), end, True)]
|
||||
# a failed metrics day has no file at all (one query for all of them): it is a gap too; a station without any
|
||||
@@ -443,7 +458,8 @@ def main():
|
||||
if system_id is None:
|
||||
print(f"{station}: unknown system")
|
||||
continue
|
||||
ranges = [(start, end, a.replace)] if start else update_ranges(a.out / station, end, tomorrow)
|
||||
ranges = ([(start, end, True)] if start and a.replace else given_ranges(a.out / station, start, end)
|
||||
if start else update_ranges(a.out / station, end, tomorrow))
|
||||
unit = acquisition_unit(conn, system_id)
|
||||
conn.rollback()
|
||||
seen = {}
|
||||
@@ -491,6 +507,15 @@ def check():
|
||||
t = write_targets(Path(d), props, {"T1": 12, "REF": 13})
|
||||
assert t.values.tolist() == [["OLD", "target", 1, 1.0], ["REF", "global_ref", 50000, 18.0],
|
||||
["T1", "target", 20000, 2.0]], t
|
||||
with tempfile.TemporaryDirectory() as d: # given_ranges: the last day pumped and after, pumped again
|
||||
f, day = Path(d) / "S", lambda n: datetime(2026, 11, n).date()
|
||||
assert given_ranges(f, day(1), day(19)) == [(day(1), day(19), False)] # nothing pumped
|
||||
(f / "displacement").mkdir(parents=True)
|
||||
for n in (2, 18):
|
||||
(f / "displacement" / f"S_advanced_202611{n:02d}.csv").touch()
|
||||
assert given_ranges(f, day(1), day(19)) == [(day(1), day(18), False), (day(18), day(19), True)]
|
||||
assert given_ranges(f, day(18), day(19)) == [(day(18), day(19), True)]
|
||||
assert given_ranges(f, day(1), day(10)) == [(day(1), day(10), False)] # the last day after the range
|
||||
print("ok")
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user