The methods that bring data in, for the callers: extraction/pumping.py (tracked stations, pump.sh, its state), weather/meteoswiss.py (from aimsight-app, with hourly())

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-17 09:12:12 +01:00
1 parent 088484c376
commit b1790fc422
6 files changed
+302 -5

No files matched your search

+5 -1
View File
@@ -8,7 +8,7 @@
Formats: `~/aimsight-labeling/knowledge/displacement-columns.md`.
**Tracked stations**: `stations.txt`, one serial number per line, optionally followed by the position of the station (`lat,lon` or a Swiss place, read by the app for the weather; the app edits this list: Data page, "Stations"). Without station nor dates, the script updates them all up to now: each station first fills its missing days between the first and the last pumped day (without displacements, or without any metric), then restarts from its last day, pumped again (it was perhaps incomplete); a station without a folder in `data/` gets the last 30 days.
**Tracked stations**: `stations.txt`, one serial number per line, optionally followed by the position of the station (`lat,lon` or a Swiss place, read by the app for the weather; the app edits this list: page Stations, through `pumping.py`). Without station nor dates, the script updates them all up to now: each station first fills its missing days between the first and the last pumped day (without displacements, or without any metric), then restarts from its last day, pumped again (it was perhaps incomplete); a station without a folder in `data/` gets the last 30 days.
Resuming: with `--from`, a file already in `data/<SERIAL>/` is not asked for again; running again on a wider range reads only the days and images that are missing. The current day is pumped as it is, its metrics up to the last completed hour; the next run without `--from` replaces it.
@@ -37,6 +37,10 @@ Read-only: dedicated engine, `default_transaction_read_only`, 60 s timeout per q
No dependency on the `monitoring-dashboard` app: its reads are copied into `pump.py`, in SQL (its docstring says which ones, and from which commit). If the app changes its reads, the script does not follow: compare again files pumped by both.
## For a caller
`pumping.py` (standard library): `tracked()` and `save(rows)` (the list), `start(stations, start, end, images)` (`pump.sh` in the background, `--delay 300`), `running()` (the lock held) and `log()` (the end of `out/pump.log`); a refused request raises `ValueError`. Its check: `python3 extraction/pumping.py`. The page Stations of aimsight-app calls it.
## With Claude
Claude may run `pump.sh` itself (user's agreement, 2026-11-12): read-only, small ranges; a long range or a history, outside busy hours. It never reads the `.env`.
+3 -2
View File
@@ -42,8 +42,9 @@ import pandas as pd
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
import pumping # the tracked stations (stations.txt), next to this script
UTC = timezone.utc
STATIONS = Path(__file__).resolve().parent / "stations.txt" # the tracked stations, one per line: "SERIAL [position]"
NEW_STATION_DAYS = 30 # without --from, a station with no pumped displacements: the last 30 days
DB_NAMES = ["AIMSIGHT_DATABASE_USER", "AIMSIGHT_DATABASE_PSWD", "AIMSIGHT_DATABASE_TURL", "AIMSIGHT_DATABASE_NAME"]
TYPES = {"images": "OPTICAL_SHIFT-IRON", "images_diff": "OPTICAL_SHIFT-DIFF"} # folder -> measurement type
@@ -361,7 +362,7 @@ def main():
tomorrow = datetime.now(UTC).date() + timedelta(days=1)
start = datetime.strptime(a.start, "%Y-%m-%d").date() if a.start else None
end = min(datetime.strptime(a.end, "%Y-%m-%d").date(), tomorrow) if a.end else tomorrow # no day to come
stations = a.stations or list(dict.fromkeys(l.split()[0] for l in STATIONS.read_text().splitlines() if l.strip()))
stations = a.stations or list(pumping.tracked())
missing = [n for n in DB_NAMES if not os.environ.get(n)]
if missing:
+95
View File
@@ -0,0 +1,95 @@
"""The pumping, for its callers (aimsight-app): the tracked stations, launching pump.sh, its state.
- `tracked()`, `save(rows)`: extraction/stations.txt, one station per line, "SERIAL [position]" (position: "lat,lon" or a
Swiss place, for the weather, weather/meteoswiss.py; empty: an indoor site, no weather); pump.py reads it too;
- `start(stations, start, end, images)`: pump.sh in the background (its lock: one pumping at a time);
- `running()`, `log()`: a pumping running (any: the scheduler's, by hand, a caller's), the end of out/pump.log.
A refused request raises ValueError, with its reason. Standard library only.
"""
import fcntl
import os
import re
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SERIAL = re.compile(r"[A-Za-z0-9_-]+$") # it becomes an argument of pump.sh and a folder of data/
DAY = re.compile(r"\d{4}-\d{2}-\d{2}$")
def tracked():
"""{serial: position ("" without)}, in the order of the file, without duplicates."""
f = ROOT / "extraction" / "stations.txt"
out = {}
for line in f.read_text().splitlines() if f.is_file() else []:
serial, _, position = line.strip().partition(" ")
if serial:
out.setdefault(serial, position.strip())
return out
def save(rows):
"""rows: [{serial, position}] -> the new stations.txt, and its content as tracked()."""
out = {}
for r in rows:
serial, position = str(r.get("serial", "")).strip(), " ".join(str(r.get("position", "")).split())
if not SERIAL.match(serial):
raise ValueError(f"not a serial number: {serial!r}")
out.setdefault(serial, position)
(ROOT / "extraction" / "stations.txt").write_text("".join(f"{s} {p}".rstrip() + "\n" for s, p in out.items()))
return out
def running():
"""A pumping running: the lock of pump.sh is held."""
try:
with open(ROOT / "out" / "pump.lock") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except FileNotFoundError:
return False
except BlockingIOError:
return True
return False
def log(lines=40):
"""The last lines of out/pump.log."""
try:
with open(ROOT / "out" / "pump.log", "rb") as f:
f.seek(max(0, f.seek(0, os.SEEK_END) - 200 * lines))
return f.read().decode(errors="replace").splitlines()[-lines:]
except FileNotFoundError:
return []
def start(stations, start="", end="", images=False):
"""pump.sh in the background, with --from start --to end (YYYY-MM-DD, end excluded), or without (the missing days of
each station up to now); --delay 300. -> the process."""
if not stations or not all(isinstance(s, str) and SERIAL.match(s) for s in stations):
raise ValueError("no station, or not a serial number")
if bool(start) != bool(end) or (start and not (DAY.match(start) and DAY.match(end) and start < end)):
raise ValueError("a range: From and To, YYYY-MM-DD, From before To")
if running():
raise ValueError("a pumping is already running")
args = ["--from", start, "--to", end] * bool(start) + ["--images"] * bool(images) + ["--delay", "300", *stations]
return subprocess.Popen([str(ROOT / "extraction" / "pump.sh"), *args], cwd=ROOT,
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
if __name__ == "__main__": # a check, in a temporary folder: python3 extraction/pumping.py
import tempfile
ROOT = Path(tempfile.mkdtemp())
(ROOT / "extraction").mkdir()
(ROOT / "extraction" / "stations.txt").write_text("NORDEND\nNIEDERHORN 46.69,7.78\n\nNORDEND Zermatt\n")
assert tracked() == {"NORDEND": "", "NIEDERHORN": "46.69,7.78"}, tracked()
assert save([{"serial": "A", "position": " Interlaken BE "}, {"serial": "B"}, {"serial": "A"}]) == {"A": "Interlaken BE", "B": ""}
assert (ROOT / "extraction" / "stations.txt").read_text() == "A Interlaken BE\nB\n"
assert not running() and log() == []
for bad in ((save, [{"serial": "../x"}]), (start, ["A;rm"]), (start, ["A"], "2026-11-10"), (start, ["A"], "2026-11-10", "2026-11-09")):
try:
bad[0](*bad[1:])
raise AssertionError(f"not refused: {bad}")
except ValueError:
pass
print("ok")