The methods that bring data in, for the callers: extraction/pumping.py (tracked stations, pump.sh, its state), weather/meteoswiss.py (from aimsight-app, with hourly())
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
088484c376
commit
b1790fc422
6 files changed
+302
-5
No files matched your search
@@ -8,7 +8,7 @@
|
||||
|
||||
Formats: `~/aimsight-labeling/knowledge/displacement-columns.md`.
|
||||
|
||||
**Tracked stations**: `stations.txt`, one serial number per line, optionally followed by the position of the station (`lat,lon` or a Swiss place, read by the app for the weather; the app edits this list: Data page, "Stations"). Without station nor dates, the script updates them all up to now: each station first fills its missing days between the first and the last pumped day (without displacements, or without any metric), then restarts from its last day, pumped again (it was perhaps incomplete); a station without a folder in `data/` gets the last 30 days.
|
||||
**Tracked stations**: `stations.txt`, one serial number per line, optionally followed by the position of the station (`lat,lon` or a Swiss place, read by the app for the weather; the app edits this list: page Stations, through `pumping.py`). Without station nor dates, the script updates them all up to now: each station first fills its missing days between the first and the last pumped day (without displacements, or without any metric), then restarts from its last day, pumped again (it was perhaps incomplete); a station without a folder in `data/` gets the last 30 days.
|
||||
|
||||
Resuming: with `--from`, a file already in `data/<SERIAL>/` is not asked for again; running again on a wider range reads only the days and images that are missing. The current day is pumped as it is, its metrics up to the last completed hour; the next run without `--from` replaces it.
|
||||
|
||||
@@ -37,6 +37,10 @@ Read-only: dedicated engine, `default_transaction_read_only`, 60 s timeout per q
|
||||
|
||||
No dependency on the `monitoring-dashboard` app: its reads are copied into `pump.py`, in SQL (its docstring says which ones, and from which commit). If the app changes its reads, the script does not follow: compare again files pumped by both.
|
||||
|
||||
## For a caller
|
||||
|
||||
`pumping.py` (standard library): `tracked()` and `save(rows)` (the list), `start(stations, start, end, images)` (`pump.sh` in the background, `--delay 300`), `running()` (the lock held) and `log()` (the end of `out/pump.log`); a refused request raises `ValueError`. Its check: `python3 extraction/pumping.py`. The page Stations of aimsight-app calls it.
|
||||
|
||||
## With Claude
|
||||
|
||||
Claude may run `pump.sh` itself (user's agreement, 2026-11-12): read-only, small ranges; a long range or a history, outside busy hours. It never reads the `.env`.
|
||||
|
||||
+3
-2
@@ -42,8 +42,9 @@ import pandas as pd
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import URL
|
||||
|
||||
import pumping # the tracked stations (stations.txt), next to this script
|
||||
|
||||
UTC = timezone.utc
|
||||
STATIONS = Path(__file__).resolve().parent / "stations.txt" # the tracked stations, one per line: "SERIAL [position]"
|
||||
NEW_STATION_DAYS = 30 # without --from, a station with no pumped displacements: the last 30 days
|
||||
DB_NAMES = ["AIMSIGHT_DATABASE_USER", "AIMSIGHT_DATABASE_PSWD", "AIMSIGHT_DATABASE_TURL", "AIMSIGHT_DATABASE_NAME"]
|
||||
TYPES = {"images": "OPTICAL_SHIFT-IRON", "images_diff": "OPTICAL_SHIFT-DIFF"} # folder -> measurement type
|
||||
@@ -361,7 +362,7 @@ def main():
|
||||
tomorrow = datetime.now(UTC).date() + timedelta(days=1)
|
||||
start = datetime.strptime(a.start, "%Y-%m-%d").date() if a.start else None
|
||||
end = min(datetime.strptime(a.end, "%Y-%m-%d").date(), tomorrow) if a.end else tomorrow # no day to come
|
||||
stations = a.stations or list(dict.fromkeys(l.split()[0] for l in STATIONS.read_text().splitlines() if l.strip()))
|
||||
stations = a.stations or list(pumping.tracked())
|
||||
|
||||
missing = [n for n in DB_NAMES if not os.environ.get(n)]
|
||||
if missing:
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
"""The pumping, for its callers (aimsight-app): the tracked stations, launching pump.sh, its state.
|
||||
|
||||
- `tracked()`, `save(rows)`: extraction/stations.txt, one station per line, "SERIAL [position]" (position: "lat,lon" or a
|
||||
Swiss place, for the weather, weather/meteoswiss.py; empty: an indoor site, no weather); pump.py reads it too;
|
||||
- `start(stations, start, end, images)`: pump.sh in the background (its lock: one pumping at a time);
|
||||
- `running()`, `log()`: a pumping running (any: the scheduler's, by hand, a caller's), the end of out/pump.log.
|
||||
|
||||
A refused request raises ValueError, with its reason. Standard library only.
|
||||
"""
|
||||
import fcntl
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERIAL = re.compile(r"[A-Za-z0-9_-]+$") # it becomes an argument of pump.sh and a folder of data/
|
||||
DAY = re.compile(r"\d{4}-\d{2}-\d{2}$")
|
||||
|
||||
|
||||
def tracked():
|
||||
"""{serial: position ("" without)}, in the order of the file, without duplicates."""
|
||||
f = ROOT / "extraction" / "stations.txt"
|
||||
out = {}
|
||||
for line in f.read_text().splitlines() if f.is_file() else []:
|
||||
serial, _, position = line.strip().partition(" ")
|
||||
if serial:
|
||||
out.setdefault(serial, position.strip())
|
||||
return out
|
||||
|
||||
|
||||
def save(rows):
|
||||
"""rows: [{serial, position}] -> the new stations.txt, and its content as tracked()."""
|
||||
out = {}
|
||||
for r in rows:
|
||||
serial, position = str(r.get("serial", "")).strip(), " ".join(str(r.get("position", "")).split())
|
||||
if not SERIAL.match(serial):
|
||||
raise ValueError(f"not a serial number: {serial!r}")
|
||||
out.setdefault(serial, position)
|
||||
(ROOT / "extraction" / "stations.txt").write_text("".join(f"{s} {p}".rstrip() + "\n" for s, p in out.items()))
|
||||
return out
|
||||
|
||||
|
||||
def running():
|
||||
"""A pumping running: the lock of pump.sh is held."""
|
||||
try:
|
||||
with open(ROOT / "out" / "pump.lock") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except BlockingIOError:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def log(lines=40):
|
||||
"""The last lines of out/pump.log."""
|
||||
try:
|
||||
with open(ROOT / "out" / "pump.log", "rb") as f:
|
||||
f.seek(max(0, f.seek(0, os.SEEK_END) - 200 * lines))
|
||||
return f.read().decode(errors="replace").splitlines()[-lines:]
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
|
||||
|
||||
def start(stations, start="", end="", images=False):
|
||||
"""pump.sh in the background, with --from start --to end (YYYY-MM-DD, end excluded), or without (the missing days of
|
||||
each station up to now); --delay 300. -> the process."""
|
||||
if not stations or not all(isinstance(s, str) and SERIAL.match(s) for s in stations):
|
||||
raise ValueError("no station, or not a serial number")
|
||||
if bool(start) != bool(end) or (start and not (DAY.match(start) and DAY.match(end) and start < end)):
|
||||
raise ValueError("a range: From and To, YYYY-MM-DD, From before To")
|
||||
if running():
|
||||
raise ValueError("a pumping is already running")
|
||||
args = ["--from", start, "--to", end] * bool(start) + ["--images"] * bool(images) + ["--delay", "300", *stations]
|
||||
return subprocess.Popen([str(ROOT / "extraction" / "pump.sh"), *args], cwd=ROOT,
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
if __name__ == "__main__": # a check, in a temporary folder: python3 extraction/pumping.py
|
||||
import tempfile
|
||||
ROOT = Path(tempfile.mkdtemp())
|
||||
(ROOT / "extraction").mkdir()
|
||||
(ROOT / "extraction" / "stations.txt").write_text("NORDEND\nNIEDERHORN 46.69,7.78\n\nNORDEND Zermatt\n")
|
||||
assert tracked() == {"NORDEND": "", "NIEDERHORN": "46.69,7.78"}, tracked()
|
||||
assert save([{"serial": "A", "position": " Interlaken BE "}, {"serial": "B"}, {"serial": "A"}]) == {"A": "Interlaken BE", "B": ""}
|
||||
assert (ROOT / "extraction" / "stations.txt").read_text() == "A Interlaken BE\nB\n"
|
||||
assert not running() and log() == []
|
||||
for bad in ((save, [{"serial": "../x"}]), (start, ["A;rm"]), (start, ["A"], "2026-11-10"), (start, ["A"], "2026-11-10", "2026-11-09")):
|
||||
try:
|
||||
bad[0](*bad[1:])
|
||||
raise AssertionError(f"not refused: {bad}")
|
||||
except ValueError:
|
||||
pass
|
||||
print("ok")
|
||||
Reference in new issue
Block a user