326 lines
19 KiB
Python
326 lines
19 KiB
Python
"""Pumps the differential images (OPTICAL_SHIFT-DIFF) of stations from the AIMsight platform, full size, with what is
|
|||
|
|
needed to measure them again: every pattern of the unit (archived ones included) and the unit's properties. READ-ONLY.
|
||
|
|
Never touches data/ nor pump.py's pumping; design: docs/superpowers/specs/2026-11-19-pump-diff-design.md.
|
||
|
|
|
||
|
|
Run through extraction/pump_diff.sh, under WSL, from anywhere (see README.md):
|
||
|
|
|
||
|
|
extraction/pump_diff.sh --from 2026-09-01 --to 2026-09-08 JUNGFRAU a range (its history starts here)
|
||
|
|
extraction/pump_diff.sh JUNGFRAU from its last indexed day up to now
|
||
|
|
extraction/pump_diff.sh the same for every station of data_diff/
|
||
|
|
|
||
|
|
Output (default: the repository's data_diff/, next to this folder), the tree of data/ so that the other repositories
|
||
|
|
read it the same way (docs/superpowers/specs/2026-11-19-configurations-design.md):
|
||
|
|
<STATION>/images_diff/<YYYYMMDD>T<HHMM>Z_<measurement_id>.png the S3 object as is, named as in data/
|
||
|
|
<STATION>/images_diff/index/<STATION>_diff_<YYYYMMDD>.csv every measurement of the day in the database:
|
||
|
|
measurement_id, time_utc (exact), measurement_group_id, file (empty without image), status (ok; no_uuid: never
|
||
|
|
reached S3; s3_missing: S3 returned nothing), properties (the measurement's JSON)
|
||
|
|
<STATION>/images_diff/sensors.json every sensor of the unit: id, name, type, time_utc, properties [{name, value,
|
||
|
|
time_utc, status}] (every property row as stored), rewritten at each run
|
||
|
|
<STATION>/images_diff/unit.json the acquisition unit: id, serial_number, product_number, properties
|
||
|
|
(under images_diff/, the tools of aimsight-tools do not see them: data.find_files skips that folder)
|
||
|
|
<STATION>/metrics/, images/ (IRON, one per --every minutes), targets.csv (the DIFF patterns), weather/: as pump.py
|
||
|
|
and weather.py write them into data/, with their own functions
|
||
|
|
|
||
|
|
Resumable: a day whose index exists and is before the last indexed day is not read again; the last indexed day and
|
||
|
|
the later ones are (pumped while running, a day is incomplete); an image on disk is never downloaded again. An index
|
||
|
|
is written after its day's metrics, IRON and DIFF images, every DIFF file through a temporary one: an indexed day is
|
||
|
|
complete; a day whose metrics failed gets no index. Without --from, a station with nothing indexed is skipped (its
|
||
|
|
history starts only on an explicit --from). Stops before a day when the disk has less than --min-free-gb left.
|
||
|
|
|
||
|
|
Same reads as pump.py (measurement, acquisition_unit; the copied queries of the app, branch `explo`), plus the
|
||
|
|
`property` rows of the sensors and of the unit (AcquisitionUnit.list_sensors, Sensor.properties of
|
||
|
|
app/aimsight_database.py). Variables read (names only, none is printed): AIMSIGHT_DATABASE_USER / _PSWD / _TURL /
|
||
|
|
_NAME / _PORT, AIMSIGHT_S3_URL / _ACCESS_KEY / _SECRET_KEY / _BUCKET.
|
||
|
|
"""
|
||
|
|
import argparse
|
||
|
|
import csv
|
||
|
|
import io
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import shutil
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
from collections import Counter
|
||
|
|
from datetime import date, datetime, timedelta, timezone
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
UTC = timezone.utc
|
||
|
|
MTYPE = "OPTICAL_SHIFT-DIFF"
|
||
|
|
INDEX = ["measurement_id", "time_utc", "measurement_group_id", "file", "status", "properties"]
|
||
|
|
|
||
|
|
|
||
|
|
def write_atomic(path, data):
|
||
|
|
"""A file is either complete or absent: a run stopped in the middle leaves nothing that looks done."""
|
||
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
tmp = path.with_name(path.name + ".tmp")
|
||
|
|
tmp.write_bytes(data)
|
||
|
|
os.replace(tmp, path)
|
||
|
|
|
||
|
|
|
||
|
|
def indexed_days(folder):
|
||
|
|
return {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date()
|
||
|
|
for p in (folder / "images_diff" / "index").glob(f"{folder.name}_diff_*.csv")}
|
||
|
|
|
||
|
|
|
||
|
|
def days_to_pump(indexed, start, end):
|
||
|
|
"""The days of [start, end) to read: the missing ones, and from the last indexed day on (perhaps incomplete).
|
||
|
|
Without start: from the last indexed day; None when nothing is indexed."""
|
||
|
|
last = max(indexed, default=None)
|
||
|
|
if start is None:
|
||
|
|
if last is None:
|
||
|
|
return None
|
||
|
|
start = last
|
||
|
|
days = (start + timedelta(days=k) for k in range((end - start).days))
|
||
|
|
return [d for d in days if last is None or d not in indexed or d >= last]
|
||
|
|
|
||
|
|
|
||
|
|
def pump_day(rows, client, bucket, ext, folder, station, day):
|
||
|
|
"""The images of one day's measurements (rows: id, time_utc, measurement_group_id, measurement_uuid, properties),
|
||
|
|
then its index. Returns the counts per status, plus the new images and their bytes."""
|
||
|
|
c, index = Counter(), []
|
||
|
|
for m in rows:
|
||
|
|
t = m.time_utc.astimezone(UTC)
|
||
|
|
name, status = f"{t:%Y%m%dT%H%M}Z_{m.id}.{ext}", "ok"
|
||
|
|
path = folder / "images_diff" / name
|
||
|
|
if m.measurement_uuid is None: # the image never reached S3 (unfinished multipart upload)
|
||
|
|
name, status = "", "no_uuid"
|
||
|
|
elif not path.exists():
|
||
|
|
try:
|
||
|
|
data = client.get_object(Bucket=bucket, Key=m.measurement_uuid)["Body"].read()
|
||
|
|
except Exception: # like the app's get_measurement_payload: S3 not answering, the image is missing
|
||
|
|
data = None
|
||
|
|
if data:
|
||
|
|
write_atomic(path, data)
|
||
|
|
c["new"], c["bytes"] = c["new"] + 1, c["bytes"] + len(data)
|
||
|
|
else:
|
||
|
|
name, status = "", "s3_missing"
|
||
|
|
c[status] += 1
|
||
|
|
index.append([m.id, t.isoformat(), "" if m.measurement_group_id is None else m.measurement_group_id, name,
|
||
|
|
status, "" if m.properties is None else json.dumps(m.properties, sort_keys=True)])
|
||
|
|
out = io.StringIO()
|
||
|
|
w = csv.writer(out, lineterminator="\n")
|
||
|
|
w.writerow(INDEX)
|
||
|
|
w.writerows(index)
|
||
|
|
write_atomic(folder / "images_diff" / "index" / f"{station}_diff_{day:%Y%m%d}.csv", out.getvalue().encode())
|
||
|
|
return c
|
||
|
|
|
||
|
|
|
||
|
|
def prop(name, value, t, status):
|
||
|
|
return {"name": name, "value": value, "time_utc": t.isoformat(), "status": status}
|
||
|
|
|
||
|
|
|
||
|
|
def sensors_json(conn, unit):
|
||
|
|
"""Every sensor of the unit, any type, archived included (`archive` is one of their properties), with every
|
||
|
|
property row: a pattern redrawn on the platform is a new sensor, the old one stays."""
|
||
|
|
from sqlalchemy import text
|
||
|
|
rows = conn.execute(text("""
|
||
|
|
SELECT s.id, s.name, s.type, s.time_utc, dt.name AS prop, p.value, p.time_utc AS ptime, p.status
|
||
|
|
FROM sensor s LEFT JOIN property p ON p.sensor_id = s.id LEFT JOIN datatype dt ON dt.id = p.datatype_id
|
||
|
|
WHERE s.acquisition_unit_id = :au ORDER BY s.id, dt.name, p.time_utc"""), {"au": unit}).all()
|
||
|
|
sensors = {}
|
||
|
|
for r in rows:
|
||
|
|
s = sensors.setdefault(r.id, {"id": r.id, "name": r.name, "type": r.type, "time_utc": r.time_utc.isoformat(),
|
||
|
|
"properties": []})
|
||
|
|
if r.prop is not None:
|
||
|
|
s["properties"].append(prop(r.prop, r.value, r.ptime, r.status))
|
||
|
|
return list(sensors.values())
|
||
|
|
|
||
|
|
|
||
|
|
def unit_json(conn, unit):
|
||
|
|
"""The acquisition unit and its own property rows (focal_distance, pixel_size, algorithm...), not its sensors'."""
|
||
|
|
from sqlalchemy import text
|
||
|
|
u = conn.execute(text("SELECT id, serial_number, product_number FROM acquisition_unit WHERE id = :au"),
|
||
|
|
{"au": unit}).one()
|
||
|
|
rows = conn.execute(text("""
|
||
|
|
SELECT dt.name, p.value, p.time_utc, p.status FROM property p JOIN datatype dt ON dt.id = p.datatype_id
|
||
|
|
WHERE p.acquisition_unit_id = :au AND p.sensor_id IS NULL ORDER BY dt.name, p.time_utc"""), {"au": unit}).all()
|
||
|
|
return {"id": u.id, "serial_number": u.serial_number, "product_number": u.product_number,
|
||
|
|
"properties": [prop(*r) for r in rows]}
|
||
|
|
|
||
|
|
|
||
|
|
def diff_patterns(sensors):
|
||
|
|
"""{sensor_name: sensor_id} of the patterns the DIFF images are measured with (sensors_json rows): type
|
||
|
|
OPTICAL_SHIFT-DIFF, a pattern, not archived; for targets.csv."""
|
||
|
|
out = {}
|
||
|
|
for s in sensors:
|
||
|
|
p = {r["name"]: r["value"] for r in s["properties"]}
|
||
|
|
if s["type"] == MTYPE and p.get("pattern") and not p.get("archive"):
|
||
|
|
out[p.get("sensor_name") or s["name"]] = s["id"]
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def to_json(obj):
|
||
|
|
return json.dumps(obj, indent=1, ensure_ascii=False, default=str).encode()
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
|
|
p.add_argument("stations", nargs="*", help="serial number of the system, as in the URL of the System page; "
|
||
|
|
"none: the stations already in --out")
|
||
|
|
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD; without: the last indexed day of each "
|
||
|
|
"station (a station with nothing indexed is skipped)")
|
||
|
|
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD; without: up to now")
|
||
|
|
p.add_argument("--delay", type=int, default=60, help="maximum duration of a query, in s (default 60: database "
|
||
|
|
"shared with production)")
|
||
|
|
p.add_argument("--every", type=int, default=60, help="IRON images: one per slot of N minutes, 0 = all (default 60, "
|
||
|
|
"as pump.sh)")
|
||
|
|
p.add_argument("--min-free-gb", type=float, default=50, help="stop before a day when less is free (default 50)")
|
||
|
|
p.add_argument("--out", type=Path, default=Path(__file__).resolve().parent.parent / "data_diff")
|
||
|
|
a = p.parse_args()
|
||
|
|
|
||
|
|
import boto3
|
||
|
|
from sqlalchemy import create_engine, text
|
||
|
|
from sqlalchemy.engine import URL
|
||
|
|
|
||
|
|
from pump import DB_NAMES, acquisition_unit, pump_images, pump_metrics, root_cause, sensors, write_targets
|
||
|
|
|
||
|
|
tomorrow = datetime.now(UTC).date() + timedelta(days=1)
|
||
|
|
start = datetime.strptime(a.start, "%Y-%m-%d").date() if a.start else None
|
||
|
|
end = min(datetime.strptime(a.end, "%Y-%m-%d").date(), tomorrow) if a.end else tomorrow
|
||
|
|
stations = a.stations or sorted(f.name for f in a.out.glob("*") if f.is_dir())
|
||
|
|
if not stations:
|
||
|
|
sys.exit(f"No station: name one, with --from (nothing in {a.out} yet)")
|
||
|
|
missing = [n for n in DB_NAMES if not os.environ.get(n)]
|
||
|
|
if missing:
|
||
|
|
sys.exit(f"Missing variables (names only): {', '.join(missing)}")
|
||
|
|
|
||
|
|
# the engine and the S3 client of pump.py's main
|
||
|
|
url = URL.create("postgresql+psycopg2", username=os.environ["AIMSIGHT_DATABASE_USER"],
|
||
|
|
password=os.environ["AIMSIGHT_DATABASE_PSWD"], host=os.environ["AIMSIGHT_DATABASE_TURL"],
|
||
|
|
port=int(os.environ.get("AIMSIGHT_DATABASE_PORT", "5432")),
|
||
|
|
database=os.environ["AIMSIGHT_DATABASE_NAME"])
|
||
|
|
engine = create_engine(url, connect_args={
|
||
|
|
"options": f"-c timezone=utc -c default_transaction_read_only=on -c statement_timeout={a.delay}s"})
|
||
|
|
client = boto3.client("s3", endpoint_url=os.environ.get("AIMSIGHT_S3_URL", "http://s3:3900"),
|
||
|
|
aws_access_key_id=os.environ.get("AIMSIGHT_S3_ACCESS_KEY", ""),
|
||
|
|
aws_secret_access_key=os.environ.get("AIMSIGHT_S3_SECRET_KEY", ""),
|
||
|
|
region_name="garage", use_ssl=False)
|
||
|
|
bucket = os.environ.get("AIMSIGHT_S3_BUCKET", "measurements-assets")
|
||
|
|
try:
|
||
|
|
with engine.connect() as conn:
|
||
|
|
mt = conn.execute(text("SELECT id, extension FROM measurement_type WHERE name = :n"), {"n": MTYPE}).first()
|
||
|
|
conn.rollback()
|
||
|
|
if mt is None:
|
||
|
|
sys.exit(f"Unknown measurement type {MTYPE}")
|
||
|
|
for station in stations:
|
||
|
|
system_id = conn.execute(text("SELECT id FROM system WHERE serial_number = :s"),
|
||
|
|
{"s": station}).scalar()
|
||
|
|
unit = system_id and acquisition_unit(conn, system_id)
|
||
|
|
conn.rollback()
|
||
|
|
if not unit:
|
||
|
|
print(f"{station}: " + ("unknown system" if system_id is None else "no recognised acquisition unit"))
|
||
|
|
continue
|
||
|
|
folder = a.out / station
|
||
|
|
days = days_to_pump(indexed_days(folder), start, end)
|
||
|
|
if days is None:
|
||
|
|
print(f"{station}: nothing indexed yet, its history starts with --from")
|
||
|
|
continue
|
||
|
|
try: # what the database holds, so that an empty day is explained (LENZSPITZE: none after 2026-10-13)
|
||
|
|
first, last = (conn.execute(text(
|
||
|
|
"SELECT time_utc FROM measurement WHERE acquisition_unit_id = :au AND measurement_type_id = :mt "
|
||
|
|
f"ORDER BY time_utc {o} LIMIT 1"), {"au": unit, "mt": mt.id}).scalar() for o in ("ASC", "DESC"))
|
||
|
|
print(f"{station}: differential images in the database from {first:%Y-%m-%d} to {last:%Y-%m-%d}"
|
||
|
|
if first else f"{station}: no differential image in the database")
|
||
|
|
sens = sensors_json(conn, unit)
|
||
|
|
write_atomic(folder / "images_diff" / "sensors.json", to_json(sens))
|
||
|
|
write_atomic(folder / "images_diff" / "unit.json", to_json(unit_json(conn, unit)))
|
||
|
|
t = write_targets(folder, sensors(conn, unit), diff_patterns(sens))
|
||
|
|
print(f"{station}: targets.csv, {len(t)} sensors")
|
||
|
|
except Exception as e: # the next run writes them
|
||
|
|
print(f"{station} images range, sensors.json, unit.json, targets.csv: failed ({root_cause(e)}), "
|
||
|
|
"run again")
|
||
|
|
finally:
|
||
|
|
conn.rollback()
|
||
|
|
print(f"{station}: {len(days)} days, from {days[0] if days else '-'} to {days[-1] if days else '-'}")
|
||
|
|
for day in days:
|
||
|
|
free = shutil.disk_usage(a.out).free / 1e9
|
||
|
|
if free < a.min_free_gb:
|
||
|
|
sys.exit(f"Stopped before {station} {day}: {free:.0f} GB free, under --min-free-gb "
|
||
|
|
f"{a.min_free_gb:g}")
|
||
|
|
nxt = day + timedelta(days=1)
|
||
|
|
try: # as pump.py into data/; a day whose metrics or IRON images failed stays open (no index)
|
||
|
|
if not pump_metrics(conn, system_id, station, day, nxt, a.out, True):
|
||
|
|
print(f"{station} {day}: left open (metrics failed), run again")
|
||
|
|
continue
|
||
|
|
pump_images(conn, (client, bucket), unit, station, day, nxt, "images", a.every, a.out)
|
||
|
|
except Exception as e:
|
||
|
|
print(f"{station} {day}: left open ({root_cause(e)}), run again")
|
||
|
|
continue
|
||
|
|
finally:
|
||
|
|
conn.rollback()
|
||
|
|
a0 = datetime.combine(day, datetime.min.time(), UTC)
|
||
|
|
try:
|
||
|
|
rows = conn.execute(text(
|
||
|
|
"SELECT id, time_utc, measurement_group_id, measurement_uuid, properties FROM measurement "
|
||
|
|
"WHERE acquisition_unit_id = :au AND measurement_type_id = :mt AND time_utc >= :a "
|
||
|
|
"AND time_utc < :b ORDER BY time_utc"),
|
||
|
|
{"au": unit, "mt": mt.id, "a": a0, "b": a0 + timedelta(days=1)}).all()
|
||
|
|
except Exception as e: # no index for the day: the next run reads it again
|
||
|
|
print(f"{station} {day}: failed ({root_cause(e)}), run again")
|
||
|
|
continue
|
||
|
|
finally:
|
||
|
|
conn.rollback()
|
||
|
|
c = pump_day(rows, client, bucket, mt.extension, folder, station, day)
|
||
|
|
print(f"{station} {day}: {c['new']} new images, {c['bytes'] / 1e6:.0f} MB ({len(rows)} in "
|
||
|
|
f"database: {c['ok']} on disk, {c['no_uuid']} never reached S3, {c['s3_missing']} not "
|
||
|
|
"returned by S3)")
|
||
|
|
time.sleep(0.5)
|
||
|
|
try: # the MeteoSwiss weather of the days now in --out, as pump.sh does for data/
|
||
|
|
import weather
|
||
|
|
weather.main(stations, a.out)
|
||
|
|
except Exception as e: # the next run fetches it
|
||
|
|
print(f"weather: failed ({type(e).__name__}: {e}), run again")
|
||
|
|
finally:
|
||
|
|
engine.dispose()
|
||
|
|
|
||
|
|
|
||
|
|
def check():
|
||
|
|
"""Without database or S3: `python3 extraction/pump_diff.py --check`."""
|
||
|
|
import tempfile
|
||
|
|
from types import SimpleNamespace
|
||
|
|
|
||
|
|
d = lambda n: date(2026, 11, n) # noqa: E731
|
||
|
|
assert days_to_pump(set(), d(1), d(4)) == [d(1), d(2), d(3)]
|
||
|
|
assert days_to_pump({d(1), d(2), d(5)}, d(1), d(7)) == [d(3), d(4), d(5), d(6)] # the last indexed day: again
|
||
|
|
assert days_to_pump({d(2)}, None, d(5)) == [d(2), d(3), d(4)]
|
||
|
|
assert days_to_pump(set(), None, d(5)) is None # no --from, nothing indexed: skipped
|
||
|
|
|
||
|
|
class S3:
|
||
|
|
def get_object(self, Bucket, Key):
|
||
|
|
if Key == "refused":
|
||
|
|
raise OSError("NoSuchKey")
|
||
|
|
return {"Body": io.BytesIO(b"png")}
|
||
|
|
|
||
|
|
t = lambda h, m: datetime(2026, 11, 3, h, m, 7, 250000, tzinfo=UTC) # noqa: E731
|
||
|
|
rows = [SimpleNamespace(id=1, time_utc=t(10, 0), measurement_group_id=5, measurement_uuid="a", properties={"e": 1}),
|
||
|
|
SimpleNamespace(id=2, time_utc=t(10, 0), measurement_group_id=5, measurement_uuid="b", properties=None),
|
||
|
|
SimpleNamespace(id=3, time_utc=t(10, 15), measurement_group_id=None, measurement_uuid=None, properties={}),
|
||
|
|
SimpleNamespace(id=4, time_utc=t(10, 30), measurement_group_id=None, measurement_uuid="refused",
|
||
|
|
properties={})]
|
||
|
|
with tempfile.TemporaryDirectory() as tmp:
|
||
|
|
folder = Path(tmp) / "S"
|
||
|
|
(folder / "images_diff").mkdir(parents=True)
|
||
|
|
(folder / "images_diff" / "20261103T1000Z_2.png").write_bytes(b"old") # a burst: same minute, other id
|
||
|
|
c = pump_day(rows, S3(), "bucket", "png", folder, "S", d(3))
|
||
|
|
assert dict(c) == {"ok": 2, "new": 1, "bytes": 3, "no_uuid": 1, "s3_missing": 1}, c
|
||
|
|
assert (folder / "images_diff" / "20261103T1000Z_1.png").read_bytes() == b"png"
|
||
|
|
assert (folder / "images_diff" / "20261103T1000Z_2.png").read_bytes() == b"old" # never downloaded again
|
||
|
|
assert not list(folder.rglob("*.tmp"))
|
||
|
|
with open(folder / "images_diff" / "index" / "S_diff_20261103.csv", newline="") as fp:
|
||
|
|
index = list(csv.DictReader(fp))
|
||
|
|
assert [r["status"] for r in index] == ["ok", "ok", "no_uuid", "s3_missing"], index
|
||
|
|
assert [r["file"] for r in index] == ["20261103T1000Z_1.png", "20261103T1000Z_2.png", "", ""], index
|
||
|
|
assert index[0]["time_utc"] == "2026-11-03T10:00:07.250000+00:00" and index[0]["properties"] == '{"e": 1}'
|
||
|
|
assert index[1]["properties"] == "" and index[2]["measurement_group_id"] == "", index
|
||
|
|
assert indexed_days(folder) == {d(3)}
|
||
|
|
row = lambda name, typ, **p: {"name": name, "type": typ, "id": len(p), # noqa: E731
|
||
|
|
"properties": [{"name": k, "value": v} for k, v in p.items()]}
|
||
|
|
assert diff_patterns([row("u1", MTYPE, pattern="b64", sensor_name="T1"), row("u2", MTYPE, pattern="b64"),
|
||
|
|
row("u3", MTYPE, pattern="b64", sensor_name="OLD", archive="2026-01-01"),
|
||
|
|
row("u4", "camera_movement", sensor_name="CAM"), row("u5", MTYPE, sensor_name="NOPAT")]) \
|
||
|
|
== {"T1": 2, "u2": 1}, "DIFF patterns: by sensor_name, else the sensor's name; archived and others left out"
|
||
|
|
print("ok")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
check() if sys.argv[1:] == ["--check"] else main()
|