Pumping of the differential images: extraction/pump_diff.sh into data_diff/<SERIAL>/, in the tree of data/ (never data/ itself)
The OPTICAL_SHIFT-DIFF images full size, with what measuring them again needs under images_diff/ (index of every measurement per day with its exact time and why an image is missing, sensors.json with every pattern archived included, unit.json), and the metrics, IRON images, weather and targets.csv as pump.py and weather.py write them into data/. Its own lock and log; a day is indexed last, and not when its metrics failed; --min-free-gb. pump_metrics now returns whether every day succeeded; weather.py takes --data. Tried on JUNGFRAU 2026-09-01 and 2026-10-12..16: DIFF images byte-identical to pump.py --diff, metrics, IRON and weather byte-identical to data/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1b918facd9
commit
81908ca370
9 files changed
+660
-8
No files matched your search
@@ -0,0 +1,326 @@
|
||||
"""Pumps the differential images (OPTICAL_SHIFT-DIFF) of stations from the AIMsight platform, full size, with what is
|
||||
needed to measure them again: every pattern of the unit (archived ones included) and the unit's properties. READ-ONLY.
|
||||
Never touches data/ nor pump.py's pumping; design: docs/superpowers/specs/2026-11-19-pump-diff-design.md.
|
||||
|
||||
Run through extraction/pump_diff.sh, under WSL, from anywhere (see README.md):
|
||||
|
||||
extraction/pump_diff.sh --from 2026-09-01 --to 2026-09-08 JUNGFRAU a range (its history starts here)
|
||||
extraction/pump_diff.sh JUNGFRAU from its last indexed day up to now
|
||||
extraction/pump_diff.sh the same for every station of data_diff/
|
||||
|
||||
Output (default: the repository's data_diff/, next to this folder), the tree of data/ so that the other repositories
|
||||
read it the same way (docs/superpowers/specs/2026-11-19-configurations-design.md):
|
||||
<STATION>/images_diff/<YYYYMMDD>T<HHMM>Z_<measurement_id>.png the S3 object as is, named as in data/
|
||||
<STATION>/images_diff/index/<STATION>_diff_<YYYYMMDD>.csv every measurement of the day in the database:
|
||||
measurement_id, time_utc (exact), measurement_group_id, file (empty without image), status (ok; no_uuid: never
|
||||
reached S3; s3_missing: S3 returned nothing), properties (the measurement's JSON)
|
||||
<STATION>/images_diff/sensors.json every sensor of the unit: id, name, type, time_utc, properties [{name, value,
|
||||
time_utc, status}] (every property row as stored), rewritten at each run
|
||||
<STATION>/images_diff/unit.json the acquisition unit: id, serial_number, product_number, properties
|
||||
(under images_diff/, the tools of aimsight-tools do not see them: data.find_files skips that folder)
|
||||
<STATION>/metrics/, images/ (IRON, one per --every minutes), targets.csv (the DIFF patterns), weather/: as pump.py
|
||||
and weather.py write them into data/, with their own functions
|
||||
|
||||
Resumable: a day whose index exists and is before the last indexed day is not read again; the last indexed day and
|
||||
the later ones are (pumped while running, a day is incomplete); an image on disk is never downloaded again. An index
|
||||
is written after its day's metrics, IRON and DIFF images, every DIFF file through a temporary one: an indexed day is
|
||||
complete; a day whose metrics failed gets no index. Without --from, a station with nothing indexed is skipped (its
|
||||
history starts only on an explicit --from). Stops before a day when the disk has less than --min-free-gb left.
|
||||
|
||||
Same reads as pump.py (measurement, acquisition_unit; the copied queries of the app, branch `explo`), plus the
|
||||
`property` rows of the sensors and of the unit (AcquisitionUnit.list_sensors, Sensor.properties of
|
||||
app/aimsight_database.py). Variables read (names only, none is printed): AIMSIGHT_DATABASE_USER / _PSWD / _TURL /
|
||||
_NAME / _PORT, AIMSIGHT_S3_URL / _ACCESS_KEY / _SECRET_KEY / _BUCKET.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
UTC = timezone.utc
|
||||
MTYPE = "OPTICAL_SHIFT-DIFF"
|
||||
INDEX = ["measurement_id", "time_utc", "measurement_group_id", "file", "status", "properties"]
|
||||
|
||||
|
||||
def write_atomic(path, data):
|
||||
"""A file is either complete or absent: a run stopped in the middle leaves nothing that looks done."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
tmp.write_bytes(data)
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def indexed_days(folder):
|
||||
return {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date()
|
||||
for p in (folder / "images_diff" / "index").glob(f"{folder.name}_diff_*.csv")}
|
||||
|
||||
|
||||
def days_to_pump(indexed, start, end):
|
||||
"""The days of [start, end) to read: the missing ones, and from the last indexed day on (perhaps incomplete).
|
||||
Without start: from the last indexed day; None when nothing is indexed."""
|
||||
last = max(indexed, default=None)
|
||||
if start is None:
|
||||
if last is None:
|
||||
return None
|
||||
start = last
|
||||
days = (start + timedelta(days=k) for k in range((end - start).days))
|
||||
return [d for d in days if last is None or d not in indexed or d >= last]
|
||||
|
||||
|
||||
def pump_day(rows, client, bucket, ext, folder, station, day):
|
||||
"""The images of one day's measurements (rows: id, time_utc, measurement_group_id, measurement_uuid, properties),
|
||||
then its index. Returns the counts per status, plus the new images and their bytes."""
|
||||
c, index = Counter(), []
|
||||
for m in rows:
|
||||
t = m.time_utc.astimezone(UTC)
|
||||
name, status = f"{t:%Y%m%dT%H%M}Z_{m.id}.{ext}", "ok"
|
||||
path = folder / "images_diff" / name
|
||||
if m.measurement_uuid is None: # the image never reached S3 (unfinished multipart upload)
|
||||
name, status = "", "no_uuid"
|
||||
elif not path.exists():
|
||||
try:
|
||||
data = client.get_object(Bucket=bucket, Key=m.measurement_uuid)["Body"].read()
|
||||
except Exception: # like the app's get_measurement_payload: S3 not answering, the image is missing
|
||||
data = None
|
||||
if data:
|
||||
write_atomic(path, data)
|
||||
c["new"], c["bytes"] = c["new"] + 1, c["bytes"] + len(data)
|
||||
else:
|
||||
name, status = "", "s3_missing"
|
||||
c[status] += 1
|
||||
index.append([m.id, t.isoformat(), "" if m.measurement_group_id is None else m.measurement_group_id, name,
|
||||
status, "" if m.properties is None else json.dumps(m.properties, sort_keys=True)])
|
||||
out = io.StringIO()
|
||||
w = csv.writer(out, lineterminator="\n")
|
||||
w.writerow(INDEX)
|
||||
w.writerows(index)
|
||||
write_atomic(folder / "images_diff" / "index" / f"{station}_diff_{day:%Y%m%d}.csv", out.getvalue().encode())
|
||||
return c
|
||||
|
||||
|
||||
def prop(name, value, t, status):
|
||||
return {"name": name, "value": value, "time_utc": t.isoformat(), "status": status}
|
||||
|
||||
|
||||
def sensors_json(conn, unit):
|
||||
"""Every sensor of the unit, any type, archived included (`archive` is one of their properties), with every
|
||||
property row: a pattern redrawn on the platform is a new sensor, the old one stays."""
|
||||
from sqlalchemy import text
|
||||
rows = conn.execute(text("""
|
||||
SELECT s.id, s.name, s.type, s.time_utc, dt.name AS prop, p.value, p.time_utc AS ptime, p.status
|
||||
FROM sensor s LEFT JOIN property p ON p.sensor_id = s.id LEFT JOIN datatype dt ON dt.id = p.datatype_id
|
||||
WHERE s.acquisition_unit_id = :au ORDER BY s.id, dt.name, p.time_utc"""), {"au": unit}).all()
|
||||
sensors = {}
|
||||
for r in rows:
|
||||
s = sensors.setdefault(r.id, {"id": r.id, "name": r.name, "type": r.type, "time_utc": r.time_utc.isoformat(),
|
||||
"properties": []})
|
||||
if r.prop is not None:
|
||||
s["properties"].append(prop(r.prop, r.value, r.ptime, r.status))
|
||||
return list(sensors.values())
|
||||
|
||||
|
||||
def unit_json(conn, unit):
|
||||
"""The acquisition unit and its own property rows (focal_distance, pixel_size, algorithm...), not its sensors'."""
|
||||
from sqlalchemy import text
|
||||
u = conn.execute(text("SELECT id, serial_number, product_number FROM acquisition_unit WHERE id = :au"),
|
||||
{"au": unit}).one()
|
||||
rows = conn.execute(text("""
|
||||
SELECT dt.name, p.value, p.time_utc, p.status FROM property p JOIN datatype dt ON dt.id = p.datatype_id
|
||||
WHERE p.acquisition_unit_id = :au AND p.sensor_id IS NULL ORDER BY dt.name, p.time_utc"""), {"au": unit}).all()
|
||||
return {"id": u.id, "serial_number": u.serial_number, "product_number": u.product_number,
|
||||
"properties": [prop(*r) for r in rows]}
|
||||
|
||||
|
||||
def diff_patterns(sensors):
|
||||
"""{sensor_name: sensor_id} of the patterns the DIFF images are measured with (sensors_json rows): type
|
||||
OPTICAL_SHIFT-DIFF, a pattern, not archived; for targets.csv."""
|
||||
out = {}
|
||||
for s in sensors:
|
||||
p = {r["name"]: r["value"] for r in s["properties"]}
|
||||
if s["type"] == MTYPE and p.get("pattern") and not p.get("archive"):
|
||||
out[p.get("sensor_name") or s["name"]] = s["id"]
|
||||
return out
|
||||
|
||||
|
||||
def to_json(obj):
|
||||
return json.dumps(obj, indent=1, ensure_ascii=False, default=str).encode()
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("stations", nargs="*", help="serial number of the system, as in the URL of the System page; "
|
||||
"none: the stations already in --out")
|
||||
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD; without: the last indexed day of each "
|
||||
"station (a station with nothing indexed is skipped)")
|
||||
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD; without: up to now")
|
||||
p.add_argument("--delay", type=int, default=60, help="maximum duration of a query, in s (default 60: database "
|
||||
"shared with production)")
|
||||
p.add_argument("--every", type=int, default=60, help="IRON images: one per slot of N minutes, 0 = all (default 60, "
|
||||
"as pump.sh)")
|
||||
p.add_argument("--min-free-gb", type=float, default=50, help="stop before a day when less is free (default 50)")
|
||||
p.add_argument("--out", type=Path, default=Path(__file__).resolve().parent.parent / "data_diff")
|
||||
a = p.parse_args()
|
||||
|
||||
import boto3
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import URL
|
||||
|
||||
from pump import DB_NAMES, acquisition_unit, pump_images, pump_metrics, root_cause, sensors, write_targets
|
||||
|
||||
tomorrow = datetime.now(UTC).date() + timedelta(days=1)
|
||||
start = datetime.strptime(a.start, "%Y-%m-%d").date() if a.start else None
|
||||
end = min(datetime.strptime(a.end, "%Y-%m-%d").date(), tomorrow) if a.end else tomorrow
|
||||
stations = a.stations or sorted(f.name for f in a.out.glob("*") if f.is_dir())
|
||||
if not stations:
|
||||
sys.exit(f"No station: name one, with --from (nothing in {a.out} yet)")
|
||||
missing = [n for n in DB_NAMES if not os.environ.get(n)]
|
||||
if missing:
|
||||
sys.exit(f"Missing variables (names only): {', '.join(missing)}")
|
||||
|
||||
# the engine and the S3 client of pump.py's main
|
||||
url = URL.create("postgresql+psycopg2", username=os.environ["AIMSIGHT_DATABASE_USER"],
|
||||
password=os.environ["AIMSIGHT_DATABASE_PSWD"], host=os.environ["AIMSIGHT_DATABASE_TURL"],
|
||||
port=int(os.environ.get("AIMSIGHT_DATABASE_PORT", "5432")),
|
||||
database=os.environ["AIMSIGHT_DATABASE_NAME"])
|
||||
engine = create_engine(url, connect_args={
|
||||
"options": f"-c timezone=utc -c default_transaction_read_only=on -c statement_timeout={a.delay}s"})
|
||||
client = boto3.client("s3", endpoint_url=os.environ.get("AIMSIGHT_S3_URL", "http://s3:3900"),
|
||||
aws_access_key_id=os.environ.get("AIMSIGHT_S3_ACCESS_KEY", ""),
|
||||
aws_secret_access_key=os.environ.get("AIMSIGHT_S3_SECRET_KEY", ""),
|
||||
region_name="garage", use_ssl=False)
|
||||
bucket = os.environ.get("AIMSIGHT_S3_BUCKET", "measurements-assets")
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
mt = conn.execute(text("SELECT id, extension FROM measurement_type WHERE name = :n"), {"n": MTYPE}).first()
|
||||
conn.rollback()
|
||||
if mt is None:
|
||||
sys.exit(f"Unknown measurement type {MTYPE}")
|
||||
for station in stations:
|
||||
system_id = conn.execute(text("SELECT id FROM system WHERE serial_number = :s"),
|
||||
{"s": station}).scalar()
|
||||
unit = system_id and acquisition_unit(conn, system_id)
|
||||
conn.rollback()
|
||||
if not unit:
|
||||
print(f"{station}: " + ("unknown system" if system_id is None else "no recognised acquisition unit"))
|
||||
continue
|
||||
folder = a.out / station
|
||||
days = days_to_pump(indexed_days(folder), start, end)
|
||||
if days is None:
|
||||
print(f"{station}: nothing indexed yet, its history starts with --from")
|
||||
continue
|
||||
try: # what the database holds, so that an empty day is explained (LENZSPITZE: none after 2026-10-13)
|
||||
first, last = (conn.execute(text(
|
||||
"SELECT time_utc FROM measurement WHERE acquisition_unit_id = :au AND measurement_type_id = :mt "
|
||||
f"ORDER BY time_utc {o} LIMIT 1"), {"au": unit, "mt": mt.id}).scalar() for o in ("ASC", "DESC"))
|
||||
print(f"{station}: differential images in the database from {first:%Y-%m-%d} to {last:%Y-%m-%d}"
|
||||
if first else f"{station}: no differential image in the database")
|
||||
sens = sensors_json(conn, unit)
|
||||
write_atomic(folder / "images_diff" / "sensors.json", to_json(sens))
|
||||
write_atomic(folder / "images_diff" / "unit.json", to_json(unit_json(conn, unit)))
|
||||
t = write_targets(folder, sensors(conn, unit), diff_patterns(sens))
|
||||
print(f"{station}: targets.csv, {len(t)} sensors")
|
||||
except Exception as e: # the next run writes them
|
||||
print(f"{station} images range, sensors.json, unit.json, targets.csv: failed ({root_cause(e)}), "
|
||||
"run again")
|
||||
finally:
|
||||
conn.rollback()
|
||||
print(f"{station}: {len(days)} days, from {days[0] if days else '-'} to {days[-1] if days else '-'}")
|
||||
for day in days:
|
||||
free = shutil.disk_usage(a.out).free / 1e9
|
||||
if free < a.min_free_gb:
|
||||
sys.exit(f"Stopped before {station} {day}: {free:.0f} GB free, under --min-free-gb "
|
||||
f"{a.min_free_gb:g}")
|
||||
nxt = day + timedelta(days=1)
|
||||
try: # as pump.py into data/; a day whose metrics or IRON images failed stays open (no index)
|
||||
if not pump_metrics(conn, system_id, station, day, nxt, a.out, True):
|
||||
print(f"{station} {day}: left open (metrics failed), run again")
|
||||
continue
|
||||
pump_images(conn, (client, bucket), unit, station, day, nxt, "images", a.every, a.out)
|
||||
except Exception as e:
|
||||
print(f"{station} {day}: left open ({root_cause(e)}), run again")
|
||||
continue
|
||||
finally:
|
||||
conn.rollback()
|
||||
a0 = datetime.combine(day, datetime.min.time(), UTC)
|
||||
try:
|
||||
rows = conn.execute(text(
|
||||
"SELECT id, time_utc, measurement_group_id, measurement_uuid, properties FROM measurement "
|
||||
"WHERE acquisition_unit_id = :au AND measurement_type_id = :mt AND time_utc >= :a "
|
||||
"AND time_utc < :b ORDER BY time_utc"),
|
||||
{"au": unit, "mt": mt.id, "a": a0, "b": a0 + timedelta(days=1)}).all()
|
||||
except Exception as e: # no index for the day: the next run reads it again
|
||||
print(f"{station} {day}: failed ({root_cause(e)}), run again")
|
||||
continue
|
||||
finally:
|
||||
conn.rollback()
|
||||
c = pump_day(rows, client, bucket, mt.extension, folder, station, day)
|
||||
print(f"{station} {day}: {c['new']} new images, {c['bytes'] / 1e6:.0f} MB ({len(rows)} in "
|
||||
f"database: {c['ok']} on disk, {c['no_uuid']} never reached S3, {c['s3_missing']} not "
|
||||
"returned by S3)")
|
||||
time.sleep(0.5)
|
||||
try: # the MeteoSwiss weather of the days now in --out, as pump.sh does for data/
|
||||
import weather
|
||||
weather.main(stations, a.out)
|
||||
except Exception as e: # the next run fetches it
|
||||
print(f"weather: failed ({type(e).__name__}: {e}), run again")
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def check():
|
||||
"""Without database or S3: `python3 extraction/pump_diff.py --check`."""
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
|
||||
d = lambda n: date(2026, 11, n) # noqa: E731
|
||||
assert days_to_pump(set(), d(1), d(4)) == [d(1), d(2), d(3)]
|
||||
assert days_to_pump({d(1), d(2), d(5)}, d(1), d(7)) == [d(3), d(4), d(5), d(6)] # the last indexed day: again
|
||||
assert days_to_pump({d(2)}, None, d(5)) == [d(2), d(3), d(4)]
|
||||
assert days_to_pump(set(), None, d(5)) is None # no --from, nothing indexed: skipped
|
||||
|
||||
class S3:
|
||||
def get_object(self, Bucket, Key):
|
||||
if Key == "refused":
|
||||
raise OSError("NoSuchKey")
|
||||
return {"Body": io.BytesIO(b"png")}
|
||||
|
||||
t = lambda h, m: datetime(2026, 11, 3, h, m, 7, 250000, tzinfo=UTC) # noqa: E731
|
||||
rows = [SimpleNamespace(id=1, time_utc=t(10, 0), measurement_group_id=5, measurement_uuid="a", properties={"e": 1}),
|
||||
SimpleNamespace(id=2, time_utc=t(10, 0), measurement_group_id=5, measurement_uuid="b", properties=None),
|
||||
SimpleNamespace(id=3, time_utc=t(10, 15), measurement_group_id=None, measurement_uuid=None, properties={}),
|
||||
SimpleNamespace(id=4, time_utc=t(10, 30), measurement_group_id=None, measurement_uuid="refused",
|
||||
properties={})]
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
folder = Path(tmp) / "S"
|
||||
(folder / "images_diff").mkdir(parents=True)
|
||||
(folder / "images_diff" / "20261103T1000Z_2.png").write_bytes(b"old") # a burst: same minute, other id
|
||||
c = pump_day(rows, S3(), "bucket", "png", folder, "S", d(3))
|
||||
assert dict(c) == {"ok": 2, "new": 1, "bytes": 3, "no_uuid": 1, "s3_missing": 1}, c
|
||||
assert (folder / "images_diff" / "20261103T1000Z_1.png").read_bytes() == b"png"
|
||||
assert (folder / "images_diff" / "20261103T1000Z_2.png").read_bytes() == b"old" # never downloaded again
|
||||
assert not list(folder.rglob("*.tmp"))
|
||||
with open(folder / "images_diff" / "index" / "S_diff_20261103.csv", newline="") as fp:
|
||||
index = list(csv.DictReader(fp))
|
||||
assert [r["status"] for r in index] == ["ok", "ok", "no_uuid", "s3_missing"], index
|
||||
assert [r["file"] for r in index] == ["20261103T1000Z_1.png", "20261103T1000Z_2.png", "", ""], index
|
||||
assert index[0]["time_utc"] == "2026-11-03T10:00:07.250000+00:00" and index[0]["properties"] == '{"e": 1}'
|
||||
assert index[1]["properties"] == "" and index[2]["measurement_group_id"] == "", index
|
||||
assert indexed_days(folder) == {d(3)}
|
||||
row = lambda name, typ, **p: {"name": name, "type": typ, "id": len(p), # noqa: E731
|
||||
"properties": [{"name": k, "value": v} for k, v in p.items()]}
|
||||
assert diff_patterns([row("u1", MTYPE, pattern="b64", sensor_name="T1"), row("u2", MTYPE, pattern="b64"),
|
||||
row("u3", MTYPE, pattern="b64", sensor_name="OLD", archive="2026-01-01"),
|
||||
row("u4", "camera_movement", sensor_name="CAM"), row("u5", MTYPE, sensor_name="NOPAT")]) \
|
||||
== {"T1": 2, "u2": 1}, "DIFF patterns: by sensor_name, else the sensor's name; archived and others left out"
|
||||
print("ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
check() if sys.argv[1:] == ["--check"] else main()
|
||||
Reference in new issue
Block a user