153 lines
8.4 KiB
Python
153 lines
8.4 KiB
Python
"""Pompe les déplacements (export « advanced ») et les images de stations de la plateforme AIMsight, dans le
|
|||
|
|
format que Support-Agent lit déjà. LECTURE SEULE. Brouillon, jamais lancé.
|
||
|
|
|
||
|
|
À lancer dans le dev container, depuis la racine de monitoring-dashboard-displacement (voir README.md) :
|
||
|
|
|
||
|
|
(set -a && . ./.env && set +a && python3 extraction/pomper.py --du 2026-08-01 --au 2026-08-08 --images WEISSHORN)
|
||
|
|
|
||
|
|
Sortie (défaut `extraction/sortie/`), à copier ensuite dans Support-Agent/data/ :
|
||
|
|
<STATION>/displacement/<STATION>_advanced_<AAAAMMJJ>_<AAAAMMJJ>.csv un fichier par lot de --lot jours
|
||
|
|
<STATION>/images/<AAAAMMJJ>T<HHMM>Z_<id>.<ext> images OPTICAL_SHIFT-IRON (--images)
|
||
|
|
<STATION>/images_diff/… images différentielles (--diff)
|
||
|
|
Reprenable : un fichier déjà écrit n'est pas relu. Jours en UTC, `--au` exclu.
|
||
|
|
|
||
|
|
Réutilise le code de l'appli : build_advanced_frame (ce que tracent l'onglet Advanced et l'export parquet),
|
||
|
|
AcquisitionUnit.measurements et get_measurement_payload (S3). Connexion : même schéma que
|
||
|
|
scripts/domain_diagnostics.py, un moteur dédié en lecture seule, délai de 60 s. Pas DatabaseEngine : son __init__
|
||
|
|
lance du DDL et un écouteur NOTIFY. Variables lues (noms seulement, aucune n'est affichée) :
|
||
|
|
AIMSIGHT_DATABASE_USER / _PSWD / _TURL / _NAME / _PORT, et pour les images AIMSIGHT_S3_URL / _ACCESS_KEY /
|
||
|
|
_SECRET_KEY / _BUCKET.
|
||
|
|
"""
|
||
|
|
import argparse
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
from datetime import datetime, timedelta, timezone
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
sys.path.insert(0, os.getcwd()) # racine de l'appli : le paquet `app`
|
||
|
|
import pandas as pd
|
||
|
|
from sqlalchemy import create_engine
|
||
|
|
from sqlalchemy.engine import URL
|
||
|
|
from sqlalchemy.orm import Session
|
||
|
|
|
||
|
|
from app.aggregates import AggregateEngine
|
||
|
|
from app.aimsight_database import DatabaseEngine, Measurement, MeasurementType, S3client, System
|
||
|
|
from app.datapoints import build_advanced_frame
|
||
|
|
|
||
|
|
UTC = timezone.utc
|
||
|
|
DB_NAMES = ["AIMSIGHT_DATABASE_USER", "AIMSIGHT_DATABASE_PSWD", "AIMSIGHT_DATABASE_TURL", "AIMSIGHT_DATABASE_NAME"]
|
||
|
|
TYPES = {"images": "OPTICAL_SHIFT-IRON", "images_diff": "OPTICAL_SHIFT-DIFF"} # dossier -> type de mesure
|
||
|
|
|
||
|
|
|
||
|
|
class ReadOnlyDB:
|
||
|
|
"""Le strict nécessaire à build_advanced_frame et à get_measurement_payload (les fonctions de DatabaseEngine)."""
|
||
|
|
get_sensor_state_aggregate = DatabaseEngine.get_sensor_state_aggregate
|
||
|
|
get_common_state_aggregate = DatabaseEngine.get_common_state_aggregate
|
||
|
|
get_measurement_payload = DatabaseEngine.get_measurement_payload
|
||
|
|
|
||
|
|
def __init__(self, engine, session, s3):
|
||
|
|
self._session, self.s3client, self.aggregates = session, s3, AggregateEngine(engine)
|
||
|
|
|
||
|
|
|
||
|
|
def acquisition_unit(session, station):
|
||
|
|
"""Même choix que Device.get_acquisition_unit (app/base/models.py:268) : l'unique unité, sinon MER* / ME2S*."""
|
||
|
|
system = session.query(System).filter_by(serial_number=station).one_or_none()
|
||
|
|
if system is None:
|
||
|
|
sys.exit(f"{station} : système inconnu")
|
||
|
|
units = list(system.acquisition_units.values())
|
||
|
|
pick = units if len(units) == 1 else [u for u in units if u.product_number.startswith(("MER", "ME2S"))]
|
||
|
|
if not pick:
|
||
|
|
sys.exit(f"{station} : pas d'unité d'acquisition reconnue")
|
||
|
|
return pick[0]
|
||
|
|
|
||
|
|
|
||
|
|
def pomper_deplacements(unit, db, station, debut, fin, lot, out):
|
||
|
|
for lo in range(0, (fin - debut).days, lot):
|
||
|
|
d0, d1 = debut + timedelta(days=lo), min(debut + timedelta(days=lo + lot), fin)
|
||
|
|
path = out / station / "displacement" / f"{station}_advanced_{d0:%Y%m%d}_{d1 - timedelta(days=1):%Y%m%d}.csv"
|
||
|
|
if path.exists():
|
||
|
|
continue
|
||
|
|
t0, frames = time.time(), []
|
||
|
|
try:
|
||
|
|
for k in range((d1 - d0).days): # un jour par requête : bornée, sous le délai de 60 s
|
||
|
|
a = datetime.combine(d0 + timedelta(days=k), datetime.min.time(), UTC)
|
||
|
|
frames.append(build_advanced_frame(unit, db, a, a + timedelta(days=1) - timedelta(microseconds=1)))
|
||
|
|
db._session.rollback() # ferme la transaction de lecture
|
||
|
|
except Exception as e: # ponytail : un lot en échec est sauté, relancer le reprend ; pas de nouvel essai
|
||
|
|
db._session.rollback()
|
||
|
|
print(f"{station} {d0} : échec du lot ({type(e).__name__}), à relancer")
|
||
|
|
continue
|
||
|
|
df = pd.concat(frames, ignore_index=True).drop_duplicates(["sensor_name", "time_utc"])
|
||
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
df.to_csv(path, index=False)
|
||
|
|
print(f"{station} {d0}..{d1}: {len(df)} lignes, {time.time() - t0:.0f} s -> {path.name}")
|
||
|
|
|
||
|
|
|
||
|
|
def pomper_images(unit, db, session, station, debut, fin, dossier, every, out):
|
||
|
|
mt = session.query(MeasurementType).filter_by(name=TYPES[dossier]).one_or_none()
|
||
|
|
if mt is None:
|
||
|
|
print(f"{station} : type {TYPES[dossier]} inconnu")
|
||
|
|
return
|
||
|
|
for k in range((fin - debut).days):
|
||
|
|
a = datetime.combine(debut + timedelta(days=k), datetime.min.time(), UTC)
|
||
|
|
q = unit.measurements.filter(Measurement.measurement_type_id == mt.id, Measurement.time_utc >= a,
|
||
|
|
Measurement.time_utc < a + timedelta(days=1)).order_by(Measurement.time_utc)
|
||
|
|
seen, n, octets = set(), 0, 0
|
||
|
|
for m in q.all():
|
||
|
|
t = m.time_utc.astimezone(UTC)
|
||
|
|
bucket = (t.hour * 60 + t.minute) // every if every else m.id # une image par tranche de `every` min
|
||
|
|
path = out / station / dossier / f"{t:%Y%m%dT%H%M}Z_{m.id}.{mt.extension}"
|
||
|
|
if bucket in seen or path.exists():
|
||
|
|
continue
|
||
|
|
seen.add(bucket)
|
||
|
|
data = db.get_measurement_payload(m.measurement_uuid) # None si S3 ne répond pas
|
||
|
|
if data:
|
||
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
path.write_bytes(data)
|
||
|
|
n, octets = n + 1, octets + len(data)
|
||
|
|
session.rollback()
|
||
|
|
print(f"{station} {a:%Y-%m-%d} {dossier}: {n} images, {octets / 1e6:.0f} Mo")
|
||
|
|
time.sleep(0.5)
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
|
|
p.add_argument("stations", nargs="+", help="numéro de série du système, comme dans l'URL de la page System")
|
||
|
|
p.add_argument("--du", required=True, help="premier jour UTC, AAAA-MM-JJ")
|
||
|
|
p.add_argument("--au", required=True, help="dernier jour UTC exclu, AAAA-MM-JJ")
|
||
|
|
p.add_argument("--lot", type=int, default=7, help="jours par fichier CSV (défaut 7)")
|
||
|
|
p.add_argument("--images", action="store_true", help="aussi les images OPTICAL_SHIFT-IRON")
|
||
|
|
p.add_argument("--diff", action="store_true", help="aussi les images OPTICAL_SHIFT-DIFF")
|
||
|
|
p.add_argument("--every", type=int, default=60, help="une image par tranche de N minutes, 0 = toutes (défaut 60)")
|
||
|
|
p.add_argument("--out", type=Path, default=Path("extraction/sortie"))
|
||
|
|
a = p.parse_args()
|
||
|
|
debut, fin = (datetime.strptime(x, "%Y-%m-%d").date() for x in (a.du, a.au))
|
||
|
|
|
||
|
|
missing = [n for n in DB_NAMES if not os.environ.get(n)]
|
||
|
|
if missing:
|
||
|
|
sys.exit(f"Variables absentes (noms seulement) : {', '.join(missing)}")
|
||
|
|
url = URL.create("postgresql+psycopg2", username=os.environ["AIMSIGHT_DATABASE_USER"],
|
||
|
|
password=os.environ["AIMSIGHT_DATABASE_PSWD"], host=os.environ["AIMSIGHT_DATABASE_TURL"],
|
||
|
|
port=int(os.environ.get("AIMSIGHT_DATABASE_PORT", "5432")),
|
||
|
|
database=os.environ["AIMSIGHT_DATABASE_NAME"])
|
||
|
|
engine = create_engine(url, connect_args={
|
||
|
|
"options": "-c timezone=utc -c default_transaction_read_only=on -c statement_timeout=60s"})
|
||
|
|
s3 = S3client(os.environ.get("AIMSIGHT_S3_URL", "http://s3:3900"), os.environ.get("AIMSIGHT_S3_ACCESS_KEY", ""),
|
||
|
|
os.environ.get("AIMSIGHT_S3_SECRET_KEY", ""),
|
||
|
|
os.environ.get("AIMSIGHT_S3_BUCKET", "measurements-assets")) if (a.images or a.diff) else None
|
||
|
|
try:
|
||
|
|
with Session(engine) as session:
|
||
|
|
db = ReadOnlyDB(engine, session, s3)
|
||
|
|
for station in a.stations:
|
||
|
|
unit = acquisition_unit(session, station)
|
||
|
|
pomper_deplacements(unit, db, station, debut, fin, a.lot, a.out)
|
||
|
|
for flag, dossier in ((a.images, "images"), (a.diff, "images_diff")):
|
||
|
|
if flag:
|
||
|
|
pomper_images(unit, db, session, station, debut, fin, dossier, a.every, a.out)
|
||
|
|
finally:
|
||
|
|
engine.dispose()
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|