The OPTICAL_SHIFT-DIFF images full size, with what measuring them again needs under images_diff/ (index of every measurement per day with its exact time and why an image is missing, sensors.json with every pattern archived included, unit.json), and the metrics, IRON images, weather and targets.csv as pump.py and weather.py write them into data/. Its own lock and log; a day is indexed last, and not when its metrics failed; --min-free-gb. pump_metrics now returns whether every day succeeded; weather.py takes --data. Tried on JUNGFRAU 2026-09-01 and 2026-10-12..16: DIFF images byte-identical to pump.py --diff, metrics, IRON and weather byte-identical to data/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
528 lines
31 KiB
Python
528 lines
31 KiB
Python
"""Pumps the system metrics, the displacements ("advanced" export) and the images of stations from the AIMsight
|
|
platform, in the format read by the app, the reading tools and the detectors. READ-ONLY. Tried on 2026-10-26
|
|
(NORDEND, 2 days); standalone since 2026-11-12.
|
|
|
|
Run through extraction/pump.sh, under WSL, from anywhere (see README.md):
|
|
|
|
extraction/pump.sh
|
|
extraction/pump.sh --from 2026-08-01 --to 2026-08-08 --images WEISSHORN
|
|
|
|
Without a station: those of `stations.txt`. Without `--from`: each station fills its missing days (displacements or
|
|
metrics) before the last one pumped, then restarts from that last day, pumped again (it was perhaps incomplete), or
|
|
from the last 30 days if it has none; without `--to`: up to now.
|
|
|
|
Output (default: the repository's `data/`, next to this folder):
|
|
<STATION>/metrics/<STATION>_<metric>_<YYYY_MM_DD>_00_00 to <YYYY_MM_DD>_23_00.csv system metrics, per day
|
|
<STATION>/displacement/<STATION>_advanced_<YYYYMMDD>.csv displacements, one file per day
|
|
+ corr_x_mm, corr_y_mm: corr * mm_per_px of the sensor that measured (references included)
|
|
<STATION>/targets.csv the targets and references measured: sensor_name, pattern_type,
|
|
distance_mm, mm_per_px (those of the last day pumped)
|
|
<STATION>/camera/<STATION>_camera_<YYYYMMDD>.csv camera movements, per day
|
|
<STATION>/images/<YYYYMMDD>T<HHMM>Z_<id>.<ext> OPTICAL_SHIFT-IRON images (--images)
|
|
<STATION>/images_diff/… differential images (--diff)
|
|
Resumable: with `--from`, a file already written is not read again, whatever the dates asked, except from the last
|
|
day pumped of the station (pumped during the day, it was incomplete); without `--from`, the days of the range are
|
|
replaced (images excepted). Days in UTC, `--to` excluded; the current day is pumped as it is,
|
|
its metrics up to the last completed hour. To redo a day, delete its files.
|
|
|
|
No dependency on the app (~/monitoring-dashboard): its reads are copied here, in SQL, as they are (branch `explo`,
|
|
commit 3dce73d of 2026-11-12): the query of System.get_metrics (the download of the System page);
|
|
build_advanced_frame (app/datapoints.py, what the Advanced tab and the parquet export plot), with
|
|
AcquisitionUnit.get_datapoints and AggregateEngine.get_sensor_aggregate / get_common_aggregate; the camera movements
|
|
of fetch_cam_movements_frame (AggregateEngine.get_cam_scalar_aggregate, what the viewer plots);
|
|
AcquisitionUnit.measurements and get_measurement_payload (S3). If the app changes its reads, this file does not follow.
|
|
Read-only connection (`default_transaction_read_only`), 60 s timeout per query (`--delay`). Variables read (names
|
|
only, none is printed): AIMSIGHT_DATABASE_USER / _PSWD / _TURL / _NAME / _PORT, and for the images
|
|
AIMSIGHT_S3_URL / _ACCESS_KEY / _SECRET_KEY / _BUCKET.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.engine import URL
|
|
|
|
import pumping # the tracked stations (stations.txt), next to this script
|
|
|
|
UTC = timezone.utc
|
|
NEW_STATION_DAYS = 30 # without --from, a station with no pumped displacements: the last 30 days
|
|
DB_NAMES = ["AIMSIGHT_DATABASE_USER", "AIMSIGHT_DATABASE_PSWD", "AIMSIGHT_DATABASE_TURL", "AIMSIGHT_DATABASE_NAME"]
|
|
TYPES = {"images": "OPTICAL_SHIFT-IRON", "images_diff": "OPTICAL_SHIFT-DIFF"} # folder -> measurement type
|
|
|
|
# --- constants of the app (app/datapoints.py, app/aggregates.py) ---
|
|
QUALITY = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"]
|
|
CAM_SIGNALS = ["pan", "tilt", "scale", "rotation", "sigma0"]
|
|
DEFAULT_ERROR_THRESHOLD = 0.7
|
|
DATATYPES = ["x", "y", "error", "corr_x", "corr_y", "reproj_err", "inlier", "sigma0"]
|
|
PROPERTIES = ["x", "y", "pattern_type", "sensor_name", "error_threshold"]
|
|
COLUMNS = ["time_utc", "sensor_name", "x", "y", "error_value", "error_threshold", "relative_x", "relative_y", "corr_x",
|
|
"corr_y", "reproj_err", "inlier", "sigma0", "pattern_type"] + QUALITY
|
|
NUMERIC = ["x", "y", "error_value", "error_threshold", "relative_x", "relative_y", "corr_x", "corr_y", "reproj_err",
|
|
"inlier", "sigma0"] + QUALITY
|
|
# inlier bands of shift_sensor_1min -> code; the last two NAMES are wrong (-1 rejected, -2 interpolated), the codes
|
|
# are right; ordered from best to worst
|
|
INLIER_BANDS = {"inlier_in_fit": 1.0, "inlier_out_fit": 0.0, "inlier_no_redundancy": -1.0, "inlier_rejected": -2.0}
|
|
|
|
|
|
def root_cause(e):
|
|
"""The original error (the database's): pandas and SQLAlchemy wrap it, the SQL query first."""
|
|
while e.__cause__ is not None:
|
|
e = e.__cause__
|
|
return f"{type(e).__name__}: {(str(e).splitlines() or [''])[0][:300]}"
|
|
|
|
|
|
def query(conn, sql, **params):
|
|
return pd.read_sql(text(sql), conn, params=params)
|
|
|
|
|
|
def acquisition_unit(conn, system_id):
|
|
"""Same choice as Device.get_acquisition_unit (app/base/models.py:268): the only unit, otherwise MER* / ME2S*."""
|
|
units = conn.execute(text("SELECT id, product_number FROM acquisition_unit WHERE system_id = :s"),
|
|
{"s": system_id}).all()
|
|
pick = units if len(units) == 1 else [u for u in units if u.product_number.startswith(("MER", "ME2S"))]
|
|
return pick[0].id if pick else None
|
|
|
|
|
|
def pumped_days(folder):
|
|
"""The days of displacements already pumped."""
|
|
return {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date()
|
|
for p in (folder / "displacement").glob(f"{folder.name}_advanced_*.csv")}
|
|
|
|
|
|
def given_ranges(folder, start, end):
|
|
"""With --from, without --replace: (start, end, replace) to pump. The missing days of the range, and from the last
|
|
day pumped (pumped during the day, it was incomplete) to the end of the range pumped again, as update_ranges."""
|
|
last = max(pumped_days(folder), default=None)
|
|
if last is None or not start <= last < end:
|
|
return [(start, end, False)]
|
|
return ([(start, last, False)] if start < last else []) + [(last, end, True)]
|
|
|
|
|
|
def update_ranges(folder, end, tomorrow):
|
|
"""Without --from: (start, end, replace) to pump, from the days of displacements already pumped. The missing days
|
|
between the first and the last (ranges pumped separately, failed day) are filled without touching the others; the
|
|
last day is pumped again (pumped during the day, it was incomplete) up to now; a station with no pumped
|
|
displacements gets the last 30 days."""
|
|
days = pumped_days(folder)
|
|
if not days:
|
|
return [(tomorrow - timedelta(days=NEW_STATION_DAYS + 1), end, True)]
|
|
# a failed metrics day has no file at all (one query for all of them): it is a gap too; a station without any
|
|
# metric never has one, its days are not gaps
|
|
metric_days = {datetime.strptime(p.name.split(" to ")[0][-16:-6], "%Y_%m_%d").date()
|
|
for p in (folder / "metrics").glob(f"{folder.name}_*_00_00 to *.csv")}
|
|
first, last = min(days), max(days)
|
|
gaps = [first + timedelta(days=k) for k in range((last - first).days)
|
|
if first + timedelta(days=k) not in days or (metric_days and first + timedelta(days=k) not in metric_days)]
|
|
return ([(gaps[0], last, False)] if gaps else []) + [(last, end, True)]
|
|
|
|
|
|
def pump_metrics(conn, system_id, station, start, end, out, replace):
|
|
"""Like the download of the System page (Dashboard/System.py, get_metrics(..., bucketing="1 hour")): per hour,
|
|
mean * transform_factor + transform_offset, or the list of values of a text metric; the labelled series of one
|
|
metric merged; an empty row for an hour without measurement. get_metrics counts in hours before now: its query is
|
|
redone here on a fixed day, one query per day for all the metrics. -> True when no day failed (pump_diff.py: a
|
|
day whose metrics failed is not marked done)."""
|
|
names = sorted(conn.execute(text("SELECT DISTINCT dt.name FROM datatype dt JOIN metric_series se "
|
|
"ON se.datatype_id = dt.id WHERE se.system_id = :s"), {"s": system_id}).scalars())
|
|
conn.rollback()
|
|
ok = True
|
|
for k in range((end - start).days):
|
|
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
|
|
paths = {n: out / station / "metrics"
|
|
/ f"{station}_{n.replace(os.sep, '-')}_{a:%Y_%m_%d}_00_00 to {a:%Y_%m_%d}_23_00.csv" for n in names}
|
|
if not replace and all(p.exists() for p in paths.values()):
|
|
continue
|
|
t0 = time.time()
|
|
try:
|
|
# date_trunc('hour'): same bucketing as time_bucket('1 hour')
|
|
df = query(conn, """
|
|
SELECT date_trunc('hour', ms.time_utc) AS hour, dt.name AS name,
|
|
avg(ms.value_num) * any_value(dt.transform_factor) + any_value(dt.transform_offset) AS num,
|
|
jsonb_agg(ms.value_text) FILTER (WHERE ms.value_text IS NOT NULL) AS txt
|
|
FROM metric_sample ms
|
|
JOIN metric_series se ON ms.series_id = se.id
|
|
JOIN datatype dt ON se.datatype_id = dt.id
|
|
WHERE se.system_id = :s AND ms.time_utc >= :a AND ms.time_utc < :b
|
|
GROUP BY 1, dt.name""", s=system_id, a=a, b=a + timedelta(days=1))
|
|
except Exception as e: # ponytail: like the displacements, a failed day is skipped, a new run picks it up
|
|
print(f"{station} {a:%Y-%m-%d} metrics: failed ({root_cause(e)}), run again")
|
|
ok = False
|
|
continue
|
|
finally:
|
|
conn.rollback()
|
|
df["hour"] = pd.to_datetime(df["hour"], utc=True)
|
|
hours = pd.to_datetime(pd.date_range(a, periods=24, freq="h"), utc=True) # same time zone as df, for reindex
|
|
# current day: the completed hours only, otherwise the hours to come would read as a gap
|
|
hours = hours[hours + pd.Timedelta(hours=1) <= pd.Timestamp.now(tz=UTC)]
|
|
for n, path in paths.items():
|
|
d = df[df["name"] == n].set_index("hour")
|
|
col = d["num"] if d["num"].notna().any() else d["txt"] # text metric: the list of the hour
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
# CSV of the System page: ',time_utc,metric', the index as first column; an empty day is written too
|
|
pd.DataFrame({"time_utc": hours, "metric": col.reindex(hours).to_numpy()}).to_csv(path)
|
|
print(f"{station} {a:%Y-%m-%d} metrics: {len(names)} files, {df['name'].nunique()} with data, "
|
|
f"{time.time() - t0:.0f} s")
|
|
return ok
|
|
|
|
|
|
# --- the "advanced" export: build_advanced_frame (app/datapoints.py) ---
|
|
|
|
def raw_datapoints(conn, unit, a, b):
|
|
"""AcquisitionUnit.get_datapoints: the datapoints of the unit, one type per column, the sensor properties next to
|
|
them (prop_x / prop_y: the reference position, namesake of a type)."""
|
|
types = ",\n".join(f"max(d.value) FILTER (WHERE dt.name = '{t}') AS {t}" for t in DATATYPES)
|
|
names = {p: f"prop_{p}" if p in DATATYPES else p for p in PROPERTIES}
|
|
props = ",\n".join(f"any_value(p_{p}.value) AS {names[p]}" for p in PROPERTIES)
|
|
joins = "\n".join(f"LEFT JOIN (SELECT p.sensor_id, p.value FROM property p JOIN datatype pdt "
|
|
f"ON p.datatype_id = pdt.id WHERE pdt.name = '{p}') p_{p} ON d.sensor_id = p_{p}.sensor_id"
|
|
for p in PROPERTIES)
|
|
return query(conn, f"""
|
|
SELECT d.time_utc, d.sensor_id, any_value(d.measurement_id) AS measurement_id,
|
|
{types},
|
|
{props}
|
|
FROM datapoint d
|
|
JOIN sensor s ON d.sensor_id = s.id
|
|
JOIN datatype dt ON d.datatype_id = dt.id
|
|
{joins}
|
|
WHERE s.acquisition_unit_id = :au AND d.time_utc >= :a AND d.time_utc <= :b
|
|
GROUP BY d.time_utc, d.sensor_id
|
|
ORDER BY d.sensor_id, d.time_utc""", au=unit, a=a, b=b)
|
|
|
|
|
|
def prepare_raw(shifts):
|
|
"""prepare_datapoints_frame: sigma0 (camera rows, without sensor_name) copied onto each sensor at its time;
|
|
relative = x - prop_x; error -> error_value; the full schema."""
|
|
if shifts.empty:
|
|
return pd.DataFrame(columns=COLUMNS + ["sensor_id"])
|
|
sig = shifts.loc[shifts["sigma0"].notna(), ["time_utc", "sigma0"]]
|
|
sigma0 = sig.groupby("time_utc")["sigma0"].median() if not sig.empty else None
|
|
shifts = shifts.dropna(subset=["sensor_name"]).reset_index(drop=True)
|
|
shifts["sigma0"] = shifts["time_utc"].map(sigma0) if sigma0 is not None else pd.NA
|
|
shifts["relative_x"] = shifts["x"] - pd.to_numeric(shifts["prop_x"], errors="coerce")
|
|
shifts["relative_y"] = shifts["y"] - pd.to_numeric(shifts["prop_y"], errors="coerce")
|
|
shifts = shifts.drop(columns=["prop_x", "prop_y"]).rename(columns={"error": "error_value"})
|
|
shifts["time_utc"] = pd.to_datetime(shifts["time_utc"])
|
|
shifts["sensor_name"] = shifts["sensor_name"].astype(str)
|
|
for c in NUMERIC:
|
|
if c in shifts:
|
|
shifts[c] = pd.to_numeric(shifts[c], errors="coerce")
|
|
return shifts.reindex(columns=COLUMNS + ["sensor_id"])
|
|
|
|
|
|
def aggregate_states(conn, unit, a, b):
|
|
"""fetch_aggregate_datapoints_frame: the per-minute medians of the state tables (shift_sensor_1min,
|
|
shift_common_1min), only source for "burst" devices, in the schema of the export (prepare_aggregate_datapoints_frame).
|
|
In the states, x_pct holds the CORRECTED x and rx_pct the raw x."""
|
|
thr = "COALESCE((th.value #>> '{}')::double precision, CAST(:thr AS double precision))"
|
|
sensor = query(conn, f"""
|
|
SELECT ss.time_utc, ss.sensor_id, COALESCE(sn.value #>> '{{}}', s.name) AS name,
|
|
approx_percentile(0.5, ss.x_pct) AS corr_x, approx_percentile(0.5, ss.y_pct) AS corr_y,
|
|
approx_percentile(0.5, ss.rx_pct) AS x, approx_percentile(0.5, ss.ry_pct) AS y,
|
|
approx_percentile(0.5, ss.error_pct) AS error, {thr} AS error_threshold,
|
|
approx_percentile(0.5, ss.reproj_err_pct) AS reproj_err,
|
|
{", ".join(f"ss.{n}" for n in INLIER_BANDS)},
|
|
{", ".join(f"approx_percentile(0.5, ss.{m}_pct) AS {m}" for m in QUALITY)}
|
|
FROM shift_sensor_1min ss
|
|
JOIN sensor s ON s.id = ss.sensor_id
|
|
LEFT JOIN (SELECT p.sensor_id, any_value(p.value) AS value FROM property p JOIN datatype dt
|
|
ON dt.id = p.datatype_id WHERE dt.name = 'sensor_name' GROUP BY p.sensor_id) sn
|
|
ON sn.sensor_id = ss.sensor_id
|
|
LEFT JOIN (SELECT p.sensor_id, any_value(p.value) AS value FROM property p JOIN datatype dt
|
|
ON dt.id = p.datatype_id WHERE dt.name = 'error_threshold' GROUP BY p.sensor_id) th
|
|
ON th.sensor_id = ss.sensor_id
|
|
WHERE s.acquisition_unit_id = :au AND ss.time_utc >= :a AND ss.time_utc <= :b
|
|
ORDER BY ss.time_utc""", au=unit, a=a, b=b, thr=DEFAULT_ERROR_THRESHOLD)
|
|
if sensor.empty:
|
|
return pd.DataFrame(columns=COLUMNS + ["sensor_id"])
|
|
common = query(conn, """
|
|
SELECT sc.time_utc, approx_percentile(0.5, sc.sigma0_pct) AS sigma0 FROM shift_common_1min sc
|
|
WHERE sc.acquisition_unit_id = :au AND sc.time_utc >= :a AND sc.time_utc <= :b""", au=unit, a=a, b=b)
|
|
out = pd.DataFrame({"time_utc": pd.to_datetime(sensor["time_utc"]), "sensor_name": sensor["name"].astype(str),
|
|
"sensor_id": sensor["sensor_id"]})
|
|
for col, src in [("error_value", "error"), ("error_threshold", "error_threshold"), ("corr_x", "corr_x"),
|
|
("corr_y", "corr_y"), ("relative_x", "corr_x"), ("relative_y", "corr_y"), ("x", "x"), ("y", "y"),
|
|
("reproj_err", "reproj_err")] + [(m, m) for m in QUALITY]:
|
|
out[col] = pd.to_numeric(sensor[src], errors="coerce")
|
|
# inlier_code_from_bands: the majority band, a tie goes to the worst; all at 0 -> NaN (no redundancy)
|
|
counts = sensor[list(INLIER_BANDS)].apply(pd.to_numeric, errors="coerce").fillna(0)
|
|
code = counts[list(INLIER_BANDS)[::-1]].idxmax(axis=1).map(INLIER_BANDS)
|
|
out["inlier"] = code.where(counts.sum(axis=1) > 0).to_numpy()
|
|
out["pattern_type"] = None
|
|
sig = common.loc[common["sigma0"].notna()]
|
|
out["sigma0"] = (out["time_utc"].map(sig.assign(time_utc=pd.to_datetime(sig["time_utc"]))
|
|
.groupby("time_utc")["sigma0"].median()) if not sig.empty else float("nan"))
|
|
return out.reindex(columns=COLUMNS + ["sensor_id"])
|
|
|
|
|
|
def merge(raw, agg):
|
|
"""merge_raw_and_aggregate_frames: the raw wins; a state is kept only where its sensor has no raw row in the same
|
|
minute."""
|
|
if agg.empty:
|
|
return raw
|
|
if raw.empty:
|
|
return agg
|
|
|
|
def utc(t):
|
|
t = pd.to_datetime(t)
|
|
return t.dt.tz_localize("UTC") if t.dt.tz is None else t.dt.tz_convert("UTC")
|
|
|
|
keys = pd.MultiIndex.from_arrays([raw["sensor_name"], utc(raw["time_utc"]).dt.floor("1min")])
|
|
agg = agg.loc[~pd.MultiIndex.from_arrays([agg["sensor_name"], utc(agg["time_utc"]).dt.floor("1min")]).isin(keys)]
|
|
if agg.empty:
|
|
return raw
|
|
agg = agg.copy()
|
|
tz = pd.to_datetime(raw["time_utc"]).dt.tz
|
|
t = utc(agg["time_utc"])
|
|
agg["time_utc"] = t.dt.tz_convert(tz) if tz is not None else t.dt.tz_localize(None)
|
|
return (pd.concat([raw, agg], ignore_index=True).sort_values(["sensor_name", "time_utc"], kind="stable")
|
|
.reset_index(drop=True).reindex(columns=COLUMNS + ["sensor_id"]))
|
|
|
|
|
|
SENSOR_PROPS = {"pattern_type": "pattern_type", "distance_mm": "distance_mm", "distance_convertion_factor": "mm_per_px"}
|
|
MM = {"corr_x": "corr_x_mm", "corr_y": "corr_y_mm"}
|
|
|
|
|
|
def sensors(conn, unit):
|
|
"""The sensors of the unit that have a pattern_type (targets and references), one row per sensor_id: sensor_name,
|
|
pattern_type, distance_mm (from the camera; 1000: not filled in on the platform) and mm_per_px
|
|
(distance_convertion_factor, the factor of the dashboard's exports in mm: DeviceOutput.get_scale,
|
|
compute_output_signals, (corr - offset) * factor). A name may have several sensors (an older configuration of the
|
|
unit, another lens): the factor of a measurement is that of its sensor_id, never looked up by name."""
|
|
prop = "max(p.value #>> '{{}}') FILTER (WHERE dt.name = '{}')".format
|
|
df = query(conn, f"""
|
|
SELECT s.id AS sensor_id, COALESCE({prop('sensor_name')}, s.name) AS sensor_name,
|
|
{", ".join(f"{prop(n)} AS {c}" for n, c in SENSOR_PROPS.items())}
|
|
FROM sensor s JOIN property p ON p.sensor_id = s.id JOIN datatype dt ON dt.id = p.datatype_id
|
|
WHERE s.acquisition_unit_id = :au
|
|
GROUP BY s.id, s.name HAVING {prop('pattern_type')} IS NOT NULL""", au=unit)
|
|
for c in ("distance_mm", "mm_per_px"):
|
|
df[c] = pd.to_numeric(df[c], errors="coerce")
|
|
return df.set_index("sensor_id")
|
|
|
|
|
|
def with_mm(df, props):
|
|
"""corr_x_mm / corr_y_mm = corr_x / corr_y * mm_per_px of the sensor_id of each row; empty without a factor."""
|
|
factor = df["sensor_id"].map(props["mm_per_px"]).astype(float)
|
|
for c, mm in MM.items():
|
|
df[mm] = pd.to_numeric(df[c], errors="coerce") * factor
|
|
return df
|
|
|
|
|
|
def advanced(conn, unit, a, b):
|
|
df = merge(prepare_raw(raw_datapoints(conn, unit, a, b)), aggregate_states(conn, unit, a, b))
|
|
return with_mm(df, sensors(conn, unit))
|
|
|
|
|
|
def camera(conn, unit, a, b):
|
|
"""Pan, tilt, scale, rotation and sigma0 of the camera, as median and bounds (median_ / min_ / max_) per minute,
|
|
hence one row per camera measurement (every 15 min at NORDEND, min = max): what the viewer plots
|
|
(fetch_cam_movements_frame), without its except, which would return a failed day as empty instead of leaving it
|
|
to run again."""
|
|
sigs = ", ".join(f"approx_percentile(0.5, sc.{s}_pct) AS median_{s}, min_val(sc.{s}_pct) AS min_{s}, "
|
|
f"max_val(sc.{s}_pct) AS max_{s}" for s in CAM_SIGNALS)
|
|
return query(conn, f"""SELECT sc.time_utc, {sigs} FROM shift_common_1min sc
|
|
WHERE sc.acquisition_unit_id = :au AND sc.time_utc >= :a AND sc.time_utc <= :b
|
|
ORDER BY sc.time_utc""", au=unit, a=a, b=b)
|
|
|
|
|
|
DAILY_FILES = (("displacement", "advanced", advanced), ("camera", "camera", camera)) # folder, file, reader
|
|
|
|
|
|
def pump_displacements(conn, unit, station, start, end, out, replace):
|
|
"""Displacements and camera movements: one file and one query per day, bounded, under the timeout (--delay)
|
|
-> {sensor_name: sensor_id} measured, the last day pumped winning (for targets.csv)."""
|
|
seen = {}
|
|
for k in range((end - start).days):
|
|
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
|
|
for folder, name, reader in DAILY_FILES:
|
|
path = out / station / folder / f"{station}_{name}_{a:%Y%m%d}.csv"
|
|
if path.exists() and not replace:
|
|
continue
|
|
t0 = time.time()
|
|
try:
|
|
df = reader(conn, unit, a, a + timedelta(days=1) - timedelta(microseconds=1))
|
|
except Exception as e: # ponytail: a failed day is skipped, a new run picks it up; no retry
|
|
print(f"{station} {a:%Y-%m-%d} {name}: failed ({root_cause(e)}), run again")
|
|
continue
|
|
finally:
|
|
conn.rollback() # closes the read transaction
|
|
if "sensor_name" in df:
|
|
df = df.drop_duplicates(["sensor_name", "time_utc"])
|
|
if "sensor_id" in df:
|
|
seen.update(df.dropna(subset=["sensor_id"]).groupby("sensor_name")["sensor_id"].last().to_dict())
|
|
df = df.drop(columns="sensor_id")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
df.to_csv(path, index=False) # an empty day is written too: the next run does not ask for it again
|
|
print(f"{station} {a:%Y-%m-%d} {name}: {len(df)} rows, {time.time() - t0:.0f} s -> {path.name}")
|
|
return seen
|
|
|
|
|
|
def write_targets(folder, props, seen):
|
|
"""targets.csv: the properties of the sensors measured (seen: {name: sensor_id}); the names not measured in this
|
|
run keep their line."""
|
|
path = folder / "targets.csv"
|
|
new = props.loc[[i for i in seen.values() if i in props.index], ["sensor_name", *SENSOR_PROPS.values()]]
|
|
old = pd.read_csv(path) if path.exists() else new.iloc[:0]
|
|
t = pd.concat([old[~old["sensor_name"].astype(str).isin(new["sensor_name"])], new]).sort_values("sensor_name")
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
t.to_csv(path, index=False)
|
|
return t
|
|
|
|
|
|
def pump_images(conn, s3, unit, station, start, end, folder, every, out):
|
|
mt = conn.execute(text("SELECT id, extension FROM measurement_type WHERE name = :n"), {"n": TYPES[folder]}).first()
|
|
if mt is None:
|
|
print(f"{station}: unknown type {TYPES[folder]}")
|
|
return
|
|
client, bucket_name = s3
|
|
for k in range((end - start).days):
|
|
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
|
|
measurements = conn.execute(text(
|
|
"SELECT id, time_utc, measurement_uuid FROM measurement WHERE acquisition_unit_id = :au "
|
|
"AND measurement_type_id = :mt AND time_utc >= :a AND time_utc < :b ORDER BY time_utc"),
|
|
{"au": unit, "mt": mt.id, "a": a, "b": a + timedelta(days=1)}).all()
|
|
conn.rollback()
|
|
seen, n, nbytes, no_uuid, unreturned = set(), 0, 0, 0, 0
|
|
for m in measurements:
|
|
# empty measurement_uuid: the image never reached S3 (unfinished multipart upload); skipped, the slot
|
|
# takes the next image if there is one
|
|
if m.measurement_uuid is None:
|
|
no_uuid += 1
|
|
continue
|
|
t = m.time_utc.astimezone(UTC)
|
|
bucket = (t.hour * 60 + t.minute) // every if every else m.id # one image per slot of `every` min
|
|
path = out / station / folder / f"{t:%Y%m%dT%H%M}Z_{m.id}.{mt.extension}"
|
|
if bucket in seen:
|
|
continue
|
|
seen.add(bucket) # even if already there: otherwise the next run would take the next image of the slot
|
|
if path.exists():
|
|
continue
|
|
try:
|
|
data = client.get_object(Bucket=bucket_name, Key=m.measurement_uuid)["Body"].read()
|
|
except Exception: # like get_measurement_payload: S3 not answering, the image is counted as missing
|
|
data = None
|
|
if data:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(data)
|
|
n, nbytes = n + 1, nbytes + len(data)
|
|
else:
|
|
unreturned += 1
|
|
# the detail says why a day gives 0: nothing in the database, never reached S3, or S3 not returning them
|
|
print(f"{station} {a:%Y-%m-%d} {folder}: {n} new images, {nbytes / 1e6:.0f} MB "
|
|
f"({len(measurements)} in database, {no_uuid} never reached S3, {unreturned} not returned by S3)")
|
|
time.sleep(0.5)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("stations", nargs="*", help="serial number of the system, as in the URL of the System page; "
|
|
"none: those of stations.txt")
|
|
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD; without: the last pumped day of each "
|
|
"station, pumped again, or the last 30 days")
|
|
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD; without: up to now")
|
|
p.add_argument("--replace", action="store_true", help="with --from: the days already pumped are pumped again "
|
|
"(a file is replaced only once its day is read)")
|
|
p.add_argument("--images", action="store_true", help="also the OPTICAL_SHIFT-IRON images")
|
|
p.add_argument("--diff", action="store_true", help="also the OPTICAL_SHIFT-DIFF images")
|
|
p.add_argument("--every", type=int, default=60, help="one image per slot of N minutes, 0 = all (default 60)")
|
|
p.add_argument("--delay", type=int, default=60, help="maximum duration of a query, in s (default 60: database "
|
|
"shared with production); longer only for a day that failed on a timeout")
|
|
p.add_argument("--out", type=Path, default=Path(__file__).resolve().parent.parent / "data")
|
|
a = p.parse_args()
|
|
tomorrow = datetime.now(UTC).date() + timedelta(days=1)
|
|
start = datetime.strptime(a.start, "%Y-%m-%d").date() if a.start else None
|
|
end = min(datetime.strptime(a.end, "%Y-%m-%d").date(), tomorrow) if a.end else tomorrow # no day to come
|
|
stations = a.stations or list(pumping.tracked())
|
|
|
|
missing = [n for n in DB_NAMES if not os.environ.get(n)]
|
|
if missing:
|
|
sys.exit(f"Missing variables (names only): {', '.join(missing)}")
|
|
url = URL.create("postgresql+psycopg2", username=os.environ["AIMSIGHT_DATABASE_USER"],
|
|
password=os.environ["AIMSIGHT_DATABASE_PSWD"], host=os.environ["AIMSIGHT_DATABASE_TURL"],
|
|
port=int(os.environ.get("AIMSIGHT_DATABASE_PORT", "5432")),
|
|
database=os.environ["AIMSIGHT_DATABASE_NAME"])
|
|
engine = create_engine(url, connect_args={
|
|
"options": f"-c timezone=utc -c default_transaction_read_only=on -c statement_timeout={a.delay}s"})
|
|
s3 = None
|
|
if a.images or a.diff:
|
|
import boto3 # like the app's S3client (Garage)
|
|
s3 = (boto3.client("s3", endpoint_url=os.environ.get("AIMSIGHT_S3_URL", "http://s3:3900"),
|
|
aws_access_key_id=os.environ.get("AIMSIGHT_S3_ACCESS_KEY", ""),
|
|
aws_secret_access_key=os.environ.get("AIMSIGHT_S3_SECRET_KEY", ""),
|
|
region_name="garage", use_ssl=False),
|
|
os.environ.get("AIMSIGHT_S3_BUCKET", "measurements-assets"))
|
|
try:
|
|
with engine.connect() as conn:
|
|
for station in stations:
|
|
system_id = conn.execute(text("SELECT id FROM system WHERE serial_number = :s"), {"s": station}).scalar()
|
|
if system_id is None:
|
|
print(f"{station}: unknown system")
|
|
continue
|
|
ranges = ([(start, end, True)] if start and a.replace else given_ranges(a.out / station, start, end)
|
|
if start else update_ranges(a.out / station, end, tomorrow))
|
|
unit = acquisition_unit(conn, system_id)
|
|
conn.rollback()
|
|
seen = {}
|
|
for range_start, range_end, replace in ranges:
|
|
print(f"{station}: from {range_start} to {range_end - timedelta(days=1)} inclusive"
|
|
+ ("" if replace else ", missing days only"))
|
|
pump_metrics(conn, system_id, station, range_start, range_end, a.out, replace)
|
|
if unit is None:
|
|
print(f"{station}: no recognised acquisition unit, no displacements, camera or images")
|
|
continue
|
|
seen.update(pump_displacements(conn, unit, station, range_start, range_end, a.out, replace))
|
|
for flag, folder in ((a.images, "images"), (a.diff, "images_diff")):
|
|
if flag:
|
|
pump_images(conn, s3, unit, station, range_start, range_end, folder, a.every, a.out)
|
|
if not seen:
|
|
continue
|
|
try:
|
|
props = sensors(conn, unit)
|
|
except Exception as e: # the next run writes them
|
|
print(f"{station} targets: failed ({root_cause(e)}), run again")
|
|
continue
|
|
finally:
|
|
conn.rollback()
|
|
t = write_targets(a.out / station, props, seen)
|
|
print(f"{station} targets.csv: {len(t)} sensors, {t['distance_mm'].notna().sum()} with distance_mm, "
|
|
f"{t['mm_per_px'].notna().sum()} with mm_per_px")
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def check():
|
|
"""with_mm and write_targets without the database: `uv run --with pandas --with sqlalchemy python
|
|
extraction/pump.py --check`."""
|
|
import tempfile
|
|
props = pd.DataFrame({"sensor_name": ["T1", "T1", "REF", "T2"], "pattern_type": ["target", "target", "global_ref",
|
|
"target"], "distance_mm": [20000, 20000, 50000, 1000], "mm_per_px": [7.0, 2.0, 18.0, None]},
|
|
index=pd.Index([11, 12, 13, 14], name="sensor_id"))
|
|
df = pd.DataFrame({"sensor_id": [12, 13, 14, 12], "corr_x": [0.5, 0.1, 0.1, None], "corr_y": [-1, 0.2, 0.2, None]})
|
|
df = with_mm(df, props) # T1 measured by its sensor 12 (2 mm/px), not by its older namesake 11 (7 mm/px)
|
|
assert df["corr_x_mm"].tolist()[:2] == [1.0, 1.8] and df["corr_y_mm"].tolist()[:2] == [-2.0, 3.6], df
|
|
assert df[["corr_x_mm", "corr_y_mm"]].iloc[2:].isna().all().all(), df # no factor, no corr: empty
|
|
with tempfile.TemporaryDirectory() as d:
|
|
pd.DataFrame({"sensor_name": ["OLD", "T1"], "pattern_type": "target", "distance_mm": [1, 1],
|
|
"mm_per_px": [1.0, 1.0]}).to_csv(Path(d) / "targets.csv", index=False)
|
|
t = write_targets(Path(d), props, {"T1": 12, "REF": 13})
|
|
assert t.values.tolist() == [["OLD", "target", 1, 1.0], ["REF", "global_ref", 50000, 18.0],
|
|
["T1", "target", 20000, 2.0]], t
|
|
with tempfile.TemporaryDirectory() as d: # given_ranges: the last day pumped and after, pumped again
|
|
f, day = Path(d) / "S", lambda n: datetime(2026, 11, n).date()
|
|
assert given_ranges(f, day(1), day(19)) == [(day(1), day(19), False)] # nothing pumped
|
|
(f / "displacement").mkdir(parents=True)
|
|
for n in (2, 18):
|
|
(f / "displacement" / f"S_advanced_202611{n:02d}.csv").touch()
|
|
assert given_ranges(f, day(1), day(19)) == [(day(1), day(18), False), (day(18), day(19), True)]
|
|
assert given_ranges(f, day(18), day(19)) == [(day(18), day(19), True)]
|
|
assert given_ranges(f, day(1), day(10)) == [(day(1), day(10), False)] # the last day after the range
|
|
print("ok")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
check() if sys.argv[1:] == ["--check"] else main()
|