410 lines
25 KiB
Python
410 lines
25 KiB
Python
"""Pumps the system metrics, the displacements ("advanced" export) and the images of stations from the AIMsight
|
|
platform, in the format read by the app, the reading tools and the detectors. READ-ONLY. Tried on 2026-10-26
|
|
(NORDEND, 2 days); standalone since 2026-11-12.
|
|
|
|
Run through extraction/pump.sh, under WSL, from anywhere (see README.md):
|
|
|
|
extraction/pump.sh
|
|
extraction/pump.sh --from 2026-08-01 --to 2026-08-08 --images WEISSHORN
|
|
|
|
Without a station: those of `stations.txt`. Without `--from`: each station fills its missing days (displacements or
|
|
metrics) before the last one pumped, then restarts from that last day, pumped again (it was perhaps incomplete), or
|
|
from the last 30 days if it has none; without `--to`: up to now.
|
|
|
|
Output (default: the repository's `data/`, next to this folder):
|
|
<STATION>/metrics/<STATION>_<metric>_<YYYY_MM_DD>_00_00 to <YYYY_MM_DD>_23_00.csv system metrics, per day
|
|
<STATION>/displacement/<STATION>_advanced_<YYYYMMDD>.csv displacements, one file per day
|
|
<STATION>/camera/<STATION>_camera_<YYYYMMDD>.csv camera movements, per day
|
|
<STATION>/images/<YYYYMMDD>T<HHMM>Z_<id>.<ext> OPTICAL_SHIFT-IRON images (--images)
|
|
<STATION>/images_diff/… differential images (--diff)
|
|
Resumable: with `--from`, a file already written is not read again, whatever the dates asked; without `--from`, the
|
|
days of the range are replaced (images excepted). Days in UTC, `--to` excluded; the current day is pumped as it is,
|
|
its metrics up to the last completed hour. To redo a day, delete its files.
|
|
|
|
No dependency on the app (~/monitoring-dashboard): its reads are copied here, in SQL, as they are (branch `explo`,
|
|
commit 3dce73d of 2026-11-12): the query of System.get_metrics (the download of the System page);
|
|
build_advanced_frame (app/datapoints.py, what the Advanced tab and the parquet export plot), with
|
|
AcquisitionUnit.get_datapoints and AggregateEngine.get_sensor_aggregate / get_common_aggregate; the camera movements
|
|
of fetch_cam_movements_frame (AggregateEngine.get_cam_scalar_aggregate, what the viewer plots);
|
|
AcquisitionUnit.measurements and get_measurement_payload (S3). If the app changes its reads, this file does not follow.
|
|
Read-only connection (`default_transaction_read_only`), 60 s timeout per query (`--delay`). Variables read (names
|
|
only, none is printed): AIMSIGHT_DATABASE_USER / _PSWD / _TURL / _NAME / _PORT, and for the images
|
|
AIMSIGHT_S3_URL / _ACCESS_KEY / _SECRET_KEY / _BUCKET.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.engine import URL
|
|
|
|
UTC = timezone.utc
|
|
STATIONS = Path(__file__).resolve().parent / "stations.txt" # the tracked stations, one per line
|
|
NEW_STATION_DAYS = 30 # without --from, a station with no pumped displacements: the last 30 days
|
|
DB_NAMES = ["AIMSIGHT_DATABASE_USER", "AIMSIGHT_DATABASE_PSWD", "AIMSIGHT_DATABASE_TURL", "AIMSIGHT_DATABASE_NAME"]
|
|
TYPES = {"images": "OPTICAL_SHIFT-IRON", "images_diff": "OPTICAL_SHIFT-DIFF"} # folder -> measurement type
|
|
|
|
# --- constants of the app (app/datapoints.py, app/aggregates.py) ---
|
|
QUALITY = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"]
|
|
CAM_SIGNALS = ["pan", "tilt", "scale", "rotation", "sigma0"]
|
|
DEFAULT_ERROR_THRESHOLD = 0.7
|
|
DATATYPES = ["x", "y", "error", "corr_x", "corr_y", "reproj_err", "inlier", "sigma0"]
|
|
PROPERTIES = ["x", "y", "pattern_type", "sensor_name", "error_threshold"]
|
|
COLUMNS = ["time_utc", "sensor_name", "x", "y", "error_value", "error_threshold", "relative_x", "relative_y", "corr_x",
|
|
"corr_y", "reproj_err", "inlier", "sigma0", "pattern_type"] + QUALITY
|
|
NUMERIC = ["x", "y", "error_value", "error_threshold", "relative_x", "relative_y", "corr_x", "corr_y", "reproj_err",
|
|
"inlier", "sigma0"] + QUALITY
|
|
# inlier bands of shift_sensor_1min -> code; the last two NAMES are wrong (-1 rejected, -2 interpolated), the codes
|
|
# are right; ordered from best to worst
|
|
INLIER_BANDS = {"inlier_in_fit": 1.0, "inlier_out_fit": 0.0, "inlier_no_redundancy": -1.0, "inlier_rejected": -2.0}
|
|
|
|
|
|
def root_cause(e):
|
|
"""The original error (the database's): pandas and SQLAlchemy wrap it, the SQL query first."""
|
|
while e.__cause__ is not None:
|
|
e = e.__cause__
|
|
return f"{type(e).__name__}: {(str(e).splitlines() or [''])[0][:300]}"
|
|
|
|
|
|
def query(conn, sql, **params):
|
|
return pd.read_sql(text(sql), conn, params=params)
|
|
|
|
|
|
def acquisition_unit(conn, system_id):
|
|
"""Same choice as Device.get_acquisition_unit (app/base/models.py:268): the only unit, otherwise MER* / ME2S*."""
|
|
units = conn.execute(text("SELECT id, product_number FROM acquisition_unit WHERE system_id = :s"),
|
|
{"s": system_id}).all()
|
|
pick = units if len(units) == 1 else [u for u in units if u.product_number.startswith(("MER", "ME2S"))]
|
|
return pick[0].id if pick else None
|
|
|
|
|
|
def update_ranges(folder, end, tomorrow):
|
|
"""Without --from: (start, end, replace) to pump, from the days of displacements already pumped. The missing days
|
|
between the first and the last (ranges pumped separately, failed day) are filled without touching the others; the
|
|
last day is pumped again (pumped during the day, it was incomplete) up to now; a station with no pumped
|
|
displacements gets the last 30 days."""
|
|
days = {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date()
|
|
for p in (folder / "displacement").glob(f"{folder.name}_advanced_*.csv")}
|
|
if not days:
|
|
return [(tomorrow - timedelta(days=NEW_STATION_DAYS + 1), end, True)]
|
|
# a failed metrics day has no file at all (one query for all of them): it is a gap too; a station without any
|
|
# metric never has one, its days are not gaps
|
|
metric_days = {datetime.strptime(p.name.split(" to ")[0][-16:-6], "%Y_%m_%d").date()
|
|
for p in (folder / "metrics").glob(f"{folder.name}_*_00_00 to *.csv")}
|
|
first, last = min(days), max(days)
|
|
gaps = [first + timedelta(days=k) for k in range((last - first).days)
|
|
if first + timedelta(days=k) not in days or (metric_days and first + timedelta(days=k) not in metric_days)]
|
|
return ([(gaps[0], last, False)] if gaps else []) + [(last, end, True)]
|
|
|
|
|
|
def pump_metrics(conn, system_id, station, start, end, out, replace):
|
|
"""Like the download of the System page (Dashboard/System.py, get_metrics(..., bucketing="1 hour")): per hour,
|
|
mean * transform_factor + transform_offset, or the list of values of a text metric; the labelled series of one
|
|
metric merged; an empty row for an hour without measurement. get_metrics counts in hours before now: its query is
|
|
redone here on a fixed day, one query per day for all the metrics."""
|
|
names = sorted(conn.execute(text("SELECT DISTINCT dt.name FROM datatype dt JOIN metric_series se "
|
|
"ON se.datatype_id = dt.id WHERE se.system_id = :s"), {"s": system_id}).scalars())
|
|
conn.rollback()
|
|
for k in range((end - start).days):
|
|
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
|
|
paths = {n: out / station / "metrics"
|
|
/ f"{station}_{n.replace(os.sep, '-')}_{a:%Y_%m_%d}_00_00 to {a:%Y_%m_%d}_23_00.csv" for n in names}
|
|
if not replace and all(p.exists() for p in paths.values()):
|
|
continue
|
|
t0 = time.time()
|
|
try:
|
|
# date_trunc('hour'): same bucketing as time_bucket('1 hour')
|
|
df = query(conn, """
|
|
SELECT date_trunc('hour', ms.time_utc) AS hour, dt.name AS name,
|
|
avg(ms.value_num) * any_value(dt.transform_factor) + any_value(dt.transform_offset) AS num,
|
|
jsonb_agg(ms.value_text) FILTER (WHERE ms.value_text IS NOT NULL) AS txt
|
|
FROM metric_sample ms
|
|
JOIN metric_series se ON ms.series_id = se.id
|
|
JOIN datatype dt ON se.datatype_id = dt.id
|
|
WHERE se.system_id = :s AND ms.time_utc >= :a AND ms.time_utc < :b
|
|
GROUP BY 1, dt.name""", s=system_id, a=a, b=a + timedelta(days=1))
|
|
except Exception as e: # ponytail: like the displacements, a failed day is skipped, a new run picks it up
|
|
print(f"{station} {a:%Y-%m-%d} metrics: failed ({root_cause(e)}), run again")
|
|
continue
|
|
finally:
|
|
conn.rollback()
|
|
df["hour"] = pd.to_datetime(df["hour"], utc=True)
|
|
hours = pd.to_datetime(pd.date_range(a, periods=24, freq="h"), utc=True) # same time zone as df, for reindex
|
|
# current day: the completed hours only, otherwise the hours to come would read as a gap
|
|
hours = hours[hours + pd.Timedelta(hours=1) <= pd.Timestamp.now(tz=UTC)]
|
|
for n, path in paths.items():
|
|
d = df[df["name"] == n].set_index("hour")
|
|
col = d["num"] if d["num"].notna().any() else d["txt"] # text metric: the list of the hour
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
# CSV of the System page: ',time_utc,metric', the index as first column; an empty day is written too
|
|
pd.DataFrame({"time_utc": hours, "metric": col.reindex(hours).to_numpy()}).to_csv(path)
|
|
print(f"{station} {a:%Y-%m-%d} metrics: {len(names)} files, {df['name'].nunique()} with data, "
|
|
f"{time.time() - t0:.0f} s")
|
|
|
|
|
|
# --- the "advanced" export: build_advanced_frame (app/datapoints.py) ---
|
|
|
|
def raw_datapoints(conn, unit, a, b):
|
|
"""AcquisitionUnit.get_datapoints: the datapoints of the unit, one type per column, the sensor properties next to
|
|
them (prop_x / prop_y: the reference position, namesake of a type)."""
|
|
types = ",\n".join(f"max(d.value) FILTER (WHERE dt.name = '{t}') AS {t}" for t in DATATYPES)
|
|
names = {p: f"prop_{p}" if p in DATATYPES else p for p in PROPERTIES}
|
|
props = ",\n".join(f"any_value(p_{p}.value) AS {names[p]}" for p in PROPERTIES)
|
|
joins = "\n".join(f"LEFT JOIN (SELECT p.sensor_id, p.value FROM property p JOIN datatype pdt "
|
|
f"ON p.datatype_id = pdt.id WHERE pdt.name = '{p}') p_{p} ON d.sensor_id = p_{p}.sensor_id"
|
|
for p in PROPERTIES)
|
|
return query(conn, f"""
|
|
SELECT d.time_utc, d.sensor_id, any_value(d.measurement_id) AS measurement_id,
|
|
{types},
|
|
{props}
|
|
FROM datapoint d
|
|
JOIN sensor s ON d.sensor_id = s.id
|
|
JOIN datatype dt ON d.datatype_id = dt.id
|
|
{joins}
|
|
WHERE s.acquisition_unit_id = :au AND d.time_utc >= :a AND d.time_utc <= :b
|
|
GROUP BY d.time_utc, d.sensor_id
|
|
ORDER BY d.sensor_id, d.time_utc""", au=unit, a=a, b=b)
|
|
|
|
|
|
def prepare_raw(shifts):
|
|
"""prepare_datapoints_frame: sigma0 (camera rows, without sensor_name) copied onto each sensor at its time;
|
|
relative = x - prop_x; error -> error_value; the full schema."""
|
|
if shifts.empty:
|
|
return pd.DataFrame(columns=COLUMNS)
|
|
sig = shifts.loc[shifts["sigma0"].notna(), ["time_utc", "sigma0"]]
|
|
sigma0 = sig.groupby("time_utc")["sigma0"].median() if not sig.empty else None
|
|
shifts = shifts.dropna(subset=["sensor_name"]).reset_index(drop=True)
|
|
shifts["sigma0"] = shifts["time_utc"].map(sigma0) if sigma0 is not None else pd.NA
|
|
shifts["relative_x"] = shifts["x"] - pd.to_numeric(shifts["prop_x"], errors="coerce")
|
|
shifts["relative_y"] = shifts["y"] - pd.to_numeric(shifts["prop_y"], errors="coerce")
|
|
shifts = shifts.drop(columns=["prop_x", "prop_y"]).rename(columns={"error": "error_value"})
|
|
shifts["time_utc"] = pd.to_datetime(shifts["time_utc"])
|
|
shifts["sensor_name"] = shifts["sensor_name"].astype(str)
|
|
for c in NUMERIC:
|
|
if c in shifts:
|
|
shifts[c] = pd.to_numeric(shifts[c], errors="coerce")
|
|
return shifts.reindex(columns=COLUMNS)
|
|
|
|
|
|
def aggregate_states(conn, unit, a, b):
|
|
"""fetch_aggregate_datapoints_frame: the per-minute medians of the state tables (shift_sensor_1min,
|
|
shift_common_1min), only source for "burst" devices, in the schema of the export (prepare_aggregate_datapoints_frame).
|
|
In the states, x_pct holds the CORRECTED x and rx_pct the raw x."""
|
|
thr = "COALESCE((th.value #>> '{}')::double precision, CAST(:thr AS double precision))"
|
|
sensor = query(conn, f"""
|
|
SELECT ss.time_utc, COALESCE(sn.value #>> '{{}}', s.name) AS name,
|
|
approx_percentile(0.5, ss.x_pct) AS corr_x, approx_percentile(0.5, ss.y_pct) AS corr_y,
|
|
approx_percentile(0.5, ss.rx_pct) AS x, approx_percentile(0.5, ss.ry_pct) AS y,
|
|
approx_percentile(0.5, ss.error_pct) AS error, {thr} AS error_threshold,
|
|
approx_percentile(0.5, ss.reproj_err_pct) AS reproj_err,
|
|
{", ".join(f"ss.{n}" for n in INLIER_BANDS)},
|
|
{", ".join(f"approx_percentile(0.5, ss.{m}_pct) AS {m}" for m in QUALITY)}
|
|
FROM shift_sensor_1min ss
|
|
JOIN sensor s ON s.id = ss.sensor_id
|
|
LEFT JOIN (SELECT p.sensor_id, any_value(p.value) AS value FROM property p JOIN datatype dt
|
|
ON dt.id = p.datatype_id WHERE dt.name = 'sensor_name' GROUP BY p.sensor_id) sn
|
|
ON sn.sensor_id = ss.sensor_id
|
|
LEFT JOIN (SELECT p.sensor_id, any_value(p.value) AS value FROM property p JOIN datatype dt
|
|
ON dt.id = p.datatype_id WHERE dt.name = 'error_threshold' GROUP BY p.sensor_id) th
|
|
ON th.sensor_id = ss.sensor_id
|
|
WHERE s.acquisition_unit_id = :au AND ss.time_utc >= :a AND ss.time_utc <= :b
|
|
ORDER BY ss.time_utc""", au=unit, a=a, b=b, thr=DEFAULT_ERROR_THRESHOLD)
|
|
if sensor.empty:
|
|
return pd.DataFrame(columns=COLUMNS)
|
|
common = query(conn, """
|
|
SELECT sc.time_utc, approx_percentile(0.5, sc.sigma0_pct) AS sigma0 FROM shift_common_1min sc
|
|
WHERE sc.acquisition_unit_id = :au AND sc.time_utc >= :a AND sc.time_utc <= :b""", au=unit, a=a, b=b)
|
|
out = pd.DataFrame({"time_utc": pd.to_datetime(sensor["time_utc"]), "sensor_name": sensor["name"].astype(str)})
|
|
for col, src in [("error_value", "error"), ("error_threshold", "error_threshold"), ("corr_x", "corr_x"),
|
|
("corr_y", "corr_y"), ("relative_x", "corr_x"), ("relative_y", "corr_y"), ("x", "x"), ("y", "y"),
|
|
("reproj_err", "reproj_err")] + [(m, m) for m in QUALITY]:
|
|
out[col] = pd.to_numeric(sensor[src], errors="coerce")
|
|
# inlier_code_from_bands: the majority band, a tie goes to the worst; all at 0 -> NaN (no redundancy)
|
|
counts = sensor[list(INLIER_BANDS)].apply(pd.to_numeric, errors="coerce").fillna(0)
|
|
code = counts[list(INLIER_BANDS)[::-1]].idxmax(axis=1).map(INLIER_BANDS)
|
|
out["inlier"] = code.where(counts.sum(axis=1) > 0).to_numpy()
|
|
out["pattern_type"] = None
|
|
sig = common.loc[common["sigma0"].notna()]
|
|
out["sigma0"] = (out["time_utc"].map(sig.assign(time_utc=pd.to_datetime(sig["time_utc"]))
|
|
.groupby("time_utc")["sigma0"].median()) if not sig.empty else float("nan"))
|
|
return out.reindex(columns=COLUMNS)
|
|
|
|
|
|
def merge(raw, agg):
|
|
"""merge_raw_and_aggregate_frames: the raw wins; a state is kept only where its sensor has no raw row in the same
|
|
minute."""
|
|
if agg.empty:
|
|
return raw
|
|
if raw.empty:
|
|
return agg
|
|
|
|
def utc(t):
|
|
t = pd.to_datetime(t)
|
|
return t.dt.tz_localize("UTC") if t.dt.tz is None else t.dt.tz_convert("UTC")
|
|
|
|
keys = pd.MultiIndex.from_arrays([raw["sensor_name"], utc(raw["time_utc"]).dt.floor("1min")])
|
|
agg = agg.loc[~pd.MultiIndex.from_arrays([agg["sensor_name"], utc(agg["time_utc"]).dt.floor("1min")]).isin(keys)]
|
|
if agg.empty:
|
|
return raw
|
|
agg = agg.copy()
|
|
tz = pd.to_datetime(raw["time_utc"]).dt.tz
|
|
t = utc(agg["time_utc"])
|
|
agg["time_utc"] = t.dt.tz_convert(tz) if tz is not None else t.dt.tz_localize(None)
|
|
return (pd.concat([raw, agg], ignore_index=True).sort_values(["sensor_name", "time_utc"], kind="stable")
|
|
.reset_index(drop=True).reindex(columns=COLUMNS))
|
|
|
|
|
|
def advanced(conn, unit, a, b):
|
|
return merge(prepare_raw(raw_datapoints(conn, unit, a, b)), aggregate_states(conn, unit, a, b))
|
|
|
|
|
|
def camera(conn, unit, a, b):
|
|
"""Pan, tilt, scale, rotation and sigma0 of the camera, as median and bounds (median_ / min_ / max_) per minute,
|
|
hence one row per camera measurement (every 15 min at NORDEND, min = max): what the viewer plots
|
|
(fetch_cam_movements_frame), without its except, which would return a failed day as empty instead of leaving it
|
|
to run again."""
|
|
sigs = ", ".join(f"approx_percentile(0.5, sc.{s}_pct) AS median_{s}, min_val(sc.{s}_pct) AS min_{s}, "
|
|
f"max_val(sc.{s}_pct) AS max_{s}" for s in CAM_SIGNALS)
|
|
return query(conn, f"""SELECT sc.time_utc, {sigs} FROM shift_common_1min sc
|
|
WHERE sc.acquisition_unit_id = :au AND sc.time_utc >= :a AND sc.time_utc <= :b
|
|
ORDER BY sc.time_utc""", au=unit, a=a, b=b)
|
|
|
|
|
|
DAILY_FILES = (("displacement", "advanced", advanced), ("camera", "camera", camera)) # folder, file, reader
|
|
|
|
|
|
def pump_displacements(conn, unit, station, start, end, out, replace):
|
|
"""Displacements and camera movements: one file and one query per day, bounded, under the timeout (--delay)."""
|
|
for k in range((end - start).days):
|
|
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
|
|
for folder, name, reader in DAILY_FILES:
|
|
path = out / station / folder / f"{station}_{name}_{a:%Y%m%d}.csv"
|
|
if path.exists() and not replace:
|
|
continue
|
|
t0 = time.time()
|
|
try:
|
|
df = reader(conn, unit, a, a + timedelta(days=1) - timedelta(microseconds=1))
|
|
except Exception as e: # ponytail: a failed day is skipped, a new run picks it up; no retry
|
|
print(f"{station} {a:%Y-%m-%d} {name}: failed ({root_cause(e)}), run again")
|
|
continue
|
|
finally:
|
|
conn.rollback() # closes the read transaction
|
|
if "sensor_name" in df:
|
|
df = df.drop_duplicates(["sensor_name", "time_utc"])
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
df.to_csv(path, index=False) # an empty day is written too: the next run does not ask for it again
|
|
print(f"{station} {a:%Y-%m-%d} {name}: {len(df)} rows, {time.time() - t0:.0f} s -> {path.name}")
|
|
|
|
|
|
def pump_images(conn, s3, unit, station, start, end, folder, every, out):
|
|
mt = conn.execute(text("SELECT id, extension FROM measurement_type WHERE name = :n"), {"n": TYPES[folder]}).first()
|
|
if mt is None:
|
|
print(f"{station}: unknown type {TYPES[folder]}")
|
|
return
|
|
client, bucket_name = s3
|
|
for k in range((end - start).days):
|
|
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
|
|
measurements = conn.execute(text(
|
|
"SELECT id, time_utc, measurement_uuid FROM measurement WHERE acquisition_unit_id = :au "
|
|
"AND measurement_type_id = :mt AND time_utc >= :a AND time_utc < :b ORDER BY time_utc"),
|
|
{"au": unit, "mt": mt.id, "a": a, "b": a + timedelta(days=1)}).all()
|
|
conn.rollback()
|
|
seen, n, nbytes, no_uuid, unreturned = set(), 0, 0, 0, 0
|
|
for m in measurements:
|
|
# empty measurement_uuid: the image never reached S3 (unfinished multipart upload); skipped, the slot
|
|
# takes the next image if there is one
|
|
if m.measurement_uuid is None:
|
|
no_uuid += 1
|
|
continue
|
|
t = m.time_utc.astimezone(UTC)
|
|
bucket = (t.hour * 60 + t.minute) // every if every else m.id # one image per slot of `every` min
|
|
path = out / station / folder / f"{t:%Y%m%dT%H%M}Z_{m.id}.{mt.extension}"
|
|
if bucket in seen:
|
|
continue
|
|
seen.add(bucket) # even if already there: otherwise the next run would take the next image of the slot
|
|
if path.exists():
|
|
continue
|
|
try:
|
|
data = client.get_object(Bucket=bucket_name, Key=m.measurement_uuid)["Body"].read()
|
|
except Exception: # like get_measurement_payload: S3 not answering, the image is counted as missing
|
|
data = None
|
|
if data:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(data)
|
|
n, nbytes = n + 1, nbytes + len(data)
|
|
else:
|
|
unreturned += 1
|
|
# the detail says why a day gives 0: nothing in the database, never reached S3, or S3 not returning them
|
|
print(f"{station} {a:%Y-%m-%d} {folder}: {n} new images, {nbytes / 1e6:.0f} MB "
|
|
f"({len(measurements)} in database, {no_uuid} never reached S3, {unreturned} not returned by S3)")
|
|
time.sleep(0.5)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("stations", nargs="*", help="serial number of the system, as in the URL of the System page; "
|
|
"none: those of stations.txt")
|
|
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD; without: the last pumped day of each "
|
|
"station, pumped again, or the last 30 days")
|
|
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD; without: up to now")
|
|
p.add_argument("--images", action="store_true", help="also the OPTICAL_SHIFT-IRON images")
|
|
p.add_argument("--diff", action="store_true", help="also the OPTICAL_SHIFT-DIFF images")
|
|
p.add_argument("--every", type=int, default=60, help="one image per slot of N minutes, 0 = all (default 60)")
|
|
p.add_argument("--delay", type=int, default=60, help="maximum duration of a query, in s (default 60: database "
|
|
"shared with production); longer only for a day that failed on a timeout")
|
|
p.add_argument("--out", type=Path, default=Path(__file__).resolve().parent.parent / "data")
|
|
a = p.parse_args()
|
|
tomorrow = datetime.now(UTC).date() + timedelta(days=1)
|
|
start = datetime.strptime(a.start, "%Y-%m-%d").date() if a.start else None
|
|
end = min(datetime.strptime(a.end, "%Y-%m-%d").date(), tomorrow) if a.end else tomorrow # no day to come
|
|
stations = a.stations or STATIONS.read_text().split()
|
|
|
|
missing = [n for n in DB_NAMES if not os.environ.get(n)]
|
|
if missing:
|
|
sys.exit(f"Missing variables (names only): {', '.join(missing)}")
|
|
url = URL.create("postgresql+psycopg2", username=os.environ["AIMSIGHT_DATABASE_USER"],
|
|
password=os.environ["AIMSIGHT_DATABASE_PSWD"], host=os.environ["AIMSIGHT_DATABASE_TURL"],
|
|
port=int(os.environ.get("AIMSIGHT_DATABASE_PORT", "5432")),
|
|
database=os.environ["AIMSIGHT_DATABASE_NAME"])
|
|
engine = create_engine(url, connect_args={
|
|
"options": f"-c timezone=utc -c default_transaction_read_only=on -c statement_timeout={a.delay}s"})
|
|
s3 = None
|
|
if a.images or a.diff:
|
|
import boto3 # like the app's S3client (Garage)
|
|
s3 = (boto3.client("s3", endpoint_url=os.environ.get("AIMSIGHT_S3_URL", "http://s3:3900"),
|
|
aws_access_key_id=os.environ.get("AIMSIGHT_S3_ACCESS_KEY", ""),
|
|
aws_secret_access_key=os.environ.get("AIMSIGHT_S3_SECRET_KEY", ""),
|
|
region_name="garage", use_ssl=False),
|
|
os.environ.get("AIMSIGHT_S3_BUCKET", "measurements-assets"))
|
|
try:
|
|
with engine.connect() as conn:
|
|
for station in stations:
|
|
system_id = conn.execute(text("SELECT id FROM system WHERE serial_number = :s"), {"s": station}).scalar()
|
|
if system_id is None:
|
|
print(f"{station}: unknown system")
|
|
continue
|
|
ranges = [(start, end, False)] if start else update_ranges(a.out / station, end, tomorrow)
|
|
unit = acquisition_unit(conn, system_id)
|
|
conn.rollback()
|
|
for range_start, range_end, replace in ranges:
|
|
print(f"{station}: from {range_start} to {range_end - timedelta(days=1)} inclusive"
|
|
+ ("" if replace else ", missing days only"))
|
|
pump_metrics(conn, system_id, station, range_start, range_end, a.out, replace)
|
|
if unit is None:
|
|
print(f"{station}: no recognised acquisition unit, no displacements, camera or images")
|
|
continue
|
|
pump_displacements(conn, unit, station, range_start, range_end, a.out, replace)
|
|
for flag, folder in ((a.images, "images"), (a.diff, "images_diff")):
|
|
if flag:
|
|
pump_images(conn, s3, unit, station, range_start, range_end, folder, a.every, a.out)
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|