English: pump.py and pump.sh (ex pomper), lenzspitze_absolute.py, options --from/--to/--delay, out/pump.log; .env loaded with +u (pump.sh -h works)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 12:40:03 +01:00
1 parent 260cd5e522
commit 5ed9f5a97a
9 files changed
+531 -530

No files matched your search

+409
View File
@@ -0,0 +1,409 @@
"""Pumps the system metrics, the displacements ("advanced" export) and the images of stations from the AIMsight
platform, in the format read by the app, the reading tools and the detectors. READ-ONLY. Tried on 2026-10-26
(NORDEND, 2 days); standalone since 2026-11-12.
Run through extraction/pump.sh, under WSL, from anywhere (see README.md):
extraction/pump.sh
extraction/pump.sh --from 2026-08-01 --to 2026-08-08 --images WEISSHORN
Without a station: those of `stations.txt`. Without `--from`: each station fills its missing days (displacements or
metrics) before the last one pumped, then restarts from that last day, pumped again (it was perhaps incomplete), or
from the last 30 days if it has none; without `--to`: up to now.
Output (default: the repository's `data/`, next to this folder):
<STATION>/metrics/<STATION>_<metric>_<YYYY_MM_DD>_00_00 to <YYYY_MM_DD>_23_00.csv system metrics, per day
<STATION>/displacement/<STATION>_advanced_<YYYYMMDD>.csv displacements, one file per day
<STATION>/camera/<STATION>_camera_<YYYYMMDD>.csv camera movements, per day
<STATION>/images/<YYYYMMDD>T<HHMM>Z_<id>.<ext> OPTICAL_SHIFT-IRON images (--images)
<STATION>/images_diff/… differential images (--diff)
Resumable: with `--from`, a file already written is not read again, whatever the dates asked; without `--from`, the
days of the range are replaced (images excepted). Days in UTC, `--to` excluded; the current day is pumped as it is,
its metrics up to the last completed hour. To redo a day, delete its files.
No dependency on the app (~/monitoring-dashboard): its reads are copied here, in SQL, as they are (branch `explo`,
commit 3dce73d of 2026-11-12): the query of System.get_metrics (the download of the System page);
build_advanced_frame (app/datapoints.py, what the Advanced tab and the parquet export plot), with
AcquisitionUnit.get_datapoints and AggregateEngine.get_sensor_aggregate / get_common_aggregate; the camera movements
of fetch_cam_movements_frame (AggregateEngine.get_cam_scalar_aggregate, what the viewer plots);
AcquisitionUnit.measurements and get_measurement_payload (S3). If the app changes its reads, this file does not follow.
Read-only connection (`default_transaction_read_only`), 60 s timeout per query (`--delay`). Variables read (names
only, none is printed): AIMSIGHT_DATABASE_USER / _PSWD / _TURL / _NAME / _PORT, and for the images
AIMSIGHT_S3_URL / _ACCESS_KEY / _SECRET_KEY / _BUCKET.
"""
import argparse
import os
import sys
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pandas as pd
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
UTC = timezone.utc
STATIONS = Path(__file__).resolve().parent / "stations.txt" # the tracked stations, one per line
NEW_STATION_DAYS = 30 # without --from, a station with no pumped displacements: the last 30 days
DB_NAMES = ["AIMSIGHT_DATABASE_USER", "AIMSIGHT_DATABASE_PSWD", "AIMSIGHT_DATABASE_TURL", "AIMSIGHT_DATABASE_NAME"]
TYPES = {"images": "OPTICAL_SHIFT-IRON", "images_diff": "OPTICAL_SHIFT-DIFF"} # folder -> measurement type
# --- constants of the app (app/datapoints.py, app/aggregates.py) ---
QUALITY = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"]
CAM_SIGNALS = ["pan", "tilt", "scale", "rotation", "sigma0"]
DEFAULT_ERROR_THRESHOLD = 0.7
DATATYPES = ["x", "y", "error", "corr_x", "corr_y", "reproj_err", "inlier", "sigma0"]
PROPERTIES = ["x", "y", "pattern_type", "sensor_name", "error_threshold"]
COLUMNS = ["time_utc", "sensor_name", "x", "y", "error_value", "error_threshold", "relative_x", "relative_y", "corr_x",
"corr_y", "reproj_err", "inlier", "sigma0", "pattern_type"] + QUALITY
NUMERIC = ["x", "y", "error_value", "error_threshold", "relative_x", "relative_y", "corr_x", "corr_y", "reproj_err",
"inlier", "sigma0"] + QUALITY
# inlier bands of shift_sensor_1min -> code; the last two NAMES are wrong (-1 rejected, -2 interpolated), the codes
# are right; ordered from best to worst
INLIER_BANDS = {"inlier_in_fit": 1.0, "inlier_out_fit": 0.0, "inlier_no_redundancy": -1.0, "inlier_rejected": -2.0}
def root_cause(e):
"""The original error (the database's): pandas and SQLAlchemy wrap it, the SQL query first."""
while e.__cause__ is not None:
e = e.__cause__
return f"{type(e).__name__}: {(str(e).splitlines() or [''])[0][:300]}"
def query(conn, sql, **params):
return pd.read_sql(text(sql), conn, params=params)
def acquisition_unit(conn, system_id):
"""Same choice as Device.get_acquisition_unit (app/base/models.py:268): the only unit, otherwise MER* / ME2S*."""
units = conn.execute(text("SELECT id, product_number FROM acquisition_unit WHERE system_id = :s"),
{"s": system_id}).all()
pick = units if len(units) == 1 else [u for u in units if u.product_number.startswith(("MER", "ME2S"))]
return pick[0].id if pick else None
def update_ranges(folder, end, tomorrow):
"""Without --from: (start, end, replace) to pump, from the days of displacements already pumped. The missing days
between the first and the last (ranges pumped separately, failed day) are filled without touching the others; the
last day is pumped again (pumped during the day, it was incomplete) up to now; a station with no pumped
displacements gets the last 30 days."""
days = {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date()
for p in (folder / "displacement").glob(f"{folder.name}_advanced_*.csv")}
if not days:
return [(tomorrow - timedelta(days=NEW_STATION_DAYS + 1), end, True)]
# a failed metrics day has no file at all (one query for all of them): it is a gap too; a station without any
# metric never has one, its days are not gaps
metric_days = {datetime.strptime(p.name.split(" to ")[0][-16:-6], "%Y_%m_%d").date()
for p in (folder / "metrics").glob(f"{folder.name}_*_00_00 to *.csv")}
first, last = min(days), max(days)
gaps = [first + timedelta(days=k) for k in range((last - first).days)
if first + timedelta(days=k) not in days or (metric_days and first + timedelta(days=k) not in metric_days)]
return ([(gaps[0], last, False)] if gaps else []) + [(last, end, True)]
def pump_metrics(conn, system_id, station, start, end, out, replace):
"""Like the download of the System page (Dashboard/System.py, get_metrics(..., bucketing="1 hour")): per hour,
mean * transform_factor + transform_offset, or the list of values of a text metric; the labelled series of one
metric merged; an empty row for an hour without measurement. get_metrics counts in hours before now: its query is
redone here on a fixed day, one query per day for all the metrics."""
names = sorted(conn.execute(text("SELECT DISTINCT dt.name FROM datatype dt JOIN metric_series se "
"ON se.datatype_id = dt.id WHERE se.system_id = :s"), {"s": system_id}).scalars())
conn.rollback()
for k in range((end - start).days):
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
paths = {n: out / station / "metrics"
/ f"{station}_{n.replace(os.sep, '-')}_{a:%Y_%m_%d}_00_00 to {a:%Y_%m_%d}_23_00.csv" for n in names}
if not replace and all(p.exists() for p in paths.values()):
continue
t0 = time.time()
try:
# date_trunc('hour'): same bucketing as time_bucket('1 hour')
df = query(conn, """
SELECT date_trunc('hour', ms.time_utc) AS hour, dt.name AS name,
avg(ms.value_num) * any_value(dt.transform_factor) + any_value(dt.transform_offset) AS num,
jsonb_agg(ms.value_text) FILTER (WHERE ms.value_text IS NOT NULL) AS txt
FROM metric_sample ms
JOIN metric_series se ON ms.series_id = se.id
JOIN datatype dt ON se.datatype_id = dt.id
WHERE se.system_id = :s AND ms.time_utc >= :a AND ms.time_utc < :b
GROUP BY 1, dt.name""", s=system_id, a=a, b=a + timedelta(days=1))
except Exception as e: # ponytail: like the displacements, a failed day is skipped, a new run picks it up
print(f"{station} {a:%Y-%m-%d} metrics: failed ({root_cause(e)}), run again")
continue
finally:
conn.rollback()
df["hour"] = pd.to_datetime(df["hour"], utc=True)
hours = pd.to_datetime(pd.date_range(a, periods=24, freq="h"), utc=True) # same time zone as df, for reindex
# current day: the completed hours only, otherwise the hours to come would read as a gap
hours = hours[hours + pd.Timedelta(hours=1) <= pd.Timestamp.now(tz=UTC)]
for n, path in paths.items():
d = df[df["name"] == n].set_index("hour")
col = d["num"] if d["num"].notna().any() else d["txt"] # text metric: the list of the hour
path.parent.mkdir(parents=True, exist_ok=True)
# CSV of the System page: ',time_utc,metric', the index as first column; an empty day is written too
pd.DataFrame({"time_utc": hours, "metric": col.reindex(hours).to_numpy()}).to_csv(path)
print(f"{station} {a:%Y-%m-%d} metrics: {len(names)} files, {df['name'].nunique()} with data, "
f"{time.time() - t0:.0f} s")
# --- the "advanced" export: build_advanced_frame (app/datapoints.py) ---
def raw_datapoints(conn, unit, a, b):
"""AcquisitionUnit.get_datapoints: the datapoints of the unit, one type per column, the sensor properties next to
them (prop_x / prop_y: the reference position, namesake of a type)."""
types = ",\n".join(f"max(d.value) FILTER (WHERE dt.name = '{t}') AS {t}" for t in DATATYPES)
names = {p: f"prop_{p}" if p in DATATYPES else p for p in PROPERTIES}
props = ",\n".join(f"any_value(p_{p}.value) AS {names[p]}" for p in PROPERTIES)
joins = "\n".join(f"LEFT JOIN (SELECT p.sensor_id, p.value FROM property p JOIN datatype pdt "
f"ON p.datatype_id = pdt.id WHERE pdt.name = '{p}') p_{p} ON d.sensor_id = p_{p}.sensor_id"
for p in PROPERTIES)
return query(conn, f"""
SELECT d.time_utc, d.sensor_id, any_value(d.measurement_id) AS measurement_id,
{types},
{props}
FROM datapoint d
JOIN sensor s ON d.sensor_id = s.id
JOIN datatype dt ON d.datatype_id = dt.id
{joins}
WHERE s.acquisition_unit_id = :au AND d.time_utc >= :a AND d.time_utc <= :b
GROUP BY d.time_utc, d.sensor_id
ORDER BY d.sensor_id, d.time_utc""", au=unit, a=a, b=b)
def prepare_raw(shifts):
"""prepare_datapoints_frame: sigma0 (camera rows, without sensor_name) copied onto each sensor at its time;
relative = x - prop_x; error -> error_value; the full schema."""
if shifts.empty:
return pd.DataFrame(columns=COLUMNS)
sig = shifts.loc[shifts["sigma0"].notna(), ["time_utc", "sigma0"]]
sigma0 = sig.groupby("time_utc")["sigma0"].median() if not sig.empty else None
shifts = shifts.dropna(subset=["sensor_name"]).reset_index(drop=True)
shifts["sigma0"] = shifts["time_utc"].map(sigma0) if sigma0 is not None else pd.NA
shifts["relative_x"] = shifts["x"] - pd.to_numeric(shifts["prop_x"], errors="coerce")
shifts["relative_y"] = shifts["y"] - pd.to_numeric(shifts["prop_y"], errors="coerce")
shifts = shifts.drop(columns=["prop_x", "prop_y"]).rename(columns={"error": "error_value"})
shifts["time_utc"] = pd.to_datetime(shifts["time_utc"])
shifts["sensor_name"] = shifts["sensor_name"].astype(str)
for c in NUMERIC:
if c in shifts:
shifts[c] = pd.to_numeric(shifts[c], errors="coerce")
return shifts.reindex(columns=COLUMNS)
def aggregate_states(conn, unit, a, b):
"""fetch_aggregate_datapoints_frame: the per-minute medians of the state tables (shift_sensor_1min,
shift_common_1min), only source for "burst" devices, in the schema of the export (prepare_aggregate_datapoints_frame).
In the states, x_pct holds the CORRECTED x and rx_pct the raw x."""
thr = "COALESCE((th.value #>> '{}')::double precision, CAST(:thr AS double precision))"
sensor = query(conn, f"""
SELECT ss.time_utc, COALESCE(sn.value #>> '{{}}', s.name) AS name,
approx_percentile(0.5, ss.x_pct) AS corr_x, approx_percentile(0.5, ss.y_pct) AS corr_y,
approx_percentile(0.5, ss.rx_pct) AS x, approx_percentile(0.5, ss.ry_pct) AS y,
approx_percentile(0.5, ss.error_pct) AS error, {thr} AS error_threshold,
approx_percentile(0.5, ss.reproj_err_pct) AS reproj_err,
{", ".join(f"ss.{n}" for n in INLIER_BANDS)},
{", ".join(f"approx_percentile(0.5, ss.{m}_pct) AS {m}" for m in QUALITY)}
FROM shift_sensor_1min ss
JOIN sensor s ON s.id = ss.sensor_id
LEFT JOIN (SELECT p.sensor_id, any_value(p.value) AS value FROM property p JOIN datatype dt
ON dt.id = p.datatype_id WHERE dt.name = 'sensor_name' GROUP BY p.sensor_id) sn
ON sn.sensor_id = ss.sensor_id
LEFT JOIN (SELECT p.sensor_id, any_value(p.value) AS value FROM property p JOIN datatype dt
ON dt.id = p.datatype_id WHERE dt.name = 'error_threshold' GROUP BY p.sensor_id) th
ON th.sensor_id = ss.sensor_id
WHERE s.acquisition_unit_id = :au AND ss.time_utc >= :a AND ss.time_utc <= :b
ORDER BY ss.time_utc""", au=unit, a=a, b=b, thr=DEFAULT_ERROR_THRESHOLD)
if sensor.empty:
return pd.DataFrame(columns=COLUMNS)
common = query(conn, """
SELECT sc.time_utc, approx_percentile(0.5, sc.sigma0_pct) AS sigma0 FROM shift_common_1min sc
WHERE sc.acquisition_unit_id = :au AND sc.time_utc >= :a AND sc.time_utc <= :b""", au=unit, a=a, b=b)
out = pd.DataFrame({"time_utc": pd.to_datetime(sensor["time_utc"]), "sensor_name": sensor["name"].astype(str)})
for col, src in [("error_value", "error"), ("error_threshold", "error_threshold"), ("corr_x", "corr_x"),
("corr_y", "corr_y"), ("relative_x", "corr_x"), ("relative_y", "corr_y"), ("x", "x"), ("y", "y"),
("reproj_err", "reproj_err")] + [(m, m) for m in QUALITY]:
out[col] = pd.to_numeric(sensor[src], errors="coerce")
# inlier_code_from_bands: the majority band, a tie goes to the worst; all at 0 -> NaN (no redundancy)
counts = sensor[list(INLIER_BANDS)].apply(pd.to_numeric, errors="coerce").fillna(0)
code = counts[list(INLIER_BANDS)[::-1]].idxmax(axis=1).map(INLIER_BANDS)
out["inlier"] = code.where(counts.sum(axis=1) > 0).to_numpy()
out["pattern_type"] = None
sig = common.loc[common["sigma0"].notna()]
out["sigma0"] = (out["time_utc"].map(sig.assign(time_utc=pd.to_datetime(sig["time_utc"]))
.groupby("time_utc")["sigma0"].median()) if not sig.empty else float("nan"))
return out.reindex(columns=COLUMNS)
def merge(raw, agg):
"""merge_raw_and_aggregate_frames: the raw wins; a state is kept only where its sensor has no raw row in the same
minute."""
if agg.empty:
return raw
if raw.empty:
return agg
def utc(t):
t = pd.to_datetime(t)
return t.dt.tz_localize("UTC") if t.dt.tz is None else t.dt.tz_convert("UTC")
keys = pd.MultiIndex.from_arrays([raw["sensor_name"], utc(raw["time_utc"]).dt.floor("1min")])
agg = agg.loc[~pd.MultiIndex.from_arrays([agg["sensor_name"], utc(agg["time_utc"]).dt.floor("1min")]).isin(keys)]
if agg.empty:
return raw
agg = agg.copy()
tz = pd.to_datetime(raw["time_utc"]).dt.tz
t = utc(agg["time_utc"])
agg["time_utc"] = t.dt.tz_convert(tz) if tz is not None else t.dt.tz_localize(None)
return (pd.concat([raw, agg], ignore_index=True).sort_values(["sensor_name", "time_utc"], kind="stable")
.reset_index(drop=True).reindex(columns=COLUMNS))
def advanced(conn, unit, a, b):
return merge(prepare_raw(raw_datapoints(conn, unit, a, b)), aggregate_states(conn, unit, a, b))
def camera(conn, unit, a, b):
"""Pan, tilt, scale, rotation and sigma0 of the camera, as median and bounds (median_ / min_ / max_) per minute,
hence one row per camera measurement (every 15 min at NORDEND, min = max): what the viewer plots
(fetch_cam_movements_frame), without its except, which would return a failed day as empty instead of leaving it
to run again."""
sigs = ", ".join(f"approx_percentile(0.5, sc.{s}_pct) AS median_{s}, min_val(sc.{s}_pct) AS min_{s}, "
f"max_val(sc.{s}_pct) AS max_{s}" for s in CAM_SIGNALS)
return query(conn, f"""SELECT sc.time_utc, {sigs} FROM shift_common_1min sc
WHERE sc.acquisition_unit_id = :au AND sc.time_utc >= :a AND sc.time_utc <= :b
ORDER BY sc.time_utc""", au=unit, a=a, b=b)
DAILY_FILES = (("displacement", "advanced", advanced), ("camera", "camera", camera)) # folder, file, reader
def pump_displacements(conn, unit, station, start, end, out, replace):
"""Displacements and camera movements: one file and one query per day, bounded, under the timeout (--delay)."""
for k in range((end - start).days):
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
for folder, name, reader in DAILY_FILES:
path = out / station / folder / f"{station}_{name}_{a:%Y%m%d}.csv"
if path.exists() and not replace:
continue
t0 = time.time()
try:
df = reader(conn, unit, a, a + timedelta(days=1) - timedelta(microseconds=1))
except Exception as e: # ponytail: a failed day is skipped, a new run picks it up; no retry
print(f"{station} {a:%Y-%m-%d} {name}: failed ({root_cause(e)}), run again")
continue
finally:
conn.rollback() # closes the read transaction
if "sensor_name" in df:
df = df.drop_duplicates(["sensor_name", "time_utc"])
path.parent.mkdir(parents=True, exist_ok=True)
df.to_csv(path, index=False) # an empty day is written too: the next run does not ask for it again
print(f"{station} {a:%Y-%m-%d} {name}: {len(df)} rows, {time.time() - t0:.0f} s -> {path.name}")
def pump_images(conn, s3, unit, station, start, end, folder, every, out):
mt = conn.execute(text("SELECT id, extension FROM measurement_type WHERE name = :n"), {"n": TYPES[folder]}).first()
if mt is None:
print(f"{station}: unknown type {TYPES[folder]}")
return
client, bucket_name = s3
for k in range((end - start).days):
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
measurements = conn.execute(text(
"SELECT id, time_utc, measurement_uuid FROM measurement WHERE acquisition_unit_id = :au "
"AND measurement_type_id = :mt AND time_utc >= :a AND time_utc < :b ORDER BY time_utc"),
{"au": unit, "mt": mt.id, "a": a, "b": a + timedelta(days=1)}).all()
conn.rollback()
seen, n, nbytes, no_uuid, unreturned = set(), 0, 0, 0, 0
for m in measurements:
# empty measurement_uuid: the image never reached S3 (unfinished multipart upload); skipped, the slot
# takes the next image if there is one
if m.measurement_uuid is None:
no_uuid += 1
continue
t = m.time_utc.astimezone(UTC)
bucket = (t.hour * 60 + t.minute) // every if every else m.id # one image per slot of `every` min
path = out / station / folder / f"{t:%Y%m%dT%H%M}Z_{m.id}.{mt.extension}"
if bucket in seen:
continue
seen.add(bucket) # even if already there: otherwise the next run would take the next image of the slot
if path.exists():
continue
try:
data = client.get_object(Bucket=bucket_name, Key=m.measurement_uuid)["Body"].read()
except Exception: # like get_measurement_payload: S3 not answering, the image is counted as missing
data = None
if data:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
n, nbytes = n + 1, nbytes + len(data)
else:
unreturned += 1
# the detail says why a day gives 0: nothing in the database, never reached S3, or S3 not returning them
print(f"{station} {a:%Y-%m-%d} {folder}: {n} new images, {nbytes / 1e6:.0f} MB "
f"({len(measurements)} in database, {no_uuid} never reached S3, {unreturned} not returned by S3)")
time.sleep(0.5)
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("stations", nargs="*", help="serial number of the system, as in the URL of the System page; "
"none: those of stations.txt")
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD; without: the last pumped day of each "
"station, pumped again, or the last 30 days")
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD; without: up to now")
p.add_argument("--images", action="store_true", help="also the OPTICAL_SHIFT-IRON images")
p.add_argument("--diff", action="store_true", help="also the OPTICAL_SHIFT-DIFF images")
p.add_argument("--every", type=int, default=60, help="one image per slot of N minutes, 0 = all (default 60)")
p.add_argument("--delay", type=int, default=60, help="maximum duration of a query, in s (default 60: database "
"shared with production); longer only for a day that failed on a timeout")
p.add_argument("--out", type=Path, default=Path(__file__).resolve().parent.parent / "data")
a = p.parse_args()
tomorrow = datetime.now(UTC).date() + timedelta(days=1)
start = datetime.strptime(a.start, "%Y-%m-%d").date() if a.start else None
end = min(datetime.strptime(a.end, "%Y-%m-%d").date(), tomorrow) if a.end else tomorrow # no day to come
stations = a.stations or STATIONS.read_text().split()
missing = [n for n in DB_NAMES if not os.environ.get(n)]
if missing:
sys.exit(f"Missing variables (names only): {', '.join(missing)}")
url = URL.create("postgresql+psycopg2", username=os.environ["AIMSIGHT_DATABASE_USER"],
password=os.environ["AIMSIGHT_DATABASE_PSWD"], host=os.environ["AIMSIGHT_DATABASE_TURL"],
port=int(os.environ.get("AIMSIGHT_DATABASE_PORT", "5432")),
database=os.environ["AIMSIGHT_DATABASE_NAME"])
engine = create_engine(url, connect_args={
"options": f"-c timezone=utc -c default_transaction_read_only=on -c statement_timeout={a.delay}s"})
s3 = None
if a.images or a.diff:
import boto3 # like the app's S3client (Garage)
s3 = (boto3.client("s3", endpoint_url=os.environ.get("AIMSIGHT_S3_URL", "http://s3:3900"),
aws_access_key_id=os.environ.get("AIMSIGHT_S3_ACCESS_KEY", ""),
aws_secret_access_key=os.environ.get("AIMSIGHT_S3_SECRET_KEY", ""),
region_name="garage", use_ssl=False),
os.environ.get("AIMSIGHT_S3_BUCKET", "measurements-assets"))
try:
with engine.connect() as conn:
for station in stations:
system_id = conn.execute(text("SELECT id FROM system WHERE serial_number = :s"), {"s": station}).scalar()
if system_id is None:
print(f"{station}: unknown system")
continue
ranges = [(start, end, False)] if start else update_ranges(a.out / station, end, tomorrow)
unit = acquisition_unit(conn, system_id)
conn.rollback()
for range_start, range_end, replace in ranges:
print(f"{station}: from {range_start} to {range_end - timedelta(days=1)} inclusive"
+ ("" if replace else ", missing days only"))
pump_metrics(conn, system_id, station, range_start, range_end, a.out, replace)
if unit is None:
print(f"{station}: no recognised acquisition unit, no displacements, camera or images")
continue
pump_displacements(conn, unit, station, range_start, range_end, a.out, replace)
for flag, folder in ((a.images, "images"), (a.diff, "images_diff")):
if flag:
pump_images(conn, s3, unit, station, range_start, range_end, folder, a.every, a.out)
finally:
engine.dispose()
if __name__ == "__main__":
main()