Pumping: the weather of the stations with a position written into data/<SERIAL>/weather/ (extraction/weather.py, after pump.py); the mm (corr_x_mm, corr_y_mm by the sensor_id of each row) and targets.csv with the references (pattern_type, distance_mm, mm_per_px); --replace pumps a range again

pump.py --check passes; the mm and targets.csv parts are not yet run against the database (no VPN).
stations.txt: NIEDERHORN and NORDEND without a position (outside Switzerland).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-18 07:17:27 +01:00
1 parent 9adc38f9b9
commit ea0228a73a
8 files changed
+205 -22

No files matched your search

+100 -12
View File
@@ -14,6 +14,9 @@ from the last 30 days if it has none; without `--to`: up to now.
Output (default: the repository's `data/`, next to this folder):
<STATION>/metrics/<STATION>_<metric>_<YYYY_MM_DD>_00_00 to <YYYY_MM_DD>_23_00.csv system metrics, per day
<STATION>/displacement/<STATION>_advanced_<YYYYMMDD>.csv displacements, one file per day
+ corr_x_mm, corr_y_mm: corr * mm_per_px of the sensor that measured (references included)
<STATION>/targets.csv the targets and references measured: sensor_name, pattern_type,
distance_mm, mm_per_px (those of the last day pumped)
<STATION>/camera/<STATION>_camera_<YYYYMMDD>.csv camera movements, per day
<STATION>/images/<YYYYMMDD>T<HHMM>Z_<id>.<ext> OPTICAL_SHIFT-IRON images (--images)
<STATION>/images_diff/… differential images (--diff)
@@ -175,7 +178,7 @@ def prepare_raw(shifts):
"""prepare_datapoints_frame: sigma0 (camera rows, without sensor_name) copied onto each sensor at its time;
relative = x - prop_x; error -> error_value; the full schema."""
if shifts.empty:
return pd.DataFrame(columns=COLUMNS)
return pd.DataFrame(columns=COLUMNS + ["sensor_id"])
sig = shifts.loc[shifts["sigma0"].notna(), ["time_utc", "sigma0"]]
sigma0 = sig.groupby("time_utc")["sigma0"].median() if not sig.empty else None
shifts = shifts.dropna(subset=["sensor_name"]).reset_index(drop=True)
@@ -188,7 +191,7 @@ def prepare_raw(shifts):
for c in NUMERIC:
if c in shifts:
shifts[c] = pd.to_numeric(shifts[c], errors="coerce")
return shifts.reindex(columns=COLUMNS)
return shifts.reindex(columns=COLUMNS + ["sensor_id"])
def aggregate_states(conn, unit, a, b):
@@ -197,7 +200,7 @@ def aggregate_states(conn, unit, a, b):
In the states, x_pct holds the CORRECTED x and rx_pct the raw x."""
thr = "COALESCE((th.value #>> '{}')::double precision, CAST(:thr AS double precision))"
sensor = query(conn, f"""
SELECT ss.time_utc, COALESCE(sn.value #>> '{{}}', s.name) AS name,
SELECT ss.time_utc, ss.sensor_id, COALESCE(sn.value #>> '{{}}', s.name) AS name,
approx_percentile(0.5, ss.x_pct) AS corr_x, approx_percentile(0.5, ss.y_pct) AS corr_y,
approx_percentile(0.5, ss.rx_pct) AS x, approx_percentile(0.5, ss.ry_pct) AS y,
approx_percentile(0.5, ss.error_pct) AS error, {thr} AS error_threshold,
@@ -215,11 +218,12 @@ def aggregate_states(conn, unit, a, b):
WHERE s.acquisition_unit_id = :au AND ss.time_utc >= :a AND ss.time_utc <= :b
ORDER BY ss.time_utc""", au=unit, a=a, b=b, thr=DEFAULT_ERROR_THRESHOLD)
if sensor.empty:
return pd.DataFrame(columns=COLUMNS)
return pd.DataFrame(columns=COLUMNS + ["sensor_id"])
common = query(conn, """
SELECT sc.time_utc, approx_percentile(0.5, sc.sigma0_pct) AS sigma0 FROM shift_common_1min sc
WHERE sc.acquisition_unit_id = :au AND sc.time_utc >= :a AND sc.time_utc <= :b""", au=unit, a=a, b=b)
out = pd.DataFrame({"time_utc": pd.to_datetime(sensor["time_utc"]), "sensor_name": sensor["name"].astype(str)})
out = pd.DataFrame({"time_utc": pd.to_datetime(sensor["time_utc"]), "sensor_name": sensor["name"].astype(str),
"sensor_id": sensor["sensor_id"]})
for col, src in [("error_value", "error"), ("error_threshold", "error_threshold"), ("corr_x", "corr_x"),
("corr_y", "corr_y"), ("relative_x", "corr_x"), ("relative_y", "corr_y"), ("x", "x"), ("y", "y"),
("reproj_err", "reproj_err")] + [(m, m) for m in QUALITY]:
@@ -232,7 +236,7 @@ def aggregate_states(conn, unit, a, b):
sig = common.loc[common["sigma0"].notna()]
out["sigma0"] = (out["time_utc"].map(sig.assign(time_utc=pd.to_datetime(sig["time_utc"]))
.groupby("time_utc")["sigma0"].median()) if not sig.empty else float("nan"))
return out.reindex(columns=COLUMNS)
return out.reindex(columns=COLUMNS + ["sensor_id"])
def merge(raw, agg):
@@ -256,11 +260,42 @@ def merge(raw, agg):
t = utc(agg["time_utc"])
agg["time_utc"] = t.dt.tz_convert(tz) if tz is not None else t.dt.tz_localize(None)
return (pd.concat([raw, agg], ignore_index=True).sort_values(["sensor_name", "time_utc"], kind="stable")
.reset_index(drop=True).reindex(columns=COLUMNS))
.reset_index(drop=True).reindex(columns=COLUMNS + ["sensor_id"]))
SENSOR_PROPS = {"pattern_type": "pattern_type", "distance_mm": "distance_mm", "distance_convertion_factor": "mm_per_px"}
MM = {"corr_x": "corr_x_mm", "corr_y": "corr_y_mm"}
def sensors(conn, unit):
"""The sensors of the unit that have a pattern_type (targets and references), one row per sensor_id: sensor_name,
pattern_type, distance_mm (from the camera; 1000: not filled in on the platform) and mm_per_px
(distance_convertion_factor, the factor of the dashboard's exports in mm: DeviceOutput.get_scale,
compute_output_signals, (corr - offset) * factor). A name may have several sensors (an older configuration of the
unit, another lens): the factor of a measurement is that of its sensor_id, never looked up by name."""
prop = "max(p.value #>> '{{}}') FILTER (WHERE dt.name = '{}')".format
df = query(conn, f"""
SELECT s.id AS sensor_id, COALESCE({prop('sensor_name')}, s.name) AS sensor_name,
{", ".join(f"{prop(n)} AS {c}" for n, c in SENSOR_PROPS.items())}
FROM sensor s JOIN property p ON p.sensor_id = s.id JOIN datatype dt ON dt.id = p.datatype_id
WHERE s.acquisition_unit_id = :au
GROUP BY s.id, s.name HAVING {prop('pattern_type')} IS NOT NULL""", au=unit)
for c in ("distance_mm", "mm_per_px"):
df[c] = pd.to_numeric(df[c], errors="coerce")
return df.set_index("sensor_id")
def with_mm(df, props):
"""corr_x_mm / corr_y_mm = corr_x / corr_y * mm_per_px of the sensor_id of each row; empty without a factor."""
factor = df["sensor_id"].map(props["mm_per_px"]).astype(float)
for c, mm in MM.items():
df[mm] = pd.to_numeric(df[c], errors="coerce") * factor
return df
def advanced(conn, unit, a, b):
return merge(prepare_raw(raw_datapoints(conn, unit, a, b)), aggregate_states(conn, unit, a, b))
df = merge(prepare_raw(raw_datapoints(conn, unit, a, b)), aggregate_states(conn, unit, a, b))
return with_mm(df, sensors(conn, unit))
def camera(conn, unit, a, b):
@@ -279,7 +314,9 @@ DAILY_FILES = (("displacement", "advanced", advanced), ("camera", "camera", came
def pump_displacements(conn, unit, station, start, end, out, replace):
"""Displacements and camera movements: one file and one query per day, bounded, under the timeout (--delay)."""
"""Displacements and camera movements: one file and one query per day, bounded, under the timeout (--delay)
-> {sensor_name: sensor_id} measured, the last day pumped winning (for targets.csv)."""
seen = {}
for k in range((end - start).days):
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
for folder, name, reader in DAILY_FILES:
@@ -296,9 +333,25 @@ def pump_displacements(conn, unit, station, start, end, out, replace):
conn.rollback() # closes the read transaction
if "sensor_name" in df:
df = df.drop_duplicates(["sensor_name", "time_utc"])
if "sensor_id" in df:
seen.update(df.dropna(subset=["sensor_id"]).groupby("sensor_name")["sensor_id"].last().to_dict())
df = df.drop(columns="sensor_id")
path.parent.mkdir(parents=True, exist_ok=True)
df.to_csv(path, index=False) # an empty day is written too: the next run does not ask for it again
print(f"{station} {a:%Y-%m-%d} {name}: {len(df)} rows, {time.time() - t0:.0f} s -> {path.name}")
return seen
def write_targets(folder, props, seen):
"""targets.csv: the properties of the sensors measured (seen: {name: sensor_id}); the names not measured in this
run keep their line."""
path = folder / "targets.csv"
new = props.loc[[i for i in seen.values() if i in props.index], ["sensor_name", *SENSOR_PROPS.values()]]
old = pd.read_csv(path) if path.exists() else new.iloc[:0]
t = pd.concat([old[~old["sensor_name"].astype(str).isin(new["sensor_name"])], new]).sort_values("sensor_name")
folder.mkdir(parents=True, exist_ok=True)
t.to_csv(path, index=False)
return t
def pump_images(conn, s3, unit, station, start, end, folder, every, out):
@@ -352,6 +405,8 @@ def main():
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD; without: the last pumped day of each "
"station, pumped again, or the last 30 days")
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD; without: up to now")
p.add_argument("--replace", action="store_true", help="with --from: the days already pumped are pumped again "
"(a file is replaced only once its day is read)")
p.add_argument("--images", action="store_true", help="also the OPTICAL_SHIFT-IRON images")
p.add_argument("--diff", action="store_true", help="also the OPTICAL_SHIFT-DIFF images")
p.add_argument("--every", type=int, default=60, help="one image per slot of N minutes, 0 = all (default 60)")
@@ -388,9 +443,10 @@ def main():
if system_id is None:
print(f"{station}: unknown system")
continue
ranges = [(start, end, False)] if start else update_ranges(a.out / station, end, tomorrow)
ranges = [(start, end, a.replace)] if start else update_ranges(a.out / station, end, tomorrow)
unit = acquisition_unit(conn, system_id)
conn.rollback()
seen = {}
for range_start, range_end, replace in ranges:
print(f"{station}: from {range_start} to {range_end - timedelta(days=1)} inclusive"
+ ("" if replace else ", missing days only"))
@@ -398,13 +454,45 @@ def main():
if unit is None:
print(f"{station}: no recognised acquisition unit, no displacements, camera or images")
continue
pump_displacements(conn, unit, station, range_start, range_end, a.out, replace)
seen.update(pump_displacements(conn, unit, station, range_start, range_end, a.out, replace))
for flag, folder in ((a.images, "images"), (a.diff, "images_diff")):
if flag:
pump_images(conn, s3, unit, station, range_start, range_end, folder, a.every, a.out)
if not seen:
continue
try:
props = sensors(conn, unit)
except Exception as e: # the next run writes them
print(f"{station} targets: failed ({root_cause(e)}), run again")
continue
finally:
conn.rollback()
t = write_targets(a.out / station, props, seen)
print(f"{station} targets.csv: {len(t)} sensors, {t['distance_mm'].notna().sum()} with distance_mm, "
f"{t['mm_per_px'].notna().sum()} with mm_per_px")
finally:
engine.dispose()
def check():
"""with_mm and write_targets without the database: `uv run --with pandas --with sqlalchemy python
extraction/pump.py --check`."""
import tempfile
props = pd.DataFrame({"sensor_name": ["T1", "T1", "REF", "T2"], "pattern_type": ["target", "target", "global_ref",
"target"], "distance_mm": [20000, 20000, 50000, 1000], "mm_per_px": [7.0, 2.0, 18.0, None]},
index=pd.Index([11, 12, 13, 14], name="sensor_id"))
df = pd.DataFrame({"sensor_id": [12, 13, 14, 12], "corr_x": [0.5, 0.1, 0.1, None], "corr_y": [-1, 0.2, 0.2, None]})
df = with_mm(df, props) # T1 measured by its sensor 12 (2 mm/px), not by its older namesake 11 (7 mm/px)
assert df["corr_x_mm"].tolist()[:2] == [1.0, 1.8] and df["corr_y_mm"].tolist()[:2] == [-2.0, 3.6], df
assert df[["corr_x_mm", "corr_y_mm"]].iloc[2:].isna().all().all(), df # no factor, no corr: empty
with tempfile.TemporaryDirectory() as d:
pd.DataFrame({"sensor_name": ["OLD", "T1"], "pattern_type": "target", "distance_mm": [1, 1],
"mm_per_px": [1.0, 1.0]}).to_csv(Path(d) / "targets.csv", index=False)
t = write_targets(Path(d), props, {"T1": 12, "REF": 13})
assert t.values.tolist() == [["OLD", "target", 1, 1.0], ["REF", "global_ref", 50000, 18.0],
["T1", "target", 20000, 2.0]], t
print("ok")
if __name__ == "__main__":
main()
check() if sys.argv[1:] == ["--check"] else main()