Pumping: the weather of the stations with a position written into data/<SERIAL>/weather/ (extraction/weather.py, after pump.py); the mm (corr_x_mm, corr_y_mm by the sensor_id of each row) and targets.csv with the references (pattern_type, distance_mm, mm_per_px); --replace pumps a range again
pump.py --check passes; the mm and targets.csv parts are not yet run against the database (no VPN). stations.txt: NIEDERHORN and NORDEND without a position (outside Switzerland). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9adc38f9b9
commit
ea0228a73a
8 files changed
+205
-22
No files matched your search
+100
-12
@@ -14,6 +14,9 @@ from the last 30 days if it has none; without `--to`: up to now.
|
||||
Output (default: the repository's `data/`, next to this folder):
|
||||
<STATION>/metrics/<STATION>_<metric>_<YYYY_MM_DD>_00_00 to <YYYY_MM_DD>_23_00.csv system metrics, per day
|
||||
<STATION>/displacement/<STATION>_advanced_<YYYYMMDD>.csv displacements, one file per day
|
||||
+ corr_x_mm, corr_y_mm: corr * mm_per_px of the sensor that measured (references included)
|
||||
<STATION>/targets.csv the targets and references measured: sensor_name, pattern_type,
|
||||
distance_mm, mm_per_px (those of the last day pumped)
|
||||
<STATION>/camera/<STATION>_camera_<YYYYMMDD>.csv camera movements, per day
|
||||
<STATION>/images/<YYYYMMDD>T<HHMM>Z_<id>.<ext> OPTICAL_SHIFT-IRON images (--images)
|
||||
<STATION>/images_diff/… differential images (--diff)
|
||||
@@ -175,7 +178,7 @@ def prepare_raw(shifts):
|
||||
"""prepare_datapoints_frame: sigma0 (camera rows, without sensor_name) copied onto each sensor at its time;
|
||||
relative = x - prop_x; error -> error_value; the full schema."""
|
||||
if shifts.empty:
|
||||
return pd.DataFrame(columns=COLUMNS)
|
||||
return pd.DataFrame(columns=COLUMNS + ["sensor_id"])
|
||||
sig = shifts.loc[shifts["sigma0"].notna(), ["time_utc", "sigma0"]]
|
||||
sigma0 = sig.groupby("time_utc")["sigma0"].median() if not sig.empty else None
|
||||
shifts = shifts.dropna(subset=["sensor_name"]).reset_index(drop=True)
|
||||
@@ -188,7 +191,7 @@ def prepare_raw(shifts):
|
||||
for c in NUMERIC:
|
||||
if c in shifts:
|
||||
shifts[c] = pd.to_numeric(shifts[c], errors="coerce")
|
||||
return shifts.reindex(columns=COLUMNS)
|
||||
return shifts.reindex(columns=COLUMNS + ["sensor_id"])
|
||||
|
||||
|
||||
def aggregate_states(conn, unit, a, b):
|
||||
@@ -197,7 +200,7 @@ def aggregate_states(conn, unit, a, b):
|
||||
In the states, x_pct holds the CORRECTED x and rx_pct the raw x."""
|
||||
thr = "COALESCE((th.value #>> '{}')::double precision, CAST(:thr AS double precision))"
|
||||
sensor = query(conn, f"""
|
||||
SELECT ss.time_utc, COALESCE(sn.value #>> '{{}}', s.name) AS name,
|
||||
SELECT ss.time_utc, ss.sensor_id, COALESCE(sn.value #>> '{{}}', s.name) AS name,
|
||||
approx_percentile(0.5, ss.x_pct) AS corr_x, approx_percentile(0.5, ss.y_pct) AS corr_y,
|
||||
approx_percentile(0.5, ss.rx_pct) AS x, approx_percentile(0.5, ss.ry_pct) AS y,
|
||||
approx_percentile(0.5, ss.error_pct) AS error, {thr} AS error_threshold,
|
||||
@@ -215,11 +218,12 @@ def aggregate_states(conn, unit, a, b):
|
||||
WHERE s.acquisition_unit_id = :au AND ss.time_utc >= :a AND ss.time_utc <= :b
|
||||
ORDER BY ss.time_utc""", au=unit, a=a, b=b, thr=DEFAULT_ERROR_THRESHOLD)
|
||||
if sensor.empty:
|
||||
return pd.DataFrame(columns=COLUMNS)
|
||||
return pd.DataFrame(columns=COLUMNS + ["sensor_id"])
|
||||
common = query(conn, """
|
||||
SELECT sc.time_utc, approx_percentile(0.5, sc.sigma0_pct) AS sigma0 FROM shift_common_1min sc
|
||||
WHERE sc.acquisition_unit_id = :au AND sc.time_utc >= :a AND sc.time_utc <= :b""", au=unit, a=a, b=b)
|
||||
out = pd.DataFrame({"time_utc": pd.to_datetime(sensor["time_utc"]), "sensor_name": sensor["name"].astype(str)})
|
||||
out = pd.DataFrame({"time_utc": pd.to_datetime(sensor["time_utc"]), "sensor_name": sensor["name"].astype(str),
|
||||
"sensor_id": sensor["sensor_id"]})
|
||||
for col, src in [("error_value", "error"), ("error_threshold", "error_threshold"), ("corr_x", "corr_x"),
|
||||
("corr_y", "corr_y"), ("relative_x", "corr_x"), ("relative_y", "corr_y"), ("x", "x"), ("y", "y"),
|
||||
("reproj_err", "reproj_err")] + [(m, m) for m in QUALITY]:
|
||||
@@ -232,7 +236,7 @@ def aggregate_states(conn, unit, a, b):
|
||||
sig = common.loc[common["sigma0"].notna()]
|
||||
out["sigma0"] = (out["time_utc"].map(sig.assign(time_utc=pd.to_datetime(sig["time_utc"]))
|
||||
.groupby("time_utc")["sigma0"].median()) if not sig.empty else float("nan"))
|
||||
return out.reindex(columns=COLUMNS)
|
||||
return out.reindex(columns=COLUMNS + ["sensor_id"])
|
||||
|
||||
|
||||
def merge(raw, agg):
|
||||
@@ -256,11 +260,42 @@ def merge(raw, agg):
|
||||
t = utc(agg["time_utc"])
|
||||
agg["time_utc"] = t.dt.tz_convert(tz) if tz is not None else t.dt.tz_localize(None)
|
||||
return (pd.concat([raw, agg], ignore_index=True).sort_values(["sensor_name", "time_utc"], kind="stable")
|
||||
.reset_index(drop=True).reindex(columns=COLUMNS))
|
||||
.reset_index(drop=True).reindex(columns=COLUMNS + ["sensor_id"]))
|
||||
|
||||
|
||||
SENSOR_PROPS = {"pattern_type": "pattern_type", "distance_mm": "distance_mm", "distance_convertion_factor": "mm_per_px"}
|
||||
MM = {"corr_x": "corr_x_mm", "corr_y": "corr_y_mm"}
|
||||
|
||||
|
||||
def sensors(conn, unit):
|
||||
"""The sensors of the unit that have a pattern_type (targets and references), one row per sensor_id: sensor_name,
|
||||
pattern_type, distance_mm (from the camera; 1000: not filled in on the platform) and mm_per_px
|
||||
(distance_convertion_factor, the factor of the dashboard's exports in mm: DeviceOutput.get_scale,
|
||||
compute_output_signals, (corr - offset) * factor). A name may have several sensors (an older configuration of the
|
||||
unit, another lens): the factor of a measurement is that of its sensor_id, never looked up by name."""
|
||||
prop = "max(p.value #>> '{{}}') FILTER (WHERE dt.name = '{}')".format
|
||||
df = query(conn, f"""
|
||||
SELECT s.id AS sensor_id, COALESCE({prop('sensor_name')}, s.name) AS sensor_name,
|
||||
{", ".join(f"{prop(n)} AS {c}" for n, c in SENSOR_PROPS.items())}
|
||||
FROM sensor s JOIN property p ON p.sensor_id = s.id JOIN datatype dt ON dt.id = p.datatype_id
|
||||
WHERE s.acquisition_unit_id = :au
|
||||
GROUP BY s.id, s.name HAVING {prop('pattern_type')} IS NOT NULL""", au=unit)
|
||||
for c in ("distance_mm", "mm_per_px"):
|
||||
df[c] = pd.to_numeric(df[c], errors="coerce")
|
||||
return df.set_index("sensor_id")
|
||||
|
||||
|
||||
def with_mm(df, props):
|
||||
"""corr_x_mm / corr_y_mm = corr_x / corr_y * mm_per_px of the sensor_id of each row; empty without a factor."""
|
||||
factor = df["sensor_id"].map(props["mm_per_px"]).astype(float)
|
||||
for c, mm in MM.items():
|
||||
df[mm] = pd.to_numeric(df[c], errors="coerce") * factor
|
||||
return df
|
||||
|
||||
|
||||
def advanced(conn, unit, a, b):
|
||||
return merge(prepare_raw(raw_datapoints(conn, unit, a, b)), aggregate_states(conn, unit, a, b))
|
||||
df = merge(prepare_raw(raw_datapoints(conn, unit, a, b)), aggregate_states(conn, unit, a, b))
|
||||
return with_mm(df, sensors(conn, unit))
|
||||
|
||||
|
||||
def camera(conn, unit, a, b):
|
||||
@@ -279,7 +314,9 @@ DAILY_FILES = (("displacement", "advanced", advanced), ("camera", "camera", came
|
||||
|
||||
|
||||
def pump_displacements(conn, unit, station, start, end, out, replace):
|
||||
"""Displacements and camera movements: one file and one query per day, bounded, under the timeout (--delay)."""
|
||||
"""Displacements and camera movements: one file and one query per day, bounded, under the timeout (--delay)
|
||||
-> {sensor_name: sensor_id} measured, the last day pumped winning (for targets.csv)."""
|
||||
seen = {}
|
||||
for k in range((end - start).days):
|
||||
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
|
||||
for folder, name, reader in DAILY_FILES:
|
||||
@@ -296,9 +333,25 @@ def pump_displacements(conn, unit, station, start, end, out, replace):
|
||||
conn.rollback() # closes the read transaction
|
||||
if "sensor_name" in df:
|
||||
df = df.drop_duplicates(["sensor_name", "time_utc"])
|
||||
if "sensor_id" in df:
|
||||
seen.update(df.dropna(subset=["sensor_id"]).groupby("sensor_name")["sensor_id"].last().to_dict())
|
||||
df = df.drop(columns="sensor_id")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.to_csv(path, index=False) # an empty day is written too: the next run does not ask for it again
|
||||
print(f"{station} {a:%Y-%m-%d} {name}: {len(df)} rows, {time.time() - t0:.0f} s -> {path.name}")
|
||||
return seen
|
||||
|
||||
|
||||
def write_targets(folder, props, seen):
|
||||
"""targets.csv: the properties of the sensors measured (seen: {name: sensor_id}); the names not measured in this
|
||||
run keep their line."""
|
||||
path = folder / "targets.csv"
|
||||
new = props.loc[[i for i in seen.values() if i in props.index], ["sensor_name", *SENSOR_PROPS.values()]]
|
||||
old = pd.read_csv(path) if path.exists() else new.iloc[:0]
|
||||
t = pd.concat([old[~old["sensor_name"].astype(str).isin(new["sensor_name"])], new]).sort_values("sensor_name")
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
t.to_csv(path, index=False)
|
||||
return t
|
||||
|
||||
|
||||
def pump_images(conn, s3, unit, station, start, end, folder, every, out):
|
||||
@@ -352,6 +405,8 @@ def main():
|
||||
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD; without: the last pumped day of each "
|
||||
"station, pumped again, or the last 30 days")
|
||||
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD; without: up to now")
|
||||
p.add_argument("--replace", action="store_true", help="with --from: the days already pumped are pumped again "
|
||||
"(a file is replaced only once its day is read)")
|
||||
p.add_argument("--images", action="store_true", help="also the OPTICAL_SHIFT-IRON images")
|
||||
p.add_argument("--diff", action="store_true", help="also the OPTICAL_SHIFT-DIFF images")
|
||||
p.add_argument("--every", type=int, default=60, help="one image per slot of N minutes, 0 = all (default 60)")
|
||||
@@ -388,9 +443,10 @@ def main():
|
||||
if system_id is None:
|
||||
print(f"{station}: unknown system")
|
||||
continue
|
||||
ranges = [(start, end, False)] if start else update_ranges(a.out / station, end, tomorrow)
|
||||
ranges = [(start, end, a.replace)] if start else update_ranges(a.out / station, end, tomorrow)
|
||||
unit = acquisition_unit(conn, system_id)
|
||||
conn.rollback()
|
||||
seen = {}
|
||||
for range_start, range_end, replace in ranges:
|
||||
print(f"{station}: from {range_start} to {range_end - timedelta(days=1)} inclusive"
|
||||
+ ("" if replace else ", missing days only"))
|
||||
@@ -398,13 +454,45 @@ def main():
|
||||
if unit is None:
|
||||
print(f"{station}: no recognised acquisition unit, no displacements, camera or images")
|
||||
continue
|
||||
pump_displacements(conn, unit, station, range_start, range_end, a.out, replace)
|
||||
seen.update(pump_displacements(conn, unit, station, range_start, range_end, a.out, replace))
|
||||
for flag, folder in ((a.images, "images"), (a.diff, "images_diff")):
|
||||
if flag:
|
||||
pump_images(conn, s3, unit, station, range_start, range_end, folder, a.every, a.out)
|
||||
if not seen:
|
||||
continue
|
||||
try:
|
||||
props = sensors(conn, unit)
|
||||
except Exception as e: # the next run writes them
|
||||
print(f"{station} targets: failed ({root_cause(e)}), run again")
|
||||
continue
|
||||
finally:
|
||||
conn.rollback()
|
||||
t = write_targets(a.out / station, props, seen)
|
||||
print(f"{station} targets.csv: {len(t)} sensors, {t['distance_mm'].notna().sum()} with distance_mm, "
|
||||
f"{t['mm_per_px'].notna().sum()} with mm_per_px")
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def check():
|
||||
"""with_mm and write_targets without the database: `uv run --with pandas --with sqlalchemy python
|
||||
extraction/pump.py --check`."""
|
||||
import tempfile
|
||||
props = pd.DataFrame({"sensor_name": ["T1", "T1", "REF", "T2"], "pattern_type": ["target", "target", "global_ref",
|
||||
"target"], "distance_mm": [20000, 20000, 50000, 1000], "mm_per_px": [7.0, 2.0, 18.0, None]},
|
||||
index=pd.Index([11, 12, 13, 14], name="sensor_id"))
|
||||
df = pd.DataFrame({"sensor_id": [12, 13, 14, 12], "corr_x": [0.5, 0.1, 0.1, None], "corr_y": [-1, 0.2, 0.2, None]})
|
||||
df = with_mm(df, props) # T1 measured by its sensor 12 (2 mm/px), not by its older namesake 11 (7 mm/px)
|
||||
assert df["corr_x_mm"].tolist()[:2] == [1.0, 1.8] and df["corr_y_mm"].tolist()[:2] == [-2.0, 3.6], df
|
||||
assert df[["corr_x_mm", "corr_y_mm"]].iloc[2:].isna().all().all(), df # no factor, no corr: empty
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
pd.DataFrame({"sensor_name": ["OLD", "T1"], "pattern_type": "target", "distance_mm": [1, 1],
|
||||
"mm_per_px": [1.0, 1.0]}).to_csv(Path(d) / "targets.csv", index=False)
|
||||
t = write_targets(Path(d), props, {"T1": 12, "REF": 13})
|
||||
assert t.values.tolist() == [["OLD", "target", 1, 1.0], ["REF", "global_ref", 50000, 18.0],
|
||||
["T1", "target", 20000, 2.0]], t
|
||||
print("ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
check() if sys.argv[1:] == ["--check"] else main()
|
||||
Reference in new issue
Block a user