2026-11-19 10:52:46 +01:00
|
|
|
"""Runs one configuration of a station: data_diff/<SERIAL>/runs/<run>/config.yaml in, the same folder's displacement/,
|
|
|
|
|
camera/ and targets.csv out, in the files of data/. The station's pumped data, the same for every run (images_diff/,
|
|
|
|
|
images/, metrics/, weather/), stay at data_diff/<SERIAL>/ and are only read; whatever depends on the run is in its folder.
|
2026-11-19 10:00:02 +01:00
|
|
|
|
2026-11-19 10:52:46 +01:00
|
|
|
processing/run_config.sh JUNGFRAU run02 --from 2026-10-12 --to 2026-10-17
|
|
|
|
|
processing/run_config.sh JUNGFRAU run02 every indexed day
|
2026-11-19 10:00:02 +01:00
|
|
|
|
2026-11-19 10:52:46 +01:00
|
|
|
Days: those indexed in data_diff/<SERIAL>/images_diff/index/ (within --from / --to, --to excluded). Running a run again
|
|
|
|
|
replaces its displacement/, camera/, targets.csv and patterns.json; nothing else is written:
|
2026-11-19 10:00:02 +01:00
|
|
|
displacement/<S>_advanced_<YYYYMMDD>.csv the columns of data/: x, y, error_value, peak..snr from optical's emulator;
|
|
|
|
|
relative = x - pattern x; corr from the camera correction; reproj_err,
|
|
|
|
|
inlier of the references; sigma0 of the epoch; corr_*_mm
|
|
|
|
|
camera/<S>_camera_<YYYYMMDD>.csv pan, tilt, scale, rotation, sigma0: median / min / max per minute
|
2026-11-19 10:52:46 +01:00
|
|
|
targets.csv as in data/, from the run's patterns: pattern_type, distance_mm, mm_per_px
|
|
|
|
|
patterns.json the patterns exactly as given to the emulator (templates included)
|
|
|
|
|
|
|
|
|
|
config.yaml states what differs from the cloud's processing (an empty file: the cloud's), checked key by key:
|
|
|
|
|
patterns: by sensor_name, over images_diff/sensors.json: any existing property, or
|
|
|
|
|
PM_G: {error_threshold: 0.7} a template redrawn from a DIFF image of images_diff/, as the
|
|
|
|
|
BL_C: {redraw: {image: <file>, corners: [[x0, y0], [x1, y1]]}} device's pattern editor does (x_ref kept)
|
|
|
|
|
camera: over images_diff/unit.json, over the cloud's defaults
|
|
|
|
|
algorithm: ransac ransac | umeyama | danish_hybrid | trusted_danish
|
|
|
|
|
ransac_threshold: 0.99
|
|
|
|
|
mode: "off" off | rotation | rotation_scale | relative (optical's refine_allpoints)
|
|
|
|
|
focal_distance: 6.0 pixel_size: 0.0024
|
|
|
|
|
per_epoch: true one measurement at a time, as the cloud; false: the whole series at once
|
|
|
|
|
emulator:
|
|
|
|
|
version: "65a4c70" optical's commit: the run stops if $OPTICAL is at another one or has local changes
|
|
|
|
|
Design: docs/superpowers/specs/2026-11-19-run-config-design.md.
|
2026-11-19 10:00:02 +01:00
|
|
|
"""
|
|
|
|
|
import argparse
|
|
|
|
|
import csv
|
2026-11-19 10:52:46 +01:00
|
|
|
import datetime
|
2026-11-19 10:00:02 +01:00
|
|
|
import json
|
|
|
|
|
import os
|
|
|
|
|
import shutil
|
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
2026-11-19 10:52:46 +01:00
|
|
|
import tempfile
|
2026-11-19 10:00:02 +01:00
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
# Before numpy loads its BLAS, as reprocess_images.py does: the re-measurement's worker processes inherit it, and
|
|
|
|
|
# 8 workers x one BLAS thread per core made a 11 s re-measurement take 2 min (45 min of CPU).
|
|
|
|
|
for _var in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS"):
|
|
|
|
|
os.environ.setdefault(_var, "1")
|
|
|
|
|
|
|
|
|
|
import numpy as np # noqa: E402
|
|
|
|
|
import pandas as pd # noqa: E402
|
|
|
|
|
import yaml # noqa: E402
|
|
|
|
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
OPTICAL = Path(os.environ.get("OPTICAL", Path.home() / "optical")).resolve()
|
|
|
|
|
sys.path[:0] = [str(OPTICAL), str(OPTICAL / "scripts" / "pcc_emulator")]
|
|
|
|
|
|
|
|
|
|
MTYPE = "OPTICAL_SHIFT-DIFF"
|
|
|
|
|
QUALITY = ["peak", "flux", "sharpness", "fwhm", "dominance", "snr"]
|
|
|
|
|
COLUMNS = ["time_utc", "sensor_name", "x", "y", "error_value", "error_threshold", "relative_x", "relative_y", "corr_x",
|
|
|
|
|
"corr_y", "reproj_err", "inlier", "sigma0", "pattern_type"] + QUALITY + ["corr_x_mm", "corr_y_mm"]
|
|
|
|
|
CAM = ["pan", "tilt", "scale", "rotation", "sigma0"]
|
|
|
|
|
# the cloud's defaults (process_shifts), then the unit's properties, then the configuration
|
|
|
|
|
DEFAULTS = {"algorithm": "ransac", "ransac_threshold": 0.99, "mode": "off", "focal_distance": 6.0,
|
|
|
|
|
"pixel_size": 0.0024, "per_epoch": True}
|
|
|
|
|
UNIT_KEYS = ("algorithm", "ransac_threshold", "focal_distance", "pixel_size")
|
2026-11-19 10:52:46 +01:00
|
|
|
OUTPUTS = ("displacement", "camera") # what a run writes into its folder, and replaces
|
2026-11-19 10:00:02 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _keys(d, allowed, where):
|
|
|
|
|
if not isinstance(d, dict):
|
|
|
|
|
raise ValueError(f"{where}: a mapping is expected")
|
|
|
|
|
bad = set(d) - set(allowed)
|
|
|
|
|
if bad:
|
|
|
|
|
raise ValueError(f"{where}: unknown key(s) {sorted(bad)}; allowed: {sorted(allowed)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_config(path):
|
|
|
|
|
"""The configuration, its keys checked (a typo is an error, not a setting silently ignored)."""
|
|
|
|
|
c = yaml.safe_load(Path(path).read_text()) or {}
|
2026-11-19 10:52:46 +01:00
|
|
|
_keys(c, {"patterns", "camera", "emulator"}, str(path))
|
|
|
|
|
_keys(c.get("camera") or {}, DEFAULTS, f"{path}: camera")
|
|
|
|
|
_keys(c.get("emulator") or {}, {"version"}, f"{path}: emulator")
|
|
|
|
|
_keys(c.get("patterns") or {}, set(c.get("patterns") or {}), f"{path}: patterns") # a mapping (names: later)
|
2026-11-19 10:00:02 +01:00
|
|
|
return c
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def latest(rows):
|
|
|
|
|
"""{name: value} of property rows ordered by name then time (sensors.json, unit.json): the latest wins."""
|
|
|
|
|
return {r["name"]: r["value"] for r in rows}
|
|
|
|
|
|
|
|
|
|
|
2026-11-19 10:52:46 +01:00
|
|
|
def camera_settings(config, unit_props):
|
2026-11-19 10:00:02 +01:00
|
|
|
cam = dict(DEFAULTS)
|
|
|
|
|
cam.update({k: unit_props[k] for k in UNIT_KEYS if k in unit_props})
|
|
|
|
|
cam.update(config.get("camera") or {})
|
|
|
|
|
return cam
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def cloud_properties(sensors):
|
|
|
|
|
"""{sensor_id: properties} of the patterns the cloud measures the DIFF images with (process_shifts): DIFF type, a
|
|
|
|
|
pattern, not archived."""
|
|
|
|
|
out = {}
|
|
|
|
|
for s in sensors:
|
|
|
|
|
p = latest(s["properties"])
|
|
|
|
|
if s["type"] == MTYPE and p.get("pattern") and not p.get("archive"):
|
|
|
|
|
out[str(s["id"])] = p
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def apply_patterns(props, overrides, images_dir):
|
|
|
|
|
"""The configuration's patterns over the cloud's, by sensor_name: a property replaced (only an existing one: a
|
|
|
|
|
typo is an error), or the template redrawn from an image."""
|
|
|
|
|
props = {sid: dict(p) for sid, p in props.items()}
|
|
|
|
|
by_name = {p.get("sensor_name"): sid for sid, p in props.items()}
|
|
|
|
|
redraws = {}
|
|
|
|
|
for name, ov in (overrides or {}).items():
|
|
|
|
|
if name not in by_name:
|
|
|
|
|
raise ValueError(f"patterns.{name}: no such pattern; known: {sorted(map(str, by_name))}")
|
|
|
|
|
sid = by_name[name]
|
|
|
|
|
_keys(ov, set(props[sid]) - {"pattern"} | {"redraw"}, f"patterns.{name}")
|
|
|
|
|
for k, v in ov.items():
|
|
|
|
|
if k == "redraw":
|
|
|
|
|
_keys(v, {"image", "corners"}, f"patterns.{name}.redraw")
|
|
|
|
|
redraws[sid] = v
|
|
|
|
|
else:
|
|
|
|
|
props[sid][k] = v
|
|
|
|
|
if redraws:
|
|
|
|
|
redraw(props, redraws, images_dir)
|
|
|
|
|
return props
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def redraw(props, redraws, images_dir):
|
|
|
|
|
"""As the device's pattern editor saves a drawn box (optical_server pages/patternCreation.py): the camera pose from
|
|
|
|
|
the references found in the image, the anchor and x_ref (kept) of saved_anchor_and_reference, the template cut
|
|
|
|
|
at the box."""
|
|
|
|
|
from aimsight_optical.imagePatternMono import ImagePatternMono, get_center, saved_anchor_and_reference
|
|
|
|
|
from aimsight_optical.pattern_converter import load_pattern_from_params
|
|
|
|
|
from aimsight_optical.pattern_pose import estimate_camera_pose
|
|
|
|
|
from reprocess_images import read_image
|
|
|
|
|
|
|
|
|
|
for image in sorted({r["image"] for r in redraws.values()}):
|
|
|
|
|
im = read_image(images_dir / image)
|
|
|
|
|
stored = {sid: load_pattern_from_params(**dict(p)) for sid, p in props.items()}
|
|
|
|
|
mine = {sid: r for sid, r in redraws.items() if r["image"] == image}
|
|
|
|
|
drawn = {sid: get_center(r["corners"]) for sid, r in mine.items()
|
|
|
|
|
if props[sid].get("pattern_type") == "global_ref"}
|
|
|
|
|
pose, seen = estimate_camera_pose(stored, im, drawn=drawn)
|
|
|
|
|
for sid, r in mine.items():
|
|
|
|
|
old, corners = stored[sid], [[int(v) for v in c] for c in r["corners"]]
|
|
|
|
|
anchor, (x_ref, y_ref) = saved_anchor_and_reference(old, corners, old.extra_attr.get("pattern_type", ""),
|
|
|
|
|
pose=pose, seen=seen.get(sid))
|
|
|
|
|
kept = {k: v for k, v in old.extra_attr.items() if k not in ("median", "p99", "x_ref", "y_ref")}
|
|
|
|
|
new = ImagePatternMono.from_image(im, corners, center=anchor, x_ref=x_ref, y_ref=y_ref, **kept)
|
|
|
|
|
props[sid] = json.loads(json.dumps(new.to_dict(), default=_plain))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _plain(o):
|
|
|
|
|
return o.item() if hasattr(o, "item") else o.tolist() if hasattr(o, "tolist") else str(o)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def correct(meas, props, cam):
|
|
|
|
|
"""The emulator's rows (time_utc, pattern_id, x, y, error, metrics) -> (advanced rows in the columns of data/,
|
|
|
|
|
camera rows per minute), as the cloud's process_shifts with per_epoch, else over the whole series."""
|
|
|
|
|
from aimsight_optical.imagePatternMono import get_center, reference_or_anchor
|
|
|
|
|
from aimsight_optical.pattern_converter import remove_camera_movement
|
|
|
|
|
|
|
|
|
|
m = meas.copy()
|
|
|
|
|
m["sensor_id"] = m["pattern_id"].astype(str)
|
|
|
|
|
m["time_utc"] = pd.to_datetime(m["time_utc"], utc=True, format="mixed")
|
|
|
|
|
p = {sid: props[sid] for sid in m["sensor_id"].unique()}
|
|
|
|
|
anchor = {sid: (float(q["x"]), float(q["y"])) if np.isfinite([q.get("x", np.nan), q.get("y", np.nan)]).all()
|
|
|
|
|
else get_center(q["corners"]) for sid, q in p.items()}
|
|
|
|
|
ref = {sid: reference_or_anchor(q.get("x_ref"), q.get("y_ref"), anchor[sid]) for sid, q in p.items()}
|
|
|
|
|
sid = m["sensor_id"]
|
|
|
|
|
m["sensor_name"] = sid.map({s: str(q.get("sensor_name", s)) for s, q in p.items()})
|
|
|
|
|
m["error_threshold"] = pd.to_numeric(sid.map({s: q.get("error_threshold", 0.7) for s, q in p.items()}))
|
|
|
|
|
m["sensor_type"] = sid.map({s: q.get("pattern_type", "") or "" for s, q in p.items()})
|
|
|
|
|
m["x_ref"], m["y_ref"] = sid.map({s: r[0] for s, r in ref.items()}), sid.map({s: r[1] for s, r in ref.items()})
|
|
|
|
|
|
|
|
|
|
inp = m[["time_utc", "sensor_id", "x", "y", "error", "error_threshold", "sensor_type", "x_ref", "y_ref"]].copy()
|
|
|
|
|
# as the cloud: a target above its threshold has no position; a reference keeps its raw one (its row is corrected,
|
|
|
|
|
# remove_camera_movement leaves it out of the fit on its error)
|
|
|
|
|
inp.loc[(inp["error"] > inp["error_threshold"]) & (inp["sensor_type"] != "global_ref"), ["x", "y"]] = np.nan
|
|
|
|
|
kw = {"algorithm": cam["algorithm"]}
|
|
|
|
|
if str(cam["algorithm"]).upper() == "RANSAC":
|
|
|
|
|
kw["ransac_threshold"] = cam["ransac_threshold"]
|
|
|
|
|
if cam["mode"] != "off":
|
|
|
|
|
kw["refine_allpoints"] = cam["mode"]
|
|
|
|
|
frames = [g for _, g in inp.groupby("time_utc")] if cam["per_epoch"] else [inp]
|
|
|
|
|
corrs, cams, stats = [], [], []
|
|
|
|
|
for g in frames:
|
|
|
|
|
c, cp, rs = remove_camera_movement(g.copy(), cam["focal_distance"], cam["pixel_size"],
|
|
|
|
|
timeserie_column="time_utc", **kw)
|
|
|
|
|
corrs.append(c[["time_utc", "sensor_id", "corr_x", "corr_y"]])
|
|
|
|
|
if not cp.empty:
|
|
|
|
|
cams.append(cp[CAM].rename_axis("time_utc").reset_index())
|
|
|
|
|
if not rs.empty:
|
|
|
|
|
stats.append(rs[["time_utc", "sensor_id", "reproj_err", "inlier"]])
|
|
|
|
|
m = m.merge(pd.concat(corrs), on=["time_utc", "sensor_id"], how="left")
|
|
|
|
|
m = m.merge(pd.concat(stats) if stats else pd.DataFrame(columns=["time_utc", "sensor_id", "reproj_err", "inlier"]),
|
|
|
|
|
on=["time_utc", "sensor_id"], how="left")
|
|
|
|
|
cp = pd.concat(cams).dropna(subset=CAM, how="all") if cams else pd.DataFrame(columns=["time_utc"] + CAM)
|
|
|
|
|
cp["time_utc"] = pd.to_datetime(cp["time_utc"], utc=True)
|
|
|
|
|
m["sigma0"] = m["time_utc"].map(cp.groupby("time_utc")["sigma0"].median())
|
|
|
|
|
|
|
|
|
|
factor = pd.to_numeric(sid.map({s: q.get("distance_convertion_factor") for s, q in p.items()}), errors="coerce")
|
|
|
|
|
adv = pd.DataFrame({
|
|
|
|
|
"time_utc": m["time_utc"], "sensor_name": m["sensor_name"], "x": m["x"], "y": m["y"],
|
|
|
|
|
"error_value": m["error"], "error_threshold": m["error_threshold"],
|
|
|
|
|
"relative_x": m["x"] - sid.map({s: a[0] for s, a in anchor.items()}),
|
|
|
|
|
"relative_y": m["y"] - sid.map({s: a[1] for s, a in anchor.items()}),
|
|
|
|
|
"corr_x": m["corr_x"] - m["x_ref"], "corr_y": m["corr_y"] - m["y_ref"],
|
|
|
|
|
"reproj_err": m["reproj_err"], "inlier": m["inlier"], "sigma0": m["sigma0"], "pattern_type": m["sensor_type"],
|
|
|
|
|
**{q: m[q] if q in m else np.nan for q in QUALITY}})
|
|
|
|
|
adv["corr_x_mm"], adv["corr_y_mm"] = adv["corr_x"] * factor.to_numpy(), adv["corr_y"] * factor.to_numpy()
|
|
|
|
|
adv = adv.sort_values(["time_utc", "sensor_name"], kind="stable").reset_index(drop=True)[COLUMNS]
|
|
|
|
|
|
|
|
|
|
cp["time_utc"] = cp["time_utc"].dt.floor("1min")
|
|
|
|
|
agg = cp.groupby("time_utc")[CAM].agg(["median", "min", "max"])
|
|
|
|
|
agg.columns = [f"{stat}_{c}" for c, stat in agg.columns]
|
|
|
|
|
return adv, agg.reset_index()
|
|
|
|
|
|
|
|
|
|
|
2026-11-19 10:52:46 +01:00
|
|
|
def write_run(adv, camera, dest, station):
|
|
|
|
|
"""displacement/ and camera/ of a run, one file per UTC day, replacing the run's previous ones."""
|
2026-11-19 10:00:02 +01:00
|
|
|
for folder, df, name in (("displacement", adv, "advanced"), ("camera", camera, "camera")):
|
2026-11-19 10:52:46 +01:00
|
|
|
if (dest / folder).exists():
|
|
|
|
|
shutil.rmtree(dest / folder)
|
|
|
|
|
(dest / folder).mkdir()
|
2026-11-19 10:00:02 +01:00
|
|
|
for day, d in df.groupby(df["time_utc"].dt.date):
|
|
|
|
|
d.to_csv(dest / folder / f"{station}_{name}_{day:%Y%m%d}.csv", index=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def optical_state():
|
|
|
|
|
git = lambda *a: subprocess.run(["git", "-C", str(OPTICAL), *a], capture_output=True, text=True).stdout.strip() # noqa: E731
|
|
|
|
|
return {"path": str(OPTICAL), "commit": git("rev-parse", "HEAD"), "branch": git("rev-parse", "--abbrev-ref", "HEAD"),
|
|
|
|
|
"dirty": bool(git("status", "--porcelain", "--untracked-files=no"))}
|
|
|
|
|
|
|
|
|
|
|
2026-11-19 10:52:46 +01:00
|
|
|
PATTERN_KEYS = ["pattern_type", "corners", "x", "y", "error_threshold", "resolution", "distance_convertion_factor"]
|
|
|
|
|
CAMERA_NOTES = {"algorithm": "ransac | umeyama | danish_hybrid | trusted_danish", "ransac_threshold": "px, ransac only",
|
|
|
|
|
"mode": "off | rotation | rotation_scale | relative", "focal_distance": "mm; pan, tilt, sigma0 only",
|
|
|
|
|
"pixel_size": "mm; pan, tilt, sigma0 only",
|
|
|
|
|
"per_epoch": "one measurement at a time, as the cloud; false: the whole series at once"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def new_run(src, name):
|
|
|
|
|
"""runs/<name>/config.yaml of a new, empty run (no result yet): the cloud's processing written in full, the patterns
|
|
|
|
|
of images_diff/sensors.json, the camera of unit.json or the cloud's defaults, optical's current commit."""
|
|
|
|
|
run = src / "runs" / name
|
|
|
|
|
if (run / "config.yaml").exists():
|
|
|
|
|
raise FileExistsError(f"{run / 'config.yaml'} exists: choose another name, or edit that file")
|
|
|
|
|
props = sorted(cloud_properties(json.loads((src / "images_diff" / "sensors.json").read_text())).values(),
|
|
|
|
|
key=lambda p: (p.get("pattern_type") != "global_ref", str(p.get("sensor_name"))))
|
|
|
|
|
unit = latest(json.loads((src / "images_diff" / "unit.json").read_text())["properties"])
|
|
|
|
|
cam, state = camera_settings({}, unit), optical_state()
|
|
|
|
|
width = max(len(str(p.get("sensor_name"))) for p in props) + 1
|
|
|
|
|
line = lambda p: f" {str(p.get('sensor_name')) + ':':{width}s} {{" + ", ".join( # noqa: E731
|
|
|
|
|
f"{k}: {json.dumps(p[k])}" for k in PATTERN_KEYS if k in p) + "}"
|
|
|
|
|
refs = [line(p) for p in props if p.get("pattern_type") == "global_ref"]
|
|
|
|
|
targets = [line(p) for p in props if p.get("pattern_type") != "global_ref"]
|
|
|
|
|
text = f"""# {name}: created on {datetime.date.today()} with the cloud's processing. Change a value, then run it:
|
|
|
|
|
# processing/run_config.sh {src.name} {name}
|
|
|
|
|
# The run also writes here targets.csv (as in data/, from these patterns) and patterns.json (the patterns exactly as
|
|
|
|
|
# given to the emulator, templates included).
|
|
|
|
|
emulator:
|
|
|
|
|
version: "{state['commit'][:7]}" # optical's commit (branch {state['branch']}); the run stops if $OPTICAL is
|
|
|
|
|
# at another one or has local changes
|
|
|
|
|
patterns:
|
|
|
|
|
# The {len(props)} patterns of the DIFF images, as in images_diff/sensors.json on {datetime.date.today()} (frozen here: a
|
|
|
|
|
# later pumping does not change them). The template stays in sensors.json; to change it:
|
|
|
|
|
# <name>: {{redraw: {{image: <file of images_diff/>, corners: [[x0, y0], [x1, y1]]}}}}
|
|
|
|
|
# pattern_type: global_ref = a reference of the camera fit, anything else a target. corners and x, y move together
|
|
|
|
|
# (same window size as the template). resolution: 1/100 px. distance_convertion_factor: mm per px (corr_*_mm only).
|
|
|
|
|
# references
|
|
|
|
|
""" + "\n".join(refs) + "\n # targets\n" + "\n".join(targets) + "\ncamera:" + "".join(
|
|
|
|
|
f"\n {f'{k}: {json.dumps(v)}':26s} # {CAMERA_NOTES[k]}" for k, v in cam.items()) + (
|
|
|
|
|
"\n # focal_distance, pixel_size: the unit has none on the platform, the cloud's values"
|
|
|
|
|
if not {"focal_distance", "pixel_size"} & set(unit) else "") + "\n"
|
|
|
|
|
run.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
(run / "config.yaml").write_text(text)
|
|
|
|
|
return run / "config.yaml", state
|
|
|
|
|
|
|
|
|
|
|
2026-11-19 10:00:02 +01:00
|
|
|
def main():
|
|
|
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
2026-11-19 10:52:46 +01:00
|
|
|
p.add_argument("station", help="serial number: the folder data_diff/<SERIAL>/")
|
|
|
|
|
p.add_argument("run", help="the folder data_diff/<SERIAL>/runs/<run>/, holding config.yaml")
|
2026-11-19 10:00:02 +01:00
|
|
|
p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD")
|
|
|
|
|
p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD")
|
|
|
|
|
p.add_argument("--jobs", type=int, default=8, help="processes of the re-measurement (default 8; 0: one per core)")
|
|
|
|
|
p.add_argument("--data-diff", type=Path, default=ROOT / "data_diff")
|
2026-11-19 10:52:46 +01:00
|
|
|
p.add_argument("--new", action="store_true", help="create the run, empty: its config.yaml only (processing/new_run.sh)")
|
2026-11-19 10:00:02 +01:00
|
|
|
a = p.parse_args()
|
2026-11-19 10:52:46 +01:00
|
|
|
src = a.data_diff / a.station
|
|
|
|
|
run = src / "runs" / a.run
|
|
|
|
|
if a.new:
|
|
|
|
|
try:
|
|
|
|
|
path, state = new_run(src, a.run)
|
|
|
|
|
except FileExistsError as e:
|
|
|
|
|
sys.exit(str(e))
|
|
|
|
|
print(f"{path}: written{', but optical has local changes: the run will stop until they are committed' if state['dirty'] else ''}"
|
|
|
|
|
f"\nRun it: processing/run_config.sh {a.station} {a.run}")
|
|
|
|
|
return
|
|
|
|
|
if not (run / "config.yaml").is_file():
|
|
|
|
|
sys.exit(f"No {run / 'config.yaml'}: create the run first, processing/new_run.sh {a.station} {a.run}")
|
|
|
|
|
config = load_config(run / "config.yaml")
|
|
|
|
|
version = (config.get("emulator") or {}).get("version")
|
|
|
|
|
if version is not None: # the code that makes the run is the one config.yaml names
|
|
|
|
|
state = optical_state()
|
|
|
|
|
want = subprocess.run(["git", "-C", str(OPTICAL), "rev-parse", "--verify", "--quiet", f"{version}^{{commit}}"],
|
|
|
|
|
capture_output=True, text=True).stdout.strip()
|
|
|
|
|
if not want:
|
|
|
|
|
sys.exit(f"emulator.version {version}: not a commit of {OPTICAL}")
|
|
|
|
|
if want != state["commit"] or state["dirty"]:
|
|
|
|
|
sys.exit(f"emulator.version {version}, but {OPTICAL} is at {state['commit'][:7]}"
|
|
|
|
|
f"{' with local changes' if state['dirty'] else ''}: check {version} out, or change the version")
|
|
|
|
|
|
|
|
|
|
days = sorted(f for f in (src / "images_diff" / "index").glob(f"{a.station}_diff_*.csv")
|
|
|
|
|
if (not a.start or f.stem[-8:] >= a.start.replace("-", ""))
|
|
|
|
|
and (not a.end or f.stem[-8:] < a.end.replace("-", "")))
|
|
|
|
|
rows = [r for f in days for r in csv.DictReader(open(f, newline="")) if r["file"]]
|
|
|
|
|
if not rows:
|
|
|
|
|
sys.exit(f"{a.station}: no differential image indexed in the range, nothing done")
|
|
|
|
|
sensors = json.loads((src / "images_diff" / "sensors.json").read_text())
|
|
|
|
|
unit = json.loads((src / "images_diff" / "unit.json").read_text())
|
|
|
|
|
cam = camera_settings(config, latest(unit["properties"]))
|
|
|
|
|
props = apply_patterns(cloud_properties(sensors), config.get("patterns"), src / "images_diff")
|
|
|
|
|
print(f"{a.station} {a.run}: {len(rows)} images, {days[0].stem[-8:]} to {days[-1].stem[-8:]}, {len(props)} "
|
|
|
|
|
f"patterns, camera {cam}, optical {optical_state()}")
|
2026-11-19 10:00:02 +01:00
|
|
|
|
|
|
|
|
import reprocess_images
|
|
|
|
|
|
2026-11-19 10:52:46 +01:00
|
|
|
sp = run / "patterns.json" # kept: the target configuration the run was made with, templates included
|
|
|
|
|
sp.write_text(json.dumps(props, indent=1))
|
|
|
|
|
pd.DataFrame([{"sensor_name": p.get("sensor_name"), "pattern_type": p.get("pattern_type"), # pump.py's columns
|
|
|
|
|
"distance_mm": p.get("distance_mm"), "mm_per_px": p.get("distance_convertion_factor")}
|
|
|
|
|
for p in props.values()]).sort_values("sensor_name").to_csv(run / "targets.csv", index=False)
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp: # the emulator's other files: not kept
|
|
|
|
|
times, remeasured = Path(tmp) / "times.csv", Path(tmp) / "rows.csv"
|
2026-11-19 10:00:02 +01:00
|
|
|
with open(times, "w", newline="") as fp:
|
|
|
|
|
w = csv.writer(fp)
|
|
|
|
|
w.writerow(["file", "time_utc"])
|
|
|
|
|
w.writerows((r["file"], r["time_utc"]) for r in rows)
|
|
|
|
|
reprocess_images.main(["--sensor-properties", str(sp), "--times", str(times), "--out", str(remeasured),
|
|
|
|
|
"--jobs", str(a.jobs)] + [str(src / "images_diff" / r["file"]) for r in rows])
|
|
|
|
|
adv, camera = correct(pd.read_csv(remeasured), props, cam)
|
2026-11-19 10:52:46 +01:00
|
|
|
write_run(adv, camera, run, a.station)
|
|
|
|
|
print(f"{a.station} {a.run}: {len(adv)} rows, {adv['corr_x'].notna().sum()} corrected, {len(camera)} camera rows "
|
|
|
|
|
f"-> {run}")
|
2026-11-19 10:00:02 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def check():
|
|
|
|
|
"""`processing/run_config.sh --check`: the configuration rules and a synthetic correction through optical."""
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
2026-11-19 10:52:46 +01:00
|
|
|
f = Path(tmp) / "config.yaml"
|
|
|
|
|
for text in ("camera: {algoritm: ransac}", "pattern: {}", "stations: {S: {}}", "patterns: [PM_G]",
|
|
|
|
|
"emulator: {versio: 65a4c70}"):
|
2026-11-19 10:00:02 +01:00
|
|
|
f.write_text(text)
|
|
|
|
|
try:
|
|
|
|
|
load_config(f)
|
|
|
|
|
raise AssertionError(f"accepted: {text}")
|
|
|
|
|
except ValueError:
|
|
|
|
|
pass
|
2026-11-19 10:52:46 +01:00
|
|
|
f.write_text("")
|
|
|
|
|
assert load_config(f) == {} # an empty file: the cloud's processing
|
|
|
|
|
f.write_text("camera: {mode: rotation, focal_distance: 6.3}\npatterns: {T1: {error_threshold: 0.9}}\n"
|
|
|
|
|
"emulator: {version: '65a4c70'}")
|
2026-11-19 10:00:02 +01:00
|
|
|
c = load_config(f)
|
2026-11-19 10:52:46 +01:00
|
|
|
assert c["emulator"]["version"] == "65a4c70"
|
|
|
|
|
cam = camera_settings(c, {"focal_distance": 6.2, "pixel_size": 0.003, "other": 1})
|
2026-11-19 10:00:02 +01:00
|
|
|
assert (cam["focal_distance"], cam["pixel_size"], cam["mode"], cam["algorithm"]) == (6.3, 0.003, "rotation",
|
|
|
|
|
"ransac"), cam
|
2026-11-19 10:52:46 +01:00
|
|
|
assert camera_settings({}, {})["focal_distance"] == 6.0
|
2026-11-19 10:00:02 +01:00
|
|
|
|
|
|
|
|
pos = {"R1": (100.0, 100.0), "R2": (900.0, 120.0), "R3": (500.0, 800.0), "T1": (300.0, 400.0), "T2": (700.0, 500.0)}
|
|
|
|
|
props = {str(i): {"sensor_name": n, "pattern_type": "global_ref" if n[0] == "R" else "target", "x": x, "y": y,
|
|
|
|
|
"error_threshold": 0.7, "distance_convertion_factor": 2.0, "corners": [[0, 0], [10, 10]]}
|
|
|
|
|
for i, (n, (x, y)) in enumerate(pos.items())}
|
|
|
|
|
sensors = [{"id": int(i), "type": MTYPE, "properties": [{"name": k, "value": v} for k, v in {**q, "pattern": "b"}
|
|
|
|
|
.items()]} for i, q in props.items()] + [{"id": 9, "type": "camera_movement", "properties": []}]
|
|
|
|
|
assert set(cloud_properties(sensors)) == set(props)
|
|
|
|
|
for bad in ({"NOPE": {"error_threshold": 1}}, {"T1": {"error_treshold": 1}}):
|
|
|
|
|
try:
|
|
|
|
|
apply_patterns(props, bad, Path("."))
|
|
|
|
|
raise AssertionError(f"accepted: {bad}")
|
|
|
|
|
except ValueError:
|
|
|
|
|
pass
|
|
|
|
|
assert apply_patterns(props, {"T1": {"error_threshold": 0.9}}, Path("."))["3"]["error_threshold"] == 0.9
|
|
|
|
|
|
|
|
|
|
t0, t1 = "2026-11-03T10:00:07.25+00:00", "2026-11-03T10:15:07.25+00:00"
|
|
|
|
|
rows = []
|
|
|
|
|
for t, (dx, dy) in ((t0, (0.0, 0.0)), (t1, (1.0, 0.5))): # the whole camera shifts by (1, 0.5) px at t1
|
|
|
|
|
for i, q in props.items():
|
|
|
|
|
rows.append({"time_utc": t, "pattern_id": int(i), "x": q["x"] + dx, "y": q["y"] + dy, "peak": 1.0,
|
|
|
|
|
"error": 0.9 if (t == t1 and q["sensor_name"] == "T2") else 0.2})
|
|
|
|
|
for per_epoch in (True, False):
|
|
|
|
|
adv, camera = correct(pd.DataFrame(rows), props, {**DEFAULTS, "per_epoch": per_epoch})
|
|
|
|
|
assert list(adv.columns) == COLUMNS and len(adv) == 10, adv.columns
|
|
|
|
|
a = adv.set_index(["time_utc", "sensor_name"])
|
|
|
|
|
t1x = pd.Timestamp(t1)
|
|
|
|
|
assert a.loc[(t1x, "T1"), ["corr_x", "corr_y"]].abs().max() < 1e-3, a # static target: no displacement
|
|
|
|
|
assert a.loc[(t1x, "T1"), "relative_x"] == 1.0 and a.loc[(t1x, "T1"), "corr_x_mm"] == \
|
|
|
|
|
2.0 * a.loc[(t1x, "T1"), "corr_x"]
|
|
|
|
|
assert np.isnan(a.loc[(t1x, "T2"), "corr_x"]) and a.loc[(t1x, "T2"), "x"] == 701.0 # rejected: raw x kept
|
|
|
|
|
assert a["sigma0"].notna().all() and a.loc[(t1x, "R1"), "inlier"] == a.loc[(t1x, "R1"), "inlier"], a
|
|
|
|
|
assert len(camera) == 2 and str(camera["time_utc"][1]) == "2026-11-03 10:15:00+00:00", camera
|
|
|
|
|
assert list(camera.columns)[:4] == ["time_utc", "median_pan", "min_pan", "max_pan"], camera.columns
|
|
|
|
|
print("ok")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
check() if sys.argv[1:] == ["--check"] else main()
|