data: corr_x_mm, corr_y_mm (corr x mm_per_px of targets.csv, load_targets; none for a name whose lines disagree, a distance of 1000 not filled in) and load_weather (weather/, pumped); inventory shows the distance and mm/px of each sensor; summary reads the mm; DATA.md: the targets, the weather

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-18 07:17:07 +01:00
1 parent e2c34c711c
commit 7bb0f4e8d1
5 files changed
+87 -11

No files matched your search

+53 -3
View File
@@ -12,6 +12,7 @@ from datetime import datetime, timezone
# Numeric columns of the "advanced" export; an empty cell becomes None.
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
"corr_x_mm", "corr_y_mm", # corr × mm_per_px of the sensor (load_targets); None without a sure factor
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
# Camera pose (camera/, pumping): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
@@ -51,11 +52,14 @@ def _number(v):
def load(folder):
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None).
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None). corr_x_mm, corr_y_mm
are corr_x, corr_y × the sensor's mm_per_px of load_targets, whatever the file holds (a factor of the pumping may
belong to a namesake sensor): None without a sure factor.
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
for a target but leaves some filled (often absurd) for a reference; they are emptied here.
for a target but leaves some filled (often absurd) for a reference; they are emptied here, with their mm.
"""
by = {}
factor = {s: t["mm_per_px"] for s, t in load_targets(folder).items()}
for path in find_files(folder)["advanced"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
@@ -64,10 +68,39 @@ def load(folder):
m["pattern_type"] = r["pattern_type"]
if rejected(m):
m["corr_x"] = m["corr_y"] = None
by.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # some exports end the name with a newline
name = r["sensor_name"].strip() # some exports end the name with a newline
k = factor.get(name)
for c in ("corr_x", "corr_y"):
m[c + "_mm"] = None if k is None or m[c] is None else m[c] * k
by.setdefault(name, {})[m["t"]] = m
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
UNFILLED_DISTANCE_MM = 1000 # the distance_mm the platform holds when nobody filled it in
def load_targets(folder):
"""targets.csv (the pumping: the properties of the targets, and of the references once pumped again) -> {sensor:
{"pattern_type", "distance_mm", "mm_per_px", "lines"}}; {} without the file. A line at distance_mm 1000 is not
filled in on the platform: ignored. A name may have several lines (several sensors of that name, an older
configuration of the unit, another lens): a value on which its remaining lines disagree is None, never guessed;
"lines" says how many lines the name has."""
path = os.path.join(folder, "targets.csv")
if not os.path.exists(path):
return {}
by = {}
with open(path, newline="") as f:
for r in csv.DictReader(f):
by.setdefault(r["sensor_name"].strip(), []).append(r)
out = {}
for s, rows in by.items():
kept = [r for r in rows if _number(r.get("distance_mm") or "") != UNFILLED_DISTANCE_MM] or rows[:0]
one = lambda c, cast: (lambda v: v.pop() if len(v) == 1 else None)({cast(r.get(c) or "") for r in kept}) # noqa: E731
out[s] = {"pattern_type": one("pattern_type", str.strip), "distance_mm": one("distance_mm", _number),
"mm_per_px": one("mm_per_px", _number), "lines": len(rows)}
return out
def load_temperature(folder):
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
hour; [] without these metrics. The components rise and fall together (the labeler reads "the temperatures of all
@@ -97,6 +130,23 @@ def load_camera(folder):
return [by[t] for t in sorted(by)]
WHERE = ("position", "place", "station", "station_name", "distance_km", "height_m") # text columns of weather/
def load_weather(folder):
"""The MeteoSwiss weather of the station (weather/, written by the pumping, one CSV per day) -> [hour] sorted by t:
t, where it was measured (WHERE, text), the measurements (precipitation_mm, temperature_c…: float or None); [] without
weather (no position in stations.txt, or a place outside Switzerland)."""
by = {}
for path in glob.glob(os.path.join(folder, "weather", "*.csv")):
with open(path, newline="") as f:
for r in csv.DictReader(f):
h = {c: v if c in WHERE else _number(v or "") for c, v in r.items() if c != "time_utc"}
h["t"] = parse_time(r["time_utc"])
by[h["t"]] = h
return [by[t] for t in sorted(by)]
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_") # name, then the date of the file (one CSV per metric and per day)