data: corr_x_mm, corr_y_mm (corr x mm_per_px of targets.csv, load_targets; none for a name whose lines disagree, a distance of 1000 not filled in) and load_weather (weather/, pumped); inventory shows the distance and mm/px of each sensor; summary reads the mm; DATA.md: the targets, the weather

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-18 07:17:07 +01:00
1 parent e2c34c711c
commit 7bb0f4e8d1
5 files changed
+87 -11

No files matched your search

+2 -1
View File
@@ -10,7 +10,8 @@ from datetime import timedelta
from data import filter_window, is_reference, load, parse_time
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err",
"corr_x_mm", "corr_y_mm")
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
STATS = {"median": statistics.median, "min": min, "max": max}
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
+53 -3
View File
@@ -12,6 +12,7 @@ from datetime import datetime, timezone
# Numeric columns of the "advanced" export; an empty cell becomes None.
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
"corr_x_mm", "corr_y_mm", # corr × mm_per_px of the sensor (load_targets); None without a sure factor
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
# Camera pose (camera/, pumping): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
@@ -51,11 +52,14 @@ def _number(v):
def load(folder):
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None).
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None). corr_x_mm, corr_y_mm
are corr_x, corr_y × the sensor's mm_per_px of load_targets, whatever the file holds (a factor of the pumping may
belong to a namesake sensor): None without a sure factor.
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
for a target but leaves some filled (often absurd) for a reference; they are emptied here.
for a target but leaves some filled (often absurd) for a reference; they are emptied here, with their mm.
"""
by = {}
factor = {s: t["mm_per_px"] for s, t in load_targets(folder).items()}
for path in find_files(folder)["advanced"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
@@ -64,10 +68,39 @@ def load(folder):
m["pattern_type"] = r["pattern_type"]
if rejected(m):
m["corr_x"] = m["corr_y"] = None
by.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # some exports end the name with a newline
name = r["sensor_name"].strip() # some exports end the name with a newline
k = factor.get(name)
for c in ("corr_x", "corr_y"):
m[c + "_mm"] = None if k is None or m[c] is None else m[c] * k
by.setdefault(name, {})[m["t"]] = m
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
UNFILLED_DISTANCE_MM = 1000 # the distance_mm the platform holds when nobody filled it in
def load_targets(folder):
"""targets.csv (the pumping: the properties of the targets, and of the references once pumped again) -> {sensor:
{"pattern_type", "distance_mm", "mm_per_px", "lines"}}; {} without the file. A line at distance_mm 1000 is not
filled in on the platform: ignored. A name may have several lines (several sensors of that name, an older
configuration of the unit, another lens): a value on which its remaining lines disagree is None, never guessed;
"lines" says how many lines the name has."""
path = os.path.join(folder, "targets.csv")
if not os.path.exists(path):
return {}
by = {}
with open(path, newline="") as f:
for r in csv.DictReader(f):
by.setdefault(r["sensor_name"].strip(), []).append(r)
out = {}
for s, rows in by.items():
kept = [r for r in rows if _number(r.get("distance_mm") or "") != UNFILLED_DISTANCE_MM] or rows[:0]
one = lambda c, cast: (lambda v: v.pop() if len(v) == 1 else None)({cast(r.get(c) or "") for r in kept}) # noqa: E731
out[s] = {"pattern_type": one("pattern_type", str.strip), "distance_mm": one("distance_mm", _number),
"mm_per_px": one("mm_per_px", _number), "lines": len(rows)}
return out
def load_temperature(folder):
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
hour; [] without these metrics. The components rise and fall together (the labeler reads "the temperatures of all
@@ -97,6 +130,23 @@ def load_camera(folder):
return [by[t] for t in sorted(by)]
WHERE = ("position", "place", "station", "station_name", "distance_km", "height_m") # text columns of weather/
def load_weather(folder):
"""The MeteoSwiss weather of the station (weather/, written by the pumping, one CSV per day) -> [hour] sorted by t:
t, where it was measured (WHERE, text), the measurements (precipitation_mm, temperature_c…: float or None); [] without
weather (no position in stations.txt, or a place outside Switzerland)."""
by = {}
for path in glob.glob(os.path.join(folder, "weather", "*.csv")):
with open(path, newline="") as f:
for r in csv.DictReader(f):
h = {c: v if c in WHERE else _number(v or "") for c, v in r.items() if c != "time_utc"}
h["t"] = parse_time(r["time_utc"])
by[h["t"]] = h
return [by[t] for t in sorted(by)]
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_") # name, then the date of the file (one CSV per metric and per day)
+17 -5
View File
@@ -24,7 +24,8 @@ import statistics
from datetime import datetime, timezone
from common import QUALITY, group, without_pose
from data import METRIC_RE, filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected
from data import (METRIC_RE, filter_window, find_files, image_time, is_reference, iso, load, load_camera, load_targets,
parse_time, rejected)
def span(a, b, n):
@@ -170,7 +171,7 @@ def missing(instants, cadence):
return [(a, b, n) for a, b in zip(instants, instants[1:]) if (n := round((b - a) / cadence) - 1) > 0]
def measurements(by):
def measurements(by, targets=None):
if not by:
print('\nNo measurement: no "advanced" export in this folder.')
return
@@ -189,12 +190,23 @@ def measurements(by):
if empty:
print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled "
"(references only, tools/DATA.md)")
print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}")
targets = targets or {}
if targets:
print(" distance (m) and mm/px: targets.csv (data.load_targets); \"-\": not in it, or not filled in "
"(distance 1000 mm); \"?\": its lines (n) disagree, no corr_x_mm / corr_y_mm")
else:
print(" no targets.csv: no distance, no corr_x_mm / corr_y_mm")
print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}"
f" {'distance':>9} {'mm/px':>8}")
for s in names:
ms = by[s]
threshold = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms}))
t = targets.get(s, {})
mark = f"?({t['lines']})" if t.get("lines", 0) > 1 else "-"
distance = mark if t.get("distance_mm") is None else f"{t['distance_mm'] / 1000:g}"
factor = mark if t.get("mm_per_px") is None else f"{t['mm_per_px']:.3f}"
print(f"{s:10} {'reference' if is_reference(ms) else 'target':10} {ms[0]['pattern_type']:11} "
f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8}")
f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8} {distance:>9} {factor:>8}")
def camera(folder, by, start=None, end=None):
@@ -275,7 +287,7 @@ def main(folder, start=None, end=None, full=False):
contents(folder, f, start, end)
images(f["images"], start, end)
by = load(folder)
measurements(by)
measurements(by, load_targets(folder))
if by:
camera(folder, by)
if by and (start or end):