data: corr_x_mm, corr_y_mm (corr x mm_per_px of targets.csv, load_targets; none for a name whose lines disagree, a distance of 1000 not filled in) and load_weather (weather/, pumped); inventory shows the distance and mm/px of each sensor; summary reads the mm; DATA.md: the targets, the weather
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e2c34c711c
commit
7bb0f4e8d1
5 files changed
+87
-11
No files matched your search
+2
-1
@@ -10,7 +10,8 @@ from datetime import timedelta
|
||||
|
||||
from data import filter_window, is_reference, load, parse_time
|
||||
|
||||
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err")
|
||||
COLUMNS = ("error_value", "corr_x", "corr_y", "relative_x", "relative_y", "x", "y", "sigma0", "reproj_err",
|
||||
"corr_x_mm", "corr_y_mm")
|
||||
QUALITY = ("peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras; read by quality.py
|
||||
STATS = {"median": statistics.median, "min": min, "max": max}
|
||||
ROUNDING = 1e-9 # below this, a difference between two measurements is the rounding of the station's computation, not noise
|
||||
|
||||
+53
-3
@@ -12,6 +12,7 @@ from datetime import datetime, timezone
|
||||
# Numeric columns of the "advanced" export; an empty cell becomes None.
|
||||
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
|
||||
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
|
||||
"corr_x_mm", "corr_y_mm", # corr × mm_per_px of the sensor (load_targets); None without a sure factor
|
||||
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
|
||||
# Camera pose (camera/, pumping): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
|
||||
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
|
||||
@@ -51,11 +52,14 @@ def _number(v):
|
||||
def load(folder):
|
||||
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
|
||||
|
||||
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None).
|
||||
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None). corr_x_mm, corr_y_mm
|
||||
are corr_x, corr_y × the sensor's mm_per_px of load_targets, whatever the file holds (a factor of the pumping may
|
||||
belong to a namesake sensor): None without a sure factor.
|
||||
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
|
||||
for a target but leaves some filled (often absurd) for a reference; they are emptied here.
|
||||
for a target but leaves some filled (often absurd) for a reference; they are emptied here, with their mm.
|
||||
"""
|
||||
by = {}
|
||||
factor = {s: t["mm_per_px"] for s, t in load_targets(folder).items()}
|
||||
for path in find_files(folder)["advanced"]:
|
||||
with open(path, newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
@@ -64,10 +68,39 @@ def load(folder):
|
||||
m["pattern_type"] = r["pattern_type"]
|
||||
if rejected(m):
|
||||
m["corr_x"] = m["corr_y"] = None
|
||||
by.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # some exports end the name with a newline
|
||||
name = r["sensor_name"].strip() # some exports end the name with a newline
|
||||
k = factor.get(name)
|
||||
for c in ("corr_x", "corr_y"):
|
||||
m[c + "_mm"] = None if k is None or m[c] is None else m[c] * k
|
||||
by.setdefault(name, {})[m["t"]] = m
|
||||
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
|
||||
|
||||
|
||||
UNFILLED_DISTANCE_MM = 1000 # the distance_mm the platform holds when nobody filled it in
|
||||
|
||||
|
||||
def load_targets(folder):
|
||||
"""targets.csv (the pumping: the properties of the targets, and of the references once pumped again) -> {sensor:
|
||||
{"pattern_type", "distance_mm", "mm_per_px", "lines"}}; {} without the file. A line at distance_mm 1000 is not
|
||||
filled in on the platform: ignored. A name may have several lines (several sensors of that name, an older
|
||||
configuration of the unit, another lens): a value on which its remaining lines disagree is None, never guessed;
|
||||
"lines" says how many lines the name has."""
|
||||
path = os.path.join(folder, "targets.csv")
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
by = {}
|
||||
with open(path, newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
by.setdefault(r["sensor_name"].strip(), []).append(r)
|
||||
out = {}
|
||||
for s, rows in by.items():
|
||||
kept = [r for r in rows if _number(r.get("distance_mm") or "") != UNFILLED_DISTANCE_MM] or rows[:0]
|
||||
one = lambda c, cast: (lambda v: v.pop() if len(v) == 1 else None)({cast(r.get(c) or "") for r in kept}) # noqa: E731
|
||||
out[s] = {"pattern_type": one("pattern_type", str.strip), "distance_mm": one("distance_mm", _number),
|
||||
"mm_per_px": one("mm_per_px", _number), "lines": len(rows)}
|
||||
return out
|
||||
|
||||
|
||||
def load_temperature(folder):
|
||||
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
|
||||
hour; [] without these metrics. The components rise and fall together (the labeler reads "the temperatures of all
|
||||
@@ -97,6 +130,23 @@ def load_camera(folder):
|
||||
return [by[t] for t in sorted(by)]
|
||||
|
||||
|
||||
WHERE = ("position", "place", "station", "station_name", "distance_km", "height_m") # text columns of weather/
|
||||
|
||||
|
||||
def load_weather(folder):
|
||||
"""The MeteoSwiss weather of the station (weather/, written by the pumping, one CSV per day) -> [hour] sorted by t:
|
||||
t, where it was measured (WHERE, text), the measurements (precipitation_mm, temperature_c…: float or None); [] without
|
||||
weather (no position in stations.txt, or a place outside Switzerland)."""
|
||||
by = {}
|
||||
for path in glob.glob(os.path.join(folder, "weather", "*.csv")):
|
||||
with open(path, newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
h = {c: v if c in WHERE else _number(v or "") for c, v in r.items() if c != "time_utc"}
|
||||
h["t"] = parse_time(r["time_utc"])
|
||||
by[h["t"]] = h
|
||||
return [by[t] for t in sorted(by)]
|
||||
|
||||
|
||||
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
|
||||
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_") # name, then the date of the file (one CSV per metric and per day)
|
||||
|
||||
|
||||
+17
-5
@@ -24,7 +24,8 @@ import statistics
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from common import QUALITY, group, without_pose
|
||||
from data import METRIC_RE, filter_window, find_files, image_time, is_reference, iso, load, load_camera, parse_time, rejected
|
||||
from data import (METRIC_RE, filter_window, find_files, image_time, is_reference, iso, load, load_camera, load_targets,
|
||||
parse_time, rejected)
|
||||
|
||||
|
||||
def span(a, b, n):
|
||||
@@ -170,7 +171,7 @@ def missing(instants, cadence):
|
||||
return [(a, b, n) for a, b in zip(instants, instants[1:]) if (n := round((b - a) / cadence) - 1) > 0]
|
||||
|
||||
|
||||
def measurements(by):
|
||||
def measurements(by, targets=None):
|
||||
if not by:
|
||||
print('\nNo measurement: no "advanced" export in this folder.')
|
||||
return
|
||||
@@ -189,12 +190,23 @@ def measurements(by):
|
||||
if empty:
|
||||
print(f" pattern_type empty for {len(empty)} sensors: reference when reproj_err or inlier is filled "
|
||||
"(references only, tools/DATA.md)")
|
||||
print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}")
|
||||
targets = targets or {}
|
||||
if targets:
|
||||
print(" distance (m) and mm/px: targets.csv (data.load_targets); \"-\": not in it, or not filled in "
|
||||
"(distance 1000 mm); \"?\": its lines (n) disagree, no corr_x_mm / corr_y_mm")
|
||||
else:
|
||||
print(" no targets.csv: no distance, no corr_x_mm / corr_y_mm")
|
||||
print(f"{'sensor':10} {'type':10} {'pattern':11} {'threshold':>9} {'measurements':>12} {'rejected':>8}"
|
||||
f" {'distance':>9} {'mm/px':>8}")
|
||||
for s in names:
|
||||
ms = by[s]
|
||||
threshold = "/".join(f"{x:g}" for x in sorted({m['error_threshold'] for m in ms}))
|
||||
t = targets.get(s, {})
|
||||
mark = f"?({t['lines']})" if t.get("lines", 0) > 1 else "-"
|
||||
distance = mark if t.get("distance_mm") is None else f"{t['distance_mm'] / 1000:g}"
|
||||
factor = mark if t.get("mm_per_px") is None else f"{t['mm_per_px']:.3f}"
|
||||
print(f"{s:10} {'reference' if is_reference(ms) else 'target':10} {ms[0]['pattern_type']:11} "
|
||||
f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8}")
|
||||
f"{threshold:>9} {len(ms):12} {sum(map(rejected, ms)):8} {distance:>9} {factor:>8}")
|
||||
|
||||
|
||||
def camera(folder, by, start=None, end=None):
|
||||
@@ -275,7 +287,7 @@ def main(folder, start=None, end=None, full=False):
|
||||
contents(folder, f, start, end)
|
||||
images(f["images"], start, end)
|
||||
by = load(folder)
|
||||
measurements(by)
|
||||
measurements(by, load_targets(folder))
|
||||
if by:
|
||||
camera(folder, by)
|
||||
if by and (start or end):
|
||||
|
||||
Reference in new issue
Block a user