214 lines
10 KiB
Python
214 lines
10 KiB
Python
"""Reading a station folder (standard library only): the one reader of the app, the reading tools and the detectors.
|
||
|
||
Formats: tools/DATA.md. Every time is UTC.
|
||
"""
|
||
import csv
|
||
import glob
|
||
import os
|
||
import re
|
||
import statistics
|
||
from datetime import datetime, timezone
|
||
|
||
# Numeric columns of the "advanced" export; an empty cell becomes None.
|
||
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
|
||
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
|
||
"corr_x_mm", "corr_y_mm", # corr × mm_per_px of the sensor (load_targets); None without a sure factor
|
||
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
|
||
# Camera pose (camera/, pumping): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
|
||
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
|
||
IMAGE_RE = re.compile(r"^(\d{8}T\d{4})Z_")
|
||
|
||
|
||
def _header(path):
|
||
with open(path, newline="") as f:
|
||
return next(csv.reader(f), [])
|
||
|
||
|
||
def find_files(folder):
|
||
"""Station folder -> {'advanced', 'hourly', 'camera', 'images', 'others'}: sorted paths, subfolders included."""
|
||
out = {"advanced": [], "hourly": [], "camera": [], "images": [], "others": []}
|
||
for p in sorted(glob.glob(os.path.join(folder, "**", "*"), recursive=True)):
|
||
if os.path.isdir(p) or "images_diff" in p.split(os.sep): # differential images of the pumping (--diff): not camera views
|
||
continue
|
||
header = _header(p) if p.endswith(".csv") else []
|
||
if p.lower().endswith((".jpg", ".jpeg", ".png")):
|
||
out["images"].append(p)
|
||
elif "sensor_name" in header and "corr_x" in header: # quality/ (optical_emulator) has sensor_name, not corr_x
|
||
out["advanced"].append(p)
|
||
elif "timestamp" in header:
|
||
out["hourly"].append(p)
|
||
elif "median_pan" in header:
|
||
out["camera"].append(p)
|
||
else:
|
||
out["others"].append(p)
|
||
return out
|
||
|
||
|
||
def _number(v):
|
||
v = v.strip()
|
||
return float(v) if v else None
|
||
|
||
|
||
def load(folder):
|
||
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
|
||
|
||
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None). corr_x_mm, corr_y_mm
|
||
are corr_x, corr_y × the sensor's mm_per_px of load_targets, whatever the file holds (a factor of the pumping may
|
||
belong to a namesake sensor): None without a sure factor.
|
||
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
|
||
for a target but leaves some filled (often absurd) for a reference; they are emptied here, with their mm.
|
||
"""
|
||
by = {}
|
||
factor = {s: t["mm_per_px"] for s, t in load_targets(folder).items()}
|
||
for path in find_files(folder)["advanced"]:
|
||
with open(path, newline="") as f:
|
||
for r in csv.DictReader(f):
|
||
m = {c: _number(r.get(c) or "") for c in NUM} # inlier is missing from old exports
|
||
m["t"] = parse_time(r["time_utc"])
|
||
m["pattern_type"] = r["pattern_type"]
|
||
if rejected(m):
|
||
m["corr_x"] = m["corr_y"] = None
|
||
name = r["sensor_name"].strip() # some exports end the name with a newline
|
||
k = factor.get(name)
|
||
for c in ("corr_x", "corr_y"):
|
||
m[c + "_mm"] = None if k is None or m[c] is None else m[c] * k
|
||
by.setdefault(name, {})[m["t"]] = m
|
||
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
|
||
|
||
|
||
UNFILLED_DISTANCE_MM = 1000 # the distance_mm the platform holds when nobody filled it in
|
||
|
||
|
||
def load_targets(folder):
|
||
"""targets.csv (the pumping: the properties of the targets, and of the references once pumped again) -> {sensor:
|
||
{"pattern_type", "distance_mm", "mm_per_px", "lines"}}; {} without the file. A line at distance_mm 1000 is not
|
||
filled in on the platform: ignored. A name may have several lines (several sensors of that name, an older
|
||
configuration of the unit, another lens): a value on which its remaining lines disagree is None, never guessed;
|
||
"lines" says how many lines the name has."""
|
||
path = os.path.join(folder, "targets.csv")
|
||
if not os.path.exists(path):
|
||
return {}
|
||
by = {}
|
||
with open(path, newline="") as f:
|
||
for r in csv.DictReader(f):
|
||
by.setdefault(r["sensor_name"].strip(), []).append(r)
|
||
out = {}
|
||
for s, rows in by.items():
|
||
kept = [r for r in rows if _number(r.get("distance_mm") or "") != UNFILLED_DISTANCE_MM] or rows[:0]
|
||
one = lambda c, cast: (lambda v: v.pop() if len(v) == 1 else None)({cast(r.get(c) or "") for r in kept}) # noqa: E731
|
||
out[s] = {"pattern_type": one("pattern_type", str.strip), "distance_mm": one("distance_mm", _number),
|
||
"mm_per_px": one("mm_per_px", _number), "lines": len(rows)}
|
||
return out
|
||
|
||
|
||
def load_temperature(folder):
|
||
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
|
||
hour; [] without these metrics. The components rise and fall together (the labeler reads "the temperatures of all
|
||
the components"); the median drops the spike of a single one, the computing GPU (D-124)."""
|
||
by = {}
|
||
for path in glob.glob(os.path.join(folder, "metrics", "*_deg_c_*.csv")):
|
||
with open(path, newline="") as f:
|
||
for r in csv.DictReader(f):
|
||
if (v := _number(r.get("metric") or "")) is not None:
|
||
by.setdefault(parse_time(r["time_utc"]), []).append(v)
|
||
return [(t, statistics.median(v)) for t, v in sorted(by.items())]
|
||
|
||
|
||
def load_camera(folder):
|
||
"""Camera movements of the folder -> [pose] sorted by t; pose: t and the POSE columns (float or None).
|
||
|
||
One row per camera registration; the medians are read (min = max in the files received: one reading per minute).
|
||
A measurement instant without a pose: the camera was not registered.
|
||
"""
|
||
by = {}
|
||
for path in find_files(folder)["camera"]:
|
||
with open(path, newline="") as f:
|
||
for r in csv.DictReader(f):
|
||
p = {c: _number(r.get("median_" + c, "")) for c in POSE}
|
||
p["t"] = parse_time(r["time_utc"])
|
||
by[p["t"]] = p
|
||
return [by[t] for t in sorted(by)]
|
||
|
||
|
||
WHERE = ("position", "place", "station", "station_name", "distance_km", "height_m") # text columns of weather/
|
||
|
||
|
||
def load_weather(folder):
|
||
"""The MeteoSwiss weather of the station (weather/, written by the pumping, one CSV per day) -> [hour] sorted by t:
|
||
t, where it was measured (WHERE, text), the measurements (precipitation_mm, temperature_c…: float or None); [] without
|
||
weather (no position in stations.txt, or a place outside Switzerland)."""
|
||
by = {}
|
||
for path in glob.glob(os.path.join(folder, "weather", "*.csv")):
|
||
with open(path, newline="") as f:
|
||
for r in csv.DictReader(f):
|
||
h = {c: v if c in WHERE else _number(v or "") for c, v in r.items() if c != "time_utc"}
|
||
h["t"] = parse_time(r["time_utc"])
|
||
by[h["t"]] = h
|
||
return [by[t] for t in sorted(by)]
|
||
|
||
|
||
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
|
||
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_") # name, then the date of the file (one CSV per metric and per day)
|
||
|
||
|
||
def load_metrics(folder):
|
||
"""Station metrics (metrics/, one CSV per metric and per day) -> {name: [(t, value)]} sorted: numeric values only
|
||
(BusVoltage, sensor_CPU_deg_c, rsrp_dBm…); text metrics (link, mode…) and identifiers (SIM_CCID, ip*) are not
|
||
read. Name: the file's, without the station or the date."""
|
||
# ponytail: text metrics are skipped; read them (link up/down) when a detector needs them
|
||
station, by = os.path.basename(os.path.normpath(folder)) + "_", {}
|
||
for path in glob.glob(os.path.join(folder, "metrics", "*.csv")):
|
||
if not (m := METRIC_RE.match(os.path.basename(path))):
|
||
continue
|
||
name = m.group(1).removeprefix(station)
|
||
if IDENTIFIERS.match(name):
|
||
continue
|
||
with open(path, newline="") as f:
|
||
for r in csv.DictReader(f):
|
||
try:
|
||
v = float((r.get("metric") or "").strip("[]'\" ")) # some metrics come as a list: ['-95']
|
||
except ValueError:
|
||
continue
|
||
by.setdefault(name, {})[parse_time(r["time_utc"])] = v
|
||
return {n: sorted(d.items()) for n, d in sorted(by.items())}
|
||
|
||
|
||
def is_reference(measurements):
|
||
"""Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty
|
||
pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is
|
||
none, filled reproj_err or inlier (references only: tools/DATA.md)."""
|
||
t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None)
|
||
if t is not None:
|
||
return t == "global_ref"
|
||
return any(m["reproj_err"] is not None or m["inlier"] is not None for m in measurements)
|
||
|
||
|
||
def rejected(m):
|
||
"""Measurement rejected by the station: error_value above the sensor's threshold (error_threshold); without these
|
||
two values, an empty corr_x."""
|
||
if m["error_value"] is not None and m["error_threshold"] is not None:
|
||
return m["error_value"] > m["error_threshold"]
|
||
return m["corr_x"] is None
|
||
|
||
|
||
def filter_window(measurements, start, end):
|
||
"""Measurements between start and end inclusive, to the minute (as iso() shows them); None = no bound."""
|
||
return [m for m in measurements if (start is None or m["t"] >= start)
|
||
and (end is None or m["t"].replace(second=0, microsecond=0) <= end)]
|
||
|
||
|
||
def image_time(path):
|
||
"""'20261013T1200Z_10167931.jpg' -> UTC datetime; None if the name does not follow this format (D-045)."""
|
||
m = IMAGE_RE.match(os.path.basename(path))
|
||
return datetime.strptime(m.group(1), "%Y%m%dT%H%M").replace(tzinfo=timezone.utc) if m else None
|
||
|
||
|
||
def parse_time(text):
|
||
"""'2026-01-01T00:00Z' or '2026-01-01 00:00' -> UTC datetime; a time without a zone is read as UTC."""
|
||
t = datetime.fromisoformat(text)
|
||
return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc)
|
||
|
||
|
||
def iso(t):
|
||
return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")
|