Files

214 lines
10 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Reading a station folder (standard library only): the one reader of the app, the reading tools and the detectors.
Formats: tools/DATA.md. Every time is UTC.
"""
import csv
import glob
import os
import re
import statistics
from datetime import datetime, timezone
# Numeric columns of the "advanced" export; an empty cell becomes None.
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
"corr_x_mm", "corr_y_mm", # corr × mm_per_px of the sensor (load_targets); None without a sure factor
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
# Camera pose (camera/, pumping): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
IMAGE_RE = re.compile(r"^(\d{8}T\d{4})Z_")
def _header(path):
with open(path, newline="") as f:
return next(csv.reader(f), [])
def find_files(folder):
"""Station folder -> {'advanced', 'hourly', 'camera', 'images', 'others'}: sorted paths, subfolders included."""
out = {"advanced": [], "hourly": [], "camera": [], "images": [], "others": []}
for p in sorted(glob.glob(os.path.join(folder, "**", "*"), recursive=True)):
if os.path.isdir(p) or "images_diff" in p.split(os.sep): # differential images of the pumping (--diff): not camera views
continue
header = _header(p) if p.endswith(".csv") else []
if p.lower().endswith((".jpg", ".jpeg", ".png")):
out["images"].append(p)
elif "sensor_name" in header and "corr_x" in header: # quality/ (optical_emulator) has sensor_name, not corr_x
out["advanced"].append(p)
elif "timestamp" in header:
out["hourly"].append(p)
elif "median_pan" in header:
out["camera"].append(p)
else:
out["others"].append(p)
return out
def _number(v):
v = v.strip()
return float(v) if v else None
def load(folder):
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None). corr_x_mm, corr_y_mm
are corr_x, corr_y × the sensor's mm_per_px of load_targets, whatever the file holds (a factor of the pumping may
belong to a namesake sensor): None without a sure factor.
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
for a target but leaves some filled (often absurd) for a reference; they are emptied here, with their mm.
"""
by = {}
factor = {s: t["mm_per_px"] for s, t in load_targets(folder).items()}
for path in find_files(folder)["advanced"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
m = {c: _number(r.get(c) or "") for c in NUM} # inlier is missing from old exports
m["t"] = parse_time(r["time_utc"])
m["pattern_type"] = r["pattern_type"]
if rejected(m):
m["corr_x"] = m["corr_y"] = None
name = r["sensor_name"].strip() # some exports end the name with a newline
k = factor.get(name)
for c in ("corr_x", "corr_y"):
m[c + "_mm"] = None if k is None or m[c] is None else m[c] * k
by.setdefault(name, {})[m["t"]] = m
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
UNFILLED_DISTANCE_MM = 1000 # the distance_mm the platform holds when nobody filled it in
def load_targets(folder):
"""targets.csv (the pumping: the properties of the targets, and of the references once pumped again) -> {sensor:
{"pattern_type", "distance_mm", "mm_per_px", "lines"}}; {} without the file. A line at distance_mm 1000 is not
filled in on the platform: ignored. A name may have several lines (several sensors of that name, an older
configuration of the unit, another lens): a value on which its remaining lines disagree is None, never guessed;
"lines" says how many lines the name has."""
path = os.path.join(folder, "targets.csv")
if not os.path.exists(path):
return {}
by = {}
with open(path, newline="") as f:
for r in csv.DictReader(f):
by.setdefault(r["sensor_name"].strip(), []).append(r)
out = {}
for s, rows in by.items():
kept = [r for r in rows if _number(r.get("distance_mm") or "") != UNFILLED_DISTANCE_MM] or rows[:0]
one = lambda c, cast: (lambda v: v.pop() if len(v) == 1 else None)({cast(r.get(c) or "") for r in kept}) # noqa: E731
out[s] = {"pattern_type": one("pattern_type", str.strip), "distance_mm": one("distance_mm", _number),
"mm_per_px": one("mm_per_px", _number), "lines": len(rows)}
return out
def load_temperature(folder):
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
hour; [] without these metrics. The components rise and fall together (the labeler reads "the temperatures of all
the components"); the median drops the spike of a single one, the computing GPU (D-124)."""
by = {}
for path in glob.glob(os.path.join(folder, "metrics", "*_deg_c_*.csv")):
with open(path, newline="") as f:
for r in csv.DictReader(f):
if (v := _number(r.get("metric") or "")) is not None:
by.setdefault(parse_time(r["time_utc"]), []).append(v)
return [(t, statistics.median(v)) for t, v in sorted(by.items())]
def load_camera(folder):
"""Camera movements of the folder -> [pose] sorted by t; pose: t and the POSE columns (float or None).
One row per camera registration; the medians are read (min = max in the files received: one reading per minute).
A measurement instant without a pose: the camera was not registered.
"""
by = {}
for path in find_files(folder)["camera"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
p = {c: _number(r.get("median_" + c, "")) for c in POSE}
p["t"] = parse_time(r["time_utc"])
by[p["t"]] = p
return [by[t] for t in sorted(by)]
WHERE = ("position", "place", "station", "station_name", "distance_km", "height_m") # text columns of weather/
def load_weather(folder):
"""The MeteoSwiss weather of the station (weather/, written by the pumping, one CSV per day) -> [hour] sorted by t:
t, where it was measured (WHERE, text), the measurements (precipitation_mm, temperature_c…: float or None); [] without
weather (no position in stations.txt, or a place outside Switzerland)."""
by = {}
for path in glob.glob(os.path.join(folder, "weather", "*.csv")):
with open(path, newline="") as f:
for r in csv.DictReader(f):
h = {c: v if c in WHERE else _number(v or "") for c, v in r.items() if c != "time_utc"}
h["t"] = parse_time(r["time_utc"])
by[h["t"]] = h
return [by[t] for t in sorted(by)]
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
METRIC_RE = re.compile(r"^(.+?)_(\d{4})_(\d{2})_(\d{2})_") # name, then the date of the file (one CSV per metric and per day)
def load_metrics(folder):
"""Station metrics (metrics/, one CSV per metric and per day) -> {name: [(t, value)]} sorted: numeric values only
(BusVoltage, sensor_CPU_deg_c, rsrp_dBm…); text metrics (link, mode…) and identifiers (SIM_CCID, ip*) are not
read. Name: the file's, without the station or the date."""
# ponytail: text metrics are skipped; read them (link up/down) when a detector needs them
station, by = os.path.basename(os.path.normpath(folder)) + "_", {}
for path in glob.glob(os.path.join(folder, "metrics", "*.csv")):
if not (m := METRIC_RE.match(os.path.basename(path))):
continue
name = m.group(1).removeprefix(station)
if IDENTIFIERS.match(name):
continue
with open(path, newline="") as f:
for r in csv.DictReader(f):
try:
v = float((r.get("metric") or "").strip("[]'\" ")) # some metrics come as a list: ['-95']
except ValueError:
continue
by.setdefault(name, {})[parse_time(r["time_utc"])] = v
return {n: sorted(d.items()) for n, d in sorted(by.items())}
def is_reference(measurements):
"""Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty
pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is
none, filled reproj_err or inlier (references only: tools/DATA.md)."""
t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None)
if t is not None:
return t == "global_ref"
return any(m["reproj_err"] is not None or m["inlier"] is not None for m in measurements)
def rejected(m):
"""Measurement rejected by the station: error_value above the sensor's threshold (error_threshold); without these
two values, an empty corr_x."""
if m["error_value"] is not None and m["error_threshold"] is not None:
return m["error_value"] > m["error_threshold"]
return m["corr_x"] is None
def filter_window(measurements, start, end):
"""Measurements between start and end inclusive, to the minute (as iso() shows them); None = no bound."""
return [m for m in measurements if (start is None or m["t"] >= start)
and (end is None or m["t"].replace(second=0, microsecond=0) <= end)]
def image_time(path):
"""'20261013T1200Z_10167931.jpg' -> UTC datetime; None if the name does not follow this format (D-045)."""
m = IMAGE_RE.match(os.path.basename(path))
return datetime.strptime(m.group(1), "%Y%m%dT%H%M").replace(tzinfo=timezone.utc) if m else None
def parse_time(text):
"""'2026-01-01T00:00Z' or '2026-01-01 00:00' -> UTC datetime; a time without a zone is read as UTC."""
t = datetime.fromisoformat(text)
return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc)
def iso(t):
return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")