Files
aimsight-detectors/scripts/data.py
T

141 lines
6.3 KiB
Python

"""Reading a station folder (standard library only). Every time is UTC.
Column formats: knowledge/displacement-columns.md in the labeling repository.
"""
import csv
import glob
import os
import re
import statistics
from datetime import datetime, timezone
# Numeric columns of the "advanced" export; an empty cell becomes None.
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
# Camera pose (camera/): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
METRIC_RE = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
def _header(path):
with open(path, newline="") as f:
return next(csv.reader(f), [])
def find_files(folder):
"""Station folder -> {'advanced': [...], 'camera': [...]}: sorted CSV paths, subfolders included."""
out = {"advanced": [], "camera": []}
for p in sorted(glob.glob(os.path.join(folder, "**", "*.csv"), recursive=True)):
header = _header(p)
if "sensor_name" in header and "corr_x" in header: # quality/ (optical_emulator) has sensor_name, not corr_x
out["advanced"].append(p)
elif "median_pan" in header and "timestamp" not in header: # timestamp: the hourly export
out["camera"].append(p)
return out
def _number(v):
v = v.strip()
return float(v) if v else None
def load(folder):
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None).
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
for a target but leaves some filled (often absurd) for a reference; they are emptied here.
"""
by = {}
for path in find_files(folder)["advanced"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
m = {c: _number(r.get(c) or "") for c in NUM} # inlier is missing from old exports
m["t"] = parse_time(r["time_utc"])
m["pattern_type"] = r["pattern_type"]
if rejected(m):
m["corr_x"] = m["corr_y"] = None
by.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # some exports end the name with a newline
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
def load_temperature(folder):
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
hour; [] without these metrics. The components rise and fall together; the median drops the spike of a single
one, the computing GPU (D-124)."""
by = {}
for path in glob.glob(os.path.join(folder, "metrics", "*_deg_c_*.csv")):
with open(path, newline="") as f:
for r in csv.DictReader(f):
if (v := _number(r.get("metric") or "")) is not None:
by.setdefault(parse_time(r["time_utc"]), []).append(v)
return [(t, statistics.median(v)) for t, v in sorted(by.items())]
def load_camera(folder):
"""Camera movements of the folder -> [pose] sorted by t; pose: t and the POSE columns (float or None).
One row per camera registration; the medians are read (min = max in the files received: one reading per minute).
A measurement instant without a pose: the camera was not registered.
"""
by = {}
for path in find_files(folder)["camera"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
p = {c: _number(r.get("median_" + c, "")) for c in POSE}
p["t"] = parse_time(r["time_utc"])
by[p["t"]] = p
return [by[t] for t in sorted(by)]
def load_metrics(folder):
"""Station metrics (metrics/, one CSV per metric and per day) -> {name: [(t, value)]} sorted: numeric values only
(BusVoltage, sensor_CPU_deg_c, rsrp_dBm…); text metrics (link, mode…) and identifiers (SIM_CCID, ip*) are not
read. Name: the file's, without the station or the date."""
# ponytail: text metrics are skipped; read them (link up/down) when a detector needs them
station, by = os.path.basename(os.path.normpath(folder)) + "_", {}
for path in glob.glob(os.path.join(folder, "metrics", "*.csv")):
if not (m := METRIC_RE.match(os.path.basename(path))):
continue
name = m.group(1).removeprefix(station)
if IDENTIFIERS.match(name):
continue
with open(path, newline="") as f:
for r in csv.DictReader(f):
try:
v = float((r.get("metric") or "").strip("[]'\" ")) # some metrics come as a list: ['-95']
except ValueError:
continue
by.setdefault(name, {})[parse_time(r["time_utc"])] = v
return {n: sorted(d.items()) for n, d in sorted(by.items())}
def is_reference(measurements):
"""Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty
pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is
none, filled reproj_err or inlier (references only)."""
t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None)
if t is not None:
return t == "global_ref"
return any(m["reproj_err"] is not None or m["inlier"] is not None for m in measurements)
def rejected(m):
"""Measurement rejected by the station: error_value above the sensor's threshold (error_threshold); without these
two values, an empty corr_x."""
if m["error_value"] is not None and m["error_threshold"] is not None:
return m["error_value"] > m["error_threshold"]
return m["corr_x"] is None
def parse_time(text):
"""'2026-01-01T00:00Z' or '2026-01-01 00:00' -> UTC datetime; a time without a zone is read as UTC."""
t = datetime.fromisoformat(text)
return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc)
def iso(t):
return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")