The single reader: data.py of the tools repository (sibling ~/aimsight-tools), located once in categories.py

scripts/data.py removed; the detectors' standalone CLIs go (categories.py runs them).
Benchmark identical to the baseline.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 15:25:01 +01:00
1 parent 908369b9e9
commit 5e9a7aa9d1
11 files changed
+16 -215

No files matched your search

+2 -2
View File
@@ -15,11 +15,11 @@ One detector per anomaly family of the displacements, in `scripts/`:
| No data | `no_data.py` |
| Displacement | `displacement.py` |
`categories.py` runs them all and classifies their episodes into events, one line per event; `data.py` reads a station folder (`displacement/`, `metrics/`, `camera/`).
`categories.py` runs them all and classifies their episodes into events, one line per event; they read a station folder through `scripts/data.py` of the tools repository, the sibling `~/aimsight-tools`.
```bash
python3 scripts/categories.py <station folder> --from YYYY-MM-DD --to YYYY-MM-DD # --chunk N, --history N
python3 scripts/visibility.py <station folder> # one detector alone
```
`thresholds.md` is the registry of the detectors' thresholds and what each was tuned on.
+1 -1
View File
@@ -19,7 +19,7 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) # the detectors
import categories # noqa: E402
from data import iso, load, parse_time # noqa: E402
from data import iso, load, parse_time # noqa: E402 (the tools repository's reader, on the path through categories)
HISTORY = timedelta(days=3) # as the batch command of categories.py
+1 -1
View File
@@ -11,7 +11,7 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
import categories # noqa: E402
from data import iso, load # noqa: E402
from data import iso, load # noqa: E402 (the tools repository's reader, on the path through categories)
p = argparse.ArgumentParser(usage=__doc__)
p.add_argument("folder")
+4 -2
View File
@@ -31,12 +31,14 @@ Usage: python3 scripts/categories.py <station folder> (one line per event: fam
import functools
import importlib.util
import inspect
import os
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
# The detectors next to this file, and the single reader of the measurements: data.py of the tools repository, the
# sibling of this one (one copy of each repository in the home directory).
HERE = Path(__file__).resolve().parent
sys.path[:0] = [str(HERE), str(HERE.parents[1] / "aimsight-tools" / "scripts")]
from data import iso, load, load_camera, load_metrics, load_temperature # noqa: E402
# What a detector can read from the station besides the measurements, if it declares it in detect (D-124, D-134):
-140
View File
@@ -1,140 +0,0 @@
"""Reading a station folder (standard library only). Every time is UTC.
Column formats: DATA.md in the tools repository.
"""
import csv
import glob
import os
import re
import statistics
from datetime import datetime, timezone
# Numeric columns of the "advanced" export; an empty cell becomes None.
NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y",
"corr_x", "corr_y", "reproj_err", "inlier", "sigma0",
"peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE)
# Camera pose (camera/): pan, tilt, rotation [deg], scale, sigma0 (registration residual).
POSE = ("pan", "tilt", "rotation", "scale", "sigma0")
IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read
METRIC_RE = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
def _header(path):
with open(path, newline="") as f:
return next(csv.reader(f), [])
def find_files(folder):
"""Station folder -> {'advanced': [...], 'camera': [...]}: sorted CSV paths, subfolders included."""
out = {"advanced": [], "camera": []}
for p in sorted(glob.glob(os.path.join(folder, "**", "*.csv"), recursive=True)):
header = _header(p)
if "sensor_name" in header and "corr_x" in header: # quality/ (optical_emulator) has sensor_name, not corr_x
out["advanced"].append(p)
elif "median_pan" in header and "timestamp" not in header: # timestamp: the hourly export
out["camera"].append(p)
return out
def _number(v):
v = v.strip()
return float(v) if v else None
def load(folder):
""""advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant.
Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None).
A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them
for a target but leaves some filled (often absurd) for a reference; they are emptied here.
"""
by = {}
for path in find_files(folder)["advanced"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
m = {c: _number(r.get(c) or "") for c in NUM} # inlier is missing from old exports
m["t"] = parse_time(r["time_utc"])
m["pattern_type"] = r["pattern_type"]
if rejected(m):
m["corr_x"] = m["corr_y"] = None
by.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # some exports end the name with a newline
return {s: [d[t] for t in sorted(d)] for s, d in by.items()}
def load_temperature(folder):
"""Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each
hour; [] without these metrics. The components rise and fall together; the median drops the spike of a single
one, the computing GPU (D-124)."""
by = {}
for path in glob.glob(os.path.join(folder, "metrics", "*_deg_c_*.csv")):
with open(path, newline="") as f:
for r in csv.DictReader(f):
if (v := _number(r.get("metric") or "")) is not None:
by.setdefault(parse_time(r["time_utc"]), []).append(v)
return [(t, statistics.median(v)) for t, v in sorted(by.items())]
def load_camera(folder):
"""Camera movements of the folder -> [pose] sorted by t; pose: t and the POSE columns (float or None).
One row per camera registration; the medians are read (min = max in the files received: one reading per minute).
A measurement instant without a pose: the camera was not registered.
"""
by = {}
for path in find_files(folder)["camera"]:
with open(path, newline="") as f:
for r in csv.DictReader(f):
p = {c: _number(r.get("median_" + c, "")) for c in POSE}
p["t"] = parse_time(r["time_utc"])
by[p["t"]] = p
return [by[t] for t in sorted(by)]
def load_metrics(folder):
"""Station metrics (metrics/, one CSV per metric and per day) -> {name: [(t, value)]} sorted: numeric values only
(BusVoltage, sensor_CPU_deg_c, rsrp_dBm…); text metrics (link, mode…) and identifiers (SIM_CCID, ip*) are not
read. Name: the file's, without the station or the date."""
# ponytail: text metrics are skipped; read them (link up/down) when a detector needs them
station, by = os.path.basename(os.path.normpath(folder)) + "_", {}
for path in glob.glob(os.path.join(folder, "metrics", "*.csv")):
if not (m := METRIC_RE.match(os.path.basename(path))):
continue
name = m.group(1).removeprefix(station)
if IDENTIFIERS.match(name):
continue
with open(path, newline="") as f:
for r in csv.DictReader(f):
try:
v = float((r.get("metric") or "").strip("[]'\" ")) # some metrics come as a list: ['-95']
except ValueError:
continue
by.setdefault(name, {})[parse_time(r["time_utc"])] = v
return {n: sorted(d.items()) for n, d in sorted(by.items())}
def is_reference(measurements):
"""Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty
pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is
none, filled reproj_err or inlier (references only)."""
t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None)
if t is not None:
return t == "global_ref"
return any(m["reproj_err"] is not None or m["inlier"] is not None for m in measurements)
def rejected(m):
"""Measurement rejected by the station: error_value above the sensor's threshold (error_threshold); without these
two values, an empty corr_x."""
if m["error_value"] is not None and m["error_threshold"] is not None:
return m["error_value"] > m["error_threshold"]
return m["corr_x"] is None
def parse_time(text):
"""'2026-01-01T00:00Z' or '2026-01-01 00:00' -> UTC datetime; a time without a zone is read as UTC."""
t = datetime.fromisoformat(text)
return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc)
def iso(t):
return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")
+1 -10
View File
@@ -32,12 +32,9 @@ belongs to the Target issue family: episodes() returns it, detect() leaves it to
daily levels); otherwise it is ignored (neither back nor too recent: only jumps of K noises or more are).
"""
import statistics
import sys
from datetime import timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent)) # data.py, next to it
import data # noqa: E402
import data
K = 20 # tuned on E-001: at 15, two healthy targets are greyed (17 and 18 noises); the trains of E-005 (17 and 19
# noises) are caught by visibility.py: do not lower K
@@ -319,9 +316,3 @@ def drift(measurements):
"text": f"{s} drifts slowly and steadily: {d:+.2f} px in {n} days. Keep watching it.",
"ignore": False, "sensors": [s]} for d, n, a, b in r]
return out
if __name__ == "__main__":
for ep in detect(data.load(sys.argv[1])):
print(ep["start"], ep["end"], "ignore" if ep["ignore"] else "KEEP ", ep["label"], "|", len(ep["sensors"]),
",".join(ep["sensors"][:6]), "|", ep["text"])
+1 -10
View File
@@ -10,14 +10,10 @@ End of the window read (end, when the caller gives it: the chunked run of catego
measurement and end, the episode running up to end (CLARIDEN, batch 01: 44 h without a measurement up to the end of
the batch, not reported). No measurement in the export does not mean the station stopped: the text only says that the
export has nothing.
Usage: python3 scripts/no_data.py <station folder> (prints the episodes)
"""
import os
import statistics
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
from data import iso, load # noqa: E402
from data import iso
MISSING = 2 # more than 2 missing steps is a hole (user's rule, 2026-10-23)
@@ -44,8 +40,3 @@ def detect(measurements, end=None):
"text": f"No measurement in the export for {duration(g)}{tail} (usually one every {duration(step)}).",
"ignore": False, "sensors": []}
for a, b, g, tail in out if g > (MISSING + 1) * step]
if __name__ == "__main__":
for e in detect(load(sys.argv[1])):
print(e)
+2 -12
View File
@@ -7,15 +7,10 @@ A sensor already in an overlapping Target issue episode (Degraded target, Target
its curve is already to be ignored, the family is Target issue (boundary "Target issue or an episode of another family"
of the families document; wf-improve session Obstruction, mod. 1: S1, problem targets of FLORIDA, NORDEND, SNCF01,
NIEDERHORN, JUNGFRAU, CLARIDEN cut into Target obscured; S3, their isolated rejections as Brief obstruction).
Usage: python3 scripts/obstruction.py <station folder> (prints the episodes)
"""
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py and the detectors, next to it
import target_issue # noqa: E402
import visibility # noqa: E402
from data import load # noqa: E402
import target_issue
import visibility
def detect(measurements):
@@ -29,8 +24,3 @@ def detect(measurements):
if (e := {**e, "sensors": [s for s in e["sensors"] if s not in taken]})["sensors"]: # minus the sensors
out.append(e) # already in Target issue
return sorted(out, key=lambda e: e["start"])
if __name__ == "__main__":
for e in detect(load(sys.argv[1])):
print(e["start"], "→", e["end"], e["label"], f"| {len(e['sensors'])}:", ",".join(e["sensors"][:8]), "|", e["text"])
+2 -11
View File
@@ -34,15 +34,11 @@ Stuck target (E-011, FLORIDA: a target defined on the prism of a reference): per
corr_y (median of the gaps from one measurement to the next) under STUCK times the median noise of the other targets, on
both axes; one day is enough (a day without noise does not happen by accident); label "Target stuck". On E-011: C2 at
7e-5 px and C3 at 2e-4 against 0.013 for the others.
Usage: python3 scripts/target_issue.py <station folder> (prints the episodes)
"""
import os
import statistics
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py and the detectors, next to it
import displacement # noqa: E402
from data import is_reference, iso, load # noqa: E402
import displacement
from data import is_reference, iso
CALM = 0.25 # share of the other sensors rejected, at most, for a calm instant
SHARE = 0.5 # half of the calm instants rejected: "most of the time"
@@ -177,8 +173,3 @@ def stuck(measurements, min_instants):
def detect(measurements):
return sorted((e for e in episodes(measurements) + displacement.episodes(measurements) if e["kind"] == "Target issue"),
key=lambda e: e["start"])
if __name__ == "__main__":
for e in detect(load(sys.argv[1])):
print(e)
+1 -17
View File
@@ -47,13 +47,10 @@ so that the threshold does not depend on the cadence. The targets may move in op
Without a pose or without a temperature, nothing. The heat is not taken: in fine weather, it is the day's cycle.
"""
import math
import os
import statistics
import sys
from datetime import datetime, timedelta, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
from data import iso, load # noqa: E402
from data import iso
MIN_HOURS = 20 # a UTC day counts in the profile if it has at least 20 measured hours
PARTIAL = 12 # an incomplete day of at least 12 hours gets its episode if it looks like the profile
@@ -280,16 +277,3 @@ def _episodes(measurements, cols, name, camera):
episodes.append({"start": iso(start), "end": iso(end),
"kind": "Thermal displacement", "label": label, "text": "", "ignore": False, "sensors": sensors})
return episodes
if __name__ == "__main__":
measurements = load(sys.argv[1])
for e in detect(measurements):
print(e["start"], "→", e["end"], e["label"], len(e["sensors"]), "targets")
for cols in (("corr_x", "corr_y"), ("x", "y")):
concerned, n = cycle(measurements, cols)
k = len({c[0] for c in concerned})
amp = statistics.median(max(c[2]) - min(c[2]) for c in concerned) if concerned else 0
yes, evaluated = sine(measurements, cols)
print(f"{cols[0][:-2] or 'raw'}: targets {k}/{n}, median amplitude {amp:.3f} px, warm hours (UTC) "
f"{warm_hours(concerned) if concerned else '-'}; sinusoid: targets {len(yes)}/{len(evaluated)}")
+1 -9
View File
@@ -47,13 +47,10 @@ A simpler version (share of targets rejected above their threshold, without comp
was tried on 2026-10-24 and dropped: worse on five data sets out of six (git history of Support-Agent,
detectors/archive/visibilite_simple.py).
"""
import os
import statistics
import sys
from datetime import timedelta
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it
from data import iso, load # noqa: E402
from data import iso
K_MAD = 10 # tuned on E-001 (fog: 11 to 57 MAD) and E-005 (rain, trains: 10 to 70 MAD)
FACTOR = 2 # drops sensors with a near-zero MAD (E-004: PWD_REF10 ×1.6 at 14 MAD, E-005)
@@ -337,8 +334,3 @@ def heat_shimmer(measurements):
"sensors": sorted(s for s, v in agit.items() if v >= RATIO),
})
return episodes
if __name__ == "__main__":
for e in detect(load(sys.argv[1])):
print(e["start"], "→", e["end"], e["label"], f"| {len(e['sensors'])}:", ",".join(e["sensors"][:8]), "|", e["text"])