diff --git a/README.md b/README.md index 05def78..17eae52 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,11 @@ One detector per anomaly family of the displacements, in `scripts/`: | No data | `no_data.py` | | Displacement | `displacement.py` | -`categories.py` runs them all and classifies their episodes into events, one line per event; `data.py` reads a station folder (`displacement/`, `metrics/`, `camera/`). +`categories.py` runs them all and classifies their episodes into events, one line per event; they read a station folder through `scripts/data.py` of the tools repository, the sibling `~/aimsight-tools`. ```bash python3 scripts/categories.py --from YYYY-MM-DD --to YYYY-MM-DD # --chunk N, --history N -python3 scripts/visibility.py # one detector alone + ``` `thresholds.md` is the registry of the detectors' thresholds and what each was tuned on. diff --git a/bench/score.py b/bench/score.py index aac2399..6dd76a8 100644 --- a/bench/score.py +++ b/bench/score.py @@ -19,7 +19,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) # the detectors import categories # noqa: E402 -from data import iso, load, parse_time # noqa: E402 +from data import iso, load, parse_time # noqa: E402 (the tools repository's reader, on the path through categories) HISTORY = timedelta(days=3) # as the batch command of categories.py diff --git a/bench/sensors.py b/bench/sensors.py index 91efcd5..560bacb 100644 --- a/bench/sensors.py +++ b/bench/sensors.py @@ -11,7 +11,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) import categories # noqa: E402 -from data import iso, load # noqa: E402 +from data import iso, load # noqa: E402 (the tools repository's reader, on the path through categories) p = argparse.ArgumentParser(usage=__doc__) p.add_argument("folder") diff --git a/scripts/categories.py b/scripts/categories.py index 2044655..41ce509 100644 --- a/scripts/categories.py +++ b/scripts/categories.py @@ -31,12 +31,14 @@ Usage: python3 scripts/categories.py (one line per event: fam import functools import importlib.util import inspect -import os import sys from datetime import datetime, timedelta, timezone from pathlib import Path -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it +# The detectors next to this file, and the single reader of the measurements: data.py of the tools repository, the +# sibling of this one (one copy of each repository in the home directory). +HERE = Path(__file__).resolve().parent +sys.path[:0] = [str(HERE), str(HERE.parents[1] / "aimsight-tools" / "scripts")] from data import iso, load, load_camera, load_metrics, load_temperature # noqa: E402 # What a detector can read from the station besides the measurements, if it declares it in detect (D-124, D-134): diff --git a/scripts/data.py b/scripts/data.py deleted file mode 100644 index f1dd93d..0000000 --- a/scripts/data.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Reading a station folder (standard library only). Every time is UTC. - -Column formats: DATA.md in the tools repository. -""" -import csv -import glob -import os -import re -import statistics -from datetime import datetime, timezone - -# Numeric columns of the "advanced" export; an empty cell becomes None. -NUM = ("x", "y", "error_value", "error_threshold", "relative_x", "relative_y", - "corr_x", "corr_y", "reproj_err", "inlier", "sigma0", - "peak", "flux", "sharpness", "fwhm", "dominance", "snr") # image quality: recent cameras only (LENZSPITZE) -# Camera pose (camera/): pan, tilt, rotation [deg], scale, sigma0 (registration residual). -POSE = ("pan", "tilt", "rotation", "scale", "sigma0") -IDENTIFIERS = re.compile(r"^(SIM_CCID|ip(_.*)?)$") # station identifiers: never read -METRIC_RE = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_") - - -def _header(path): - with open(path, newline="") as f: - return next(csv.reader(f), []) - - -def find_files(folder): - """Station folder -> {'advanced': [...], 'camera': [...]}: sorted CSV paths, subfolders included.""" - out = {"advanced": [], "camera": []} - for p in sorted(glob.glob(os.path.join(folder, "**", "*.csv"), recursive=True)): - header = _header(p) - if "sensor_name" in header and "corr_x" in header: # quality/ (optical_emulator) has sensor_name, not corr_x - out["advanced"].append(p) - elif "median_pan" in header and "timestamp" not in header: # timestamp: the hourly export - out["camera"].append(p) - return out - - -def _number(v): - v = v.strip() - return float(v) if v else None - - -def load(folder): - """"advanced" exports of the folder -> {sensor: [measurement]}, sorted by t, one reading per instant. - - Measurement: dict with t (UTC datetime), pattern_type, and the NUM columns (float or None). - A rejected measurement (see rejected) has empty corr_x, corr_y, as the column docs say: the station empties them - for a target but leaves some filled (often absurd) for a reference; they are emptied here. - """ - by = {} - for path in find_files(folder)["advanced"]: - with open(path, newline="") as f: - for r in csv.DictReader(f): - m = {c: _number(r.get(c) or "") for c in NUM} # inlier is missing from old exports - m["t"] = parse_time(r["time_utc"]) - m["pattern_type"] = r["pattern_type"] - if rejected(m): - m["corr_x"] = m["corr_y"] = None - by.setdefault(r["sensor_name"].strip(), {})[m["t"]] = m # some exports end the name with a newline - return {s: [d[t] for t in sorted(d)] for s, d in by.items()} - - -def load_temperature(folder): - """Station board temperatures (metrics/, sensor_*_deg_c) -> [(t, °C)] sorted: the median of the components at each - hour; [] without these metrics. The components rise and fall together; the median drops the spike of a single - one, the computing GPU (D-124).""" - by = {} - for path in glob.glob(os.path.join(folder, "metrics", "*_deg_c_*.csv")): - with open(path, newline="") as f: - for r in csv.DictReader(f): - if (v := _number(r.get("metric") or "")) is not None: - by.setdefault(parse_time(r["time_utc"]), []).append(v) - return [(t, statistics.median(v)) for t, v in sorted(by.items())] - - -def load_camera(folder): - """Camera movements of the folder -> [pose] sorted by t; pose: t and the POSE columns (float or None). - - One row per camera registration; the medians are read (min = max in the files received: one reading per minute). - A measurement instant without a pose: the camera was not registered. - """ - by = {} - for path in find_files(folder)["camera"]: - with open(path, newline="") as f: - for r in csv.DictReader(f): - p = {c: _number(r.get("median_" + c, "")) for c in POSE} - p["t"] = parse_time(r["time_utc"]) - by[p["t"]] = p - return [by[t] for t in sorted(by)] - - -def load_metrics(folder): - """Station metrics (metrics/, one CSV per metric and per day) -> {name: [(t, value)]} sorted: numeric values only - (BusVoltage, sensor_CPU_deg_c, rsrp_dBm…); text metrics (link, mode…) and identifiers (SIM_CCID, ip*) are not - read. Name: the file's, without the station or the date.""" - # ponytail: text metrics are skipped; read them (link up/down) when a detector needs them - station, by = os.path.basename(os.path.normpath(folder)) + "_", {} - for path in glob.glob(os.path.join(folder, "metrics", "*.csv")): - if not (m := METRIC_RE.match(os.path.basename(path))): - continue - name = m.group(1).removeprefix(station) - if IDENTIFIERS.match(name): - continue - with open(path, newline="") as f: - for r in csv.DictReader(f): - try: - v = float((r.get("metric") or "").strip("[]'\" ")) # some metrics come as a list: ['-95'] - except ValueError: - continue - by.setdefault(name, {})[parse_time(r["time_utc"])] = v - return {n: sorted(d.items()) for n, d in sorted(by.items())} - - -def is_reference(measurements): - """Reference of the compensation network (pattern_type global_ref), otherwise a target. Some exports have an empty - pattern_type, over the whole period or at the start: the first filled pattern_type is read then and, if there is - none, filled reproj_err or inlier (references only).""" - t = next((m["pattern_type"] for m in measurements if m["pattern_type"]), None) - if t is not None: - return t == "global_ref" - return any(m["reproj_err"] is not None or m["inlier"] is not None for m in measurements) - - -def rejected(m): - """Measurement rejected by the station: error_value above the sensor's threshold (error_threshold); without these - two values, an empty corr_x.""" - if m["error_value"] is not None and m["error_threshold"] is not None: - return m["error_value"] > m["error_threshold"] - return m["corr_x"] is None - - -def parse_time(text): - """'2026-01-01T00:00Z' or '2026-01-01 00:00' -> UTC datetime; a time without a zone is read as UTC.""" - t = datetime.fromisoformat(text) - return t.astimezone(timezone.utc) if t.tzinfo else t.replace(tzinfo=timezone.utc) - - -def iso(t): - return t.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%MZ") diff --git a/scripts/displacement.py b/scripts/displacement.py index e70c6f0..66f968a 100644 --- a/scripts/displacement.py +++ b/scripts/displacement.py @@ -32,12 +32,9 @@ belongs to the Target issue family: episodes() returns it, detect() leaves it to daily levels); otherwise it is ignored (neither back nor too recent: only jumps of K noises or more are). """ import statistics -import sys from datetime import timedelta -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) # data.py, next to it -import data # noqa: E402 +import data K = 20 # tuned on E-001: at 15, two healthy targets are greyed (17 and 18 noises); the trains of E-005 (17 and 19 # noises) are caught by visibility.py: do not lower K @@ -319,9 +316,3 @@ def drift(measurements): "text": f"{s} drifts slowly and steadily: {d:+.2f} px in {n} days. Keep watching it.", "ignore": False, "sensors": [s]} for d, n, a, b in r] return out - - -if __name__ == "__main__": - for ep in detect(data.load(sys.argv[1])): - print(ep["start"], ep["end"], "ignore" if ep["ignore"] else "KEEP ", ep["label"], "|", len(ep["sensors"]), - ",".join(ep["sensors"][:6]), "|", ep["text"]) diff --git a/scripts/no_data.py b/scripts/no_data.py index c33dcf5..660681d 100644 --- a/scripts/no_data.py +++ b/scripts/no_data.py @@ -10,14 +10,10 @@ End of the window read (end, when the caller gives it: the chunked run of catego measurement and end, the episode running up to end (CLARIDEN, batch 01: 44 h without a measurement up to the end of the batch, not reported). No measurement in the export does not mean the station stopped: the text only says that the export has nothing. -Usage: python3 scripts/no_data.py (prints the episodes) """ -import os import statistics -import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it -from data import iso, load # noqa: E402 +from data import iso MISSING = 2 # more than 2 missing steps is a hole (user's rule, 2026-10-23) @@ -44,8 +40,3 @@ def detect(measurements, end=None): "text": f"No measurement in the export for {duration(g)}{tail} (usually one every {duration(step)}).", "ignore": False, "sensors": []} for a, b, g, tail in out if g > (MISSING + 1) * step] - - -if __name__ == "__main__": - for e in detect(load(sys.argv[1])): - print(e) diff --git a/scripts/obstruction.py b/scripts/obstruction.py index 2ce9711..f710072 100644 --- a/scripts/obstruction.py +++ b/scripts/obstruction.py @@ -7,15 +7,10 @@ A sensor already in an overlapping Target issue episode (Degraded target, Target its curve is already to be ignored, the family is Target issue (boundary "Target issue or an episode of another family" of the families document; wf-improve session Obstruction, mod. 1: S1, problem targets of FLORIDA, NORDEND, SNCF01, NIEDERHORN, JUNGFRAU, CLARIDEN cut into Target obscured; S3, their isolated rejections as Brief obstruction). -Usage: python3 scripts/obstruction.py (prints the episodes) """ -import os -import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py and the detectors, next to it -import target_issue # noqa: E402 -import visibility # noqa: E402 -from data import load # noqa: E402 +import target_issue +import visibility def detect(measurements): @@ -29,8 +24,3 @@ def detect(measurements): if (e := {**e, "sensors": [s for s in e["sensors"] if s not in taken]})["sensors"]: # minus the sensors out.append(e) # already in Target issue return sorted(out, key=lambda e: e["start"]) - - -if __name__ == "__main__": - for e in detect(load(sys.argv[1])): - print(e["start"], "→", e["end"], e["label"], f"| {len(e['sensors'])}:", ",".join(e["sensors"][:8]), "|", e["text"]) diff --git a/scripts/target_issue.py b/scripts/target_issue.py index 41789a6..218b7ed 100644 --- a/scripts/target_issue.py +++ b/scripts/target_issue.py @@ -34,15 +34,11 @@ Stuck target (E-011, FLORIDA: a target defined on the prism of a reference): per corr_y (median of the gaps from one measurement to the next) under STUCK times the median noise of the other targets, on both axes; one day is enough (a day without noise does not happen by accident); label "Target stuck". On E-011: C2 at 7e-5 px and C3 at 2e-4 against 0.013 for the others. -Usage: python3 scripts/target_issue.py (prints the episodes) """ -import os import statistics -import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py and the detectors, next to it -import displacement # noqa: E402 -from data import is_reference, iso, load # noqa: E402 +import displacement +from data import is_reference, iso CALM = 0.25 # share of the other sensors rejected, at most, for a calm instant SHARE = 0.5 # half of the calm instants rejected: "most of the time" @@ -177,8 +173,3 @@ def stuck(measurements, min_instants): def detect(measurements): return sorted((e for e in episodes(measurements) + displacement.episodes(measurements) if e["kind"] == "Target issue"), key=lambda e: e["start"]) - - -if __name__ == "__main__": - for e in detect(load(sys.argv[1])): - print(e) diff --git a/scripts/thermal_displacement.py b/scripts/thermal_displacement.py index 0aab52c..c972d23 100644 --- a/scripts/thermal_displacement.py +++ b/scripts/thermal_displacement.py @@ -47,13 +47,10 @@ so that the threshold does not depend on the cadence. The targets may move in op Without a pose or without a temperature, nothing. The heat is not taken: in fine weather, it is the day's cycle. """ import math -import os import statistics -import sys from datetime import datetime, timedelta, timezone -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it -from data import iso, load # noqa: E402 +from data import iso MIN_HOURS = 20 # a UTC day counts in the profile if it has at least 20 measured hours PARTIAL = 12 # an incomplete day of at least 12 hours gets its episode if it looks like the profile @@ -280,16 +277,3 @@ def _episodes(measurements, cols, name, camera): episodes.append({"start": iso(start), "end": iso(end), "kind": "Thermal displacement", "label": label, "text": "", "ignore": False, "sensors": sensors}) return episodes - - -if __name__ == "__main__": - measurements = load(sys.argv[1]) - for e in detect(measurements): - print(e["start"], "→", e["end"], e["label"], len(e["sensors"]), "targets") - for cols in (("corr_x", "corr_y"), ("x", "y")): - concerned, n = cycle(measurements, cols) - k = len({c[0] for c in concerned}) - amp = statistics.median(max(c[2]) - min(c[2]) for c in concerned) if concerned else 0 - yes, evaluated = sine(measurements, cols) - print(f"{cols[0][:-2] or 'raw'}: targets {k}/{n}, median amplitude {amp:.3f} px, warm hours (UTC) " - f"{warm_hours(concerned) if concerned else '-'}; sinusoid: targets {len(yes)}/{len(evaluated)}") diff --git a/scripts/visibility.py b/scripts/visibility.py index 481dcf7..c82e41d 100644 --- a/scripts/visibility.py +++ b/scripts/visibility.py @@ -47,13 +47,10 @@ A simpler version (share of targets rejected above their threshold, without comp was tried on 2026-10-24 and dropped: worse on five data sets out of six (git history of Support-Agent, detectors/archive/visibilite_simple.py). """ -import os import statistics -import sys from datetime import timedelta -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # data.py, next to it -from data import iso, load # noqa: E402 +from data import iso K_MAD = 10 # tuned on E-001 (fog: 11 to 57 MAD) and E-005 (rain, trains: 10 to 70 MAD) FACTOR = 2 # drops sensors with a near-zero MAD (E-004: PWD_REF10 ×1.6 at 14 MAD, E-005) @@ -337,8 +334,3 @@ def heat_shimmer(measurements): "sensors": sorted(s for s, v in agit.items() if v >= RATIO), }) return episodes - - -if __name__ == "__main__": - for e in detect(load(sys.argv[1])): - print(e["start"], "→", e["end"], e["label"], f"| {len(e['sensors'])}:", ",".join(e["sensors"][:8]), "|", e["text"])