"""The data of the Data page: a station's images, its curves, its labels. Data are read as the agents read them, tools/scripts/data.py (the tools repository, through the tools/ link); the detectors' events are those of detectors/scripts/categories.py (events). Read-only on data/; writes only in labeling/{auto,manual}/. """ import bisect import functools import csv import glob import json import os import re import statistics import sys from datetime import datetime, timedelta from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path[:0] = [str(ROOT / "tools" / "scripts"), str(ROOT / "detectors" / "scripts")] import data # noqa: E402 from categories import events # noqa: E402 # data/ and labeling/ are links to ~/aimsight-data/data and ~/aimsight-labeling: no resolve(), the agents see data/ and labeling/ DATA = Path(os.environ.get("AIMSIGHT_DATA", ROOT / "data")).absolute() LABELS = Path(os.environ.get("AIMSIGHT_LABELS", ROOT / "labeling")).absolute() TOOLS = ROOT / "tools" # the reading harness (~/aimsight-tools), read by the analyst and written by the upgrader (Optimizer) def links(): """--add-dir for claude -p: the real target of the data/, labeling/ and tools/ links, outside the repository.""" return [a for d in (DATA, LABELS, TOOLS) for a in ("--add-dir", str(d.resolve()))] FOLDERS = ("auto", "manual") # those of the agents and of the page (D-105), those of the expert (D-102) FAMILIES = ("Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data", "None") # D-109 FORMAT = "%Y-%m-%dT%H:%MZ" MARGIN_PX = 150 # Support-Agent detecteurs.MARGE_PX: x, y are positions in the image only far from the corner COLORS = ["#2a78d6", "#eb6834", "#1baf7a", "#8a5cd6", "#d6a12a", "#d63b8a", "#3bb3d6", "#6b6b66"] # one per sensor POSE = (("pan", "#1a1a19"), ("tilt", "#8a8a85"), ("rotation", "#b5651d")) # The Advanced menu of Explore, in its order (Support-Agent labeling.PLOTS, QUALITE) PLOTS = (("error_value", "Error value"), ("x", "X [px]"), ("y", "Y [px]"), ("relative_x", "Relative Horizontal Shift"), ("relative_y", "Relative Vertical Shift"), ("corr_x", "Relative Corrected Horizontal Shift"), ("corr_y", "Relative Corrected Vertical Shift"), ("reproj_err", "References (reproj. error)")) QUALITY = (("peak", "Quality: Peak"), ("flux", "Quality: Flux"), ("sharpness", "Quality: Sharpness"), ("fwhm", "Quality: FWHM [px]"), ("dominance", "Quality: Dominance"), ("snr", "Quality: SNR")) IDENTIFIER = re.compile(r"(SIM_CCID|ip(_.*)?)$") # never read (CLAUDE.md, "data/") METRIC = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_") class Refusal(ValueError): """A request to refuse (400), with its reason.""" def stations(): return sorted(d.name for d in DATA.iterdir() if d.is_dir()) def folder(station): if station not in stations(): # the name becomes a path: a folder of data/, nothing else raise Refusal(f"unknown station: {station}") return DATA / station def bounds(start, end): return data.parse_time(start or "0001-01-01"), data.parse_time(end or "9999-01-01") # --- the positions of the sensors in the image def positions(measurements, a, b): """The median position of each sensor in the image over [a, b) ({} if x, y are not positions there).""" homes = {s: (statistics.median(xs), statistics.median(ys)) for s, ms in measurements.items() if (xs := [m["x"] for m in ms if a <= m["t"] < b and m["x"] is not None]) and (ys := [m["y"] for m in ms if a <= m["t"] < b and m["y"] is not None])} return homes if any(abs(x) > MARGIN_PX or abs(y) > MARGIN_PX for x, y in homes.values()) else {} def references(measurements, a, b): """Sensors of the compensation over the range: those with a reproj_err (references only).""" return {s for s, ms in measurements.items() if any(m["reproj_err"] is not None for m in ms if a <= m["t"] < b)} # --- the images of the Data page def images(station, start, end): """The images of [start, end), the median positions of the sensors, the references, and for each circled sensor its corr_x, corr_y at the time of each image: the nearest valid measurement, within 30 min, otherwise 0.""" d, (a, b) = folder(station), bounds(start, end) views = [(t, im) for im in sorted(glob.glob(str(d / "images" / "*"))) if (t := data.image_time(im)) and a <= t < b] if not views: raise Refusal(f"no image of {station} in this range") measurements = data.load(d) homes, refs = positions(measurements, a, b), references(measurements, a, b) # ponytail: a single measurement at each bound, noisy; a median over a window if the arrows shake too much margin, corr = timedelta(minutes=30), {} for s in homes: series = [(m["t"], m["corr_x"], m["corr_y"]) for m in measurements[s] if m["corr_x"] is not None and m["corr_y"] is not None] times, corr[s] = [m[0] for m in series], [] for t, _ in views: k = bisect.bisect_left(times, t) k = min((n for n in (k - 1, k) if 0 <= n < len(series)), key=lambda n: abs(times[n] - t), default=None) corr[s].append([round(series[k][1], 4), round(series[k][2], 4)] if k is not None and abs(times[k] - t) <= margin else 0) return {"images": [{"t": data.iso(t).replace("T", " ").rstrip("Z"), "name": os.path.basename(im)} for t, im in views], "positions": {s: [round(x, 1), round(y, 1)] for s, (x, y) in sorted(homes.items())}, "references": sorted(refs), "corr": corr} def image(station, name): f = (folder(station) / "images" / name).resolve() if f.parent != (DATA / station / "images").resolve() or not f.is_file(): raise Refusal("unknown image") return f # --- the curves of the Data page def metrics(d, a, b): """The numeric system metrics of metrics/ over [a, b) -> {name: {t: value or None}} (an empty value: None, the gap shows), without the text metrics, the identifiers, or those with no value at all.""" station, series, text = d.name + "_", {}, set() for p in sorted(glob.glob(str(d / "metrics" / "*.csv"))): if not (m := METRIC.match(os.path.basename(p))) or IDENTIFIER.match(name := m.group(1).removeprefix(station)): continue with open(p, newline="") as f: for r in csv.DictReader(f): t, v = data.parse_time(r["time_utc"]), (r.get("metric") or "").strip("[]'\" ") if not a <= t < b: continue try: series.setdefault(name, {})[t] = float(v) if v else None except ValueError: text.add(name) # link, mode…: no curve return {n: dict(sorted(s.items())) for n, s in series.items() if n not in text and any(v is not None for v in s.values())} def span(measurements, start, end): """The range read: from start to end (excluded), otherwise from the first measurement to the last, plus one step.""" every = sorted({m["t"] for ms in measurements.values() for m in ms}) a = data.parse_time(start) if start else every[0] b = data.parse_time(end) if end else every[-1] + (every[-1] - every[-2] if len(every) > 1 else timedelta(hours=1)) return a, b def curves(station, start, end): """-> {A, B, T, plots}: T, the times of each series, once (key: the sensor, "" for the camera, "system:" and its name for a metric); plot: title and traces (name, t, y, line, mode), without empty plots or sensors.""" d = folder(station) measurements = data.load(d) if not measurements: raise Refusal(f"no \"advanced\" export for {station}") a, b = span(measurements, start, end) poses = [p for p in data.load_camera(d) if a <= p["t"] < b] hour = lambda t: t.strftime("%Y-%m-%d %H:%M") # noqa: E731 (UTC, without time zone: as Plotly shows it) values = lambda s, c: [None if m[c] is None else round(m[c], 5) for m in s] # noqa: E731 inside = {s: [m for m in measurements[s] if a <= m["t"] < b] for s in sorted(measurements)} refs, out = references(measurements, a, b), [] def per_sensor(columns): for col, title in columns: traces = [{"name": s + " (ref.)" * (s in refs), "t": s, "y": y, "line": {"width": 1.2, "color": COLORS[i % len(COLORS)], "dash": "dot" if s in refs else "solid"}} for i, (s, y) in enumerate((s, values(ms, col)) for s, ms in inside.items()) if any(v is not None for v in y)] if traces: out.append({"title": title, "traces": traces}) per_sensor(PLOTS) if poses: out.append({"title": "Camera movement (deg)", "traces": [{"name": c, "t": "", "y": values(poses, c), "line": {"width": 1.2, "color": col}} for c, col in POSE]}) per_sensor(QUALITY) T = {"": [hour(p["t"]) for p in poses], **{s: [hour(m["t"]) for m in ms] for s, ms in inside.items()}} system = metrics(d, a, b) for name in sorted(system, key=str.lower): T["system:" + name] = [hour(t) for t in system[name]] out.append({"title": "System: " + name, "traces": [{"name": name, "t": "system:" + name, "mode": "lines+markers", "y": [None if v is None else round(v, 5) for v in system[name].values()], "line": {"width": 1.2, "color": COLORS[0]}}]}) if not out: raise Refusal(f"no measurement of {station} in this range") return {"A": data.iso(a), "B": data.iso(b), "T": T, "plots": out} # --- the Data page, "Detectors" labels: the events of the journal, read-only HISTORY = timedelta(days=3) # days before given to the detectors, as categories.py does (rule of 2026-10-24) # ponytail: kept until the server restarts (a minute or more per range); restart it after an /optimize run outside the app @functools.lru_cache(maxsize=32) def detectors(station, start, end, targets=""): """-> the detectors' events over the range, in the label format (start, end, family, description); plus kind, sensors, ignore (greyed area of the journal). targets: the sensors kept, comma-separated (empty: all); the others are removed before the detectors, as if they did not exist.""" d = folder(station) every = data.load(d) if targets: keep = set(targets.split(",")) every = {s: ms for s, ms in every.items() if s in keep} if not every: raise Refusal(f"no \"advanced\" export for {station}" + " with these targets" * bool(targets)) a, b = span(every, start, end) measurements = {s: k for s, ms in every.items() if (k := [m for m in ms if a - HISTORY <= m["t"] < b])} evs = [v for v in events(measurements, b, str(d)) if v["end"] >= data.iso(a)] if measurements else [] refs = {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"} return [{"id": f"D-{i + 1}", "start": v["start"], "end": v["end"], "family": v["category"], "kind": v["episodes"][0]["label"], "description": f"{v['episodes'][0]['label']}. {v['text']}", "sensors": v["sensors"], "ignore": v["ignore"], "targets": [s for s in v["sensors"] if s not in refs], "references": [s for s in v["sensors"] if s in refs], "message": message(v["text"]), "action": action(v)} for i, v in enumerate(evs)] def message(text): """What happens: the first sentence of the journal (detectors/scripts/categories.py, sentence), up to its ";"; the rest says what to do about it, except the sentence of the absorbed jumps, kept. ". " only: "1.5×" or "0.30 px" do not cut.""" same = re.search(r" (The jumps? of .* same cause\.)$", text) return re.split(r"\. |; ", text, maxsplit=1)[0].rstrip(".") + "." + (" " + same.group(1) if same else "") # What to do about it: the "For the client" column of labeling/families.md, per family; refined per kind (the label of the # main episode, with its level) after the journal sentences validated on 2026-10-24 (detectors/scripts/categories.py, sentence) ACTION_FAMILY = { "Visibility": "Do not trust the curves in this period: this is not a movement.", "Obstruction": "Ignore the curves of the hidden targets in this period.", "Thermal displacement": "Normal, nothing to do.", "Displacement": "To check: look at the images, then keep watching.", "Target issue": "Act on the target (prism, mount); ignore its curve in the meantime.", "No data": "The curves are missing in this period; its cause tells whether to act.", } ACTION_KIND = { "Poor visibility reduced": "Use the data in this period with care: they are less precise.", "Heat shimmer": "Nothing to do: the small fast wiggles are not movement.", "Camera thermal movement": "Nothing to do: this is not a movement of the structure.", "Brief obstruction": "Ignore this single point on the curves.", "Sudden jump, back to normal": "Look at the images: the cause is not established.", "Lasting shift": "Check whether the structure moved: look at the images.", "Jump, not back yet": "Check whether the structure moved: look at the images.", "Large shift": "Check whether the structure moved: look at the images.", "Slow drift": "Keep watching it.", } def action(v): ep = v["episodes"][0] return ACTION_KIND.get(f"{ep['label']} {ep.get('level') or ''}".strip(), ACTION_KIND.get(ep["label"])) \ or ACTION_FAMILY.get(v["category"], "") # --- the labels def label_file(station, source): if source not in FOLDERS: raise Refusal("unknown label folder") folder(station) return LABELS / source / f"{station}.jsonl" def read(station, source): f = label_file(station, source) # split and not splitlines: the latter would also cut on the Unicode separators the text may contain return [json.loads(l) for l in f.read_text(encoding="utf-8").split("\n") if l.strip()] if f.exists() else [] def assign_ids(station, labels): """As labeling/scripts/ids.py does (D-131): an id for the labels without one, the next of the station, as first field.""" n = max((int(e["id"].rsplit("-", 1)[1]) for e in labels if "id" in e), default=0) for e in labels: if "id" not in e: n += 1 e["id"] = f"{station}-{n}" if next(iter(e)) != "id": rest = {k: v for k, v in e.items() if k != "id"} ident = e["id"] e.clear() e.update({"id": ident}, **rest) def _validate(body, start, end): description = (body.get("description") or "").strip() try: empty = datetime.strptime(start, FORMAT) >= datetime.strptime(end, FORMAT) except (TypeError, ValueError): raise Refusal("unreadable bounds (YYYY-MM-DDTHH:MMZ)") from None if not description or empty: raise Refusal("empty text or empty range") family = body.get("family") or "" if family and family not in FAMILIES: raise Refusal("unknown family") return description, family def add(body): station, source = body.get("station"), body.get("folder", "manual") f = label_file(station, source) description, family = _validate(body, body.get("start"), body.get("end")) label = {"start": body["start"], "end": body["end"]} | ({"family": family} if family else {}) | {"description": description} if source == "auto": # an automatic label carries a stable id, the next of its station (D-131) assign_ids(station, read(station, "auto") + [label]) f.parent.mkdir(parents=True, exist_ok=True) before = f.read_bytes() if f.exists() else b"" # a line removed by hand may leave the file without a final newline with open(f, "ab") as out: out.write(b"\n" * bool(before and not before.endswith(b"\n")) + json.dumps(label, ensure_ascii=False).encode() + b"\n") return label def update(body): before = body.get("before") or {} start, end = body.get("start", before.get("start")), body.get("end", before.get("end")) description, family = _validate(body, start, end) label = dict.fromkeys(("id", "start", "end", "family")) | before | {"start": start, "end": end, "family": family, "description": description} if label["id"] is None: del label["id"] if not family: del label["family"] label.pop("review", None) # changed: no longer reviewed return replace(body.get("station"), body.get("folder", "manual"), before, label) def delete(body): replace(body.get("station"), body.get("folder", "manual"), body.get("before"), None) return body.get("before") def replace(station, source, before, label): """Replaces the label before by label (None: removes it), found by its content, not by its rank: the file may have changed since the page was loaded. Writes next to it then replaces: an interruption does not damage the file.""" f = label_file(station, source) lines = f.read_text(encoding="utf-8").split("\n") if f.exists() else [] for i, line in enumerate(lines): if line.strip() and json.loads(line) == before: lines[i:i + 1] = [] if label is None else [json.dumps(label, ensure_ascii=False)] temp = f.with_name(f.name + ".tmp") temp.write_text("\n".join(lines), encoding="utf-8") os.replace(temp, f) return label raise Refusal("this label is no longer in the file: reload the page")