Files
aimsight-app/app/server/station.py
T
2026-11-16 13:20:31 +01:00

339 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""The data of the Data page: a station's images, its curves, its labels.
Data are read as the agents read them, tools/scripts/data.py (the tools repository, through the tools/ link); the detectors'
events are those of detectors/scripts/categories.py (events). Read-only on data/; writes only in labeling/{auto,manual}/.
"""
import bisect
import functools
import csv
import glob
import json
import os
import re
import statistics
import sys
from datetime import datetime, timedelta
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(ROOT / "tools" / "scripts"), str(ROOT / "detectors" / "scripts")]
import data # noqa: E402
from categories import events # noqa: E402
# data/ and labeling/ are links to ~/aimsight-data/data and ~/labeling: no resolve(), the agents see data/ and labeling/
DATA = Path(os.environ.get("AIMSIGHT_DATA", ROOT / "data")).absolute()
LABELS = Path(os.environ.get("AIMSIGHT_LABELS", ROOT / "labeling")).absolute()
TOOLS = ROOT / "tools" # the reading harness (~/tools), read by the analyst and written by the tool-builder
def links():
"""--add-dir for claude -p: the real target of the data/, labeling/ and tools/ links, outside the repository."""
return [a for d in (DATA, LABELS, TOOLS) for a in ("--add-dir", str(d.resolve()))]
FOLDERS = ("auto", "manual") # those of the agents and of the page (D-105), those of the expert (D-102)
FAMILIES = ("Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data", "None") # D-109
FORMAT = "%Y-%m-%dT%H:%MZ"
MARGIN_PX = 150 # Support-Agent detecteurs.MARGE_PX: x, y are positions in the image only far from the corner
COLORS = ["#2a78d6", "#eb6834", "#1baf7a", "#8a5cd6", "#d6a12a", "#d63b8a", "#3bb3d6", "#6b6b66"] # one per sensor
POSE = (("pan", "#1a1a19"), ("tilt", "#8a8a85"), ("rotation", "#b5651d"))
# The Advanced menu of Explore, in its order (Support-Agent labeling.PLOTS, QUALITE)
PLOTS = (("error_value", "Error value"), ("x", "X [px]"), ("y", "Y [px]"),
("relative_x", "Relative Horizontal Shift"), ("relative_y", "Relative Vertical Shift"),
("corr_x", "Relative Corrected Horizontal Shift"), ("corr_y", "Relative Corrected Vertical Shift"),
("reproj_err", "References (reproj. error)"))
QUALITY = (("peak", "Quality: Peak"), ("flux", "Quality: Flux"), ("sharpness", "Quality: Sharpness"),
("fwhm", "Quality: FWHM [px]"), ("dominance", "Quality: Dominance"), ("snr", "Quality: SNR"))
IDENTIFIER = re.compile(r"(SIM_CCID|ip(_.*)?)$") # never read (CLAUDE.md, "data/")
METRIC = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
class Refusal(ValueError):
"""A request to refuse (400), with its reason."""
def stations():
return sorted(d.name for d in DATA.iterdir() if d.is_dir())
def folder(station):
if station not in stations(): # the name becomes a path: a folder of data/, nothing else
raise Refusal(f"unknown station: {station}")
return DATA / station
def bounds(start, end):
return data.parse_time(start or "0001-01-01"), data.parse_time(end or "9999-01-01")
# --- the positions of the sensors in the image
def positions(measurements, a, b):
"""The median position of each sensor in the image over [a, b) ({} if x, y are not positions there)."""
homes = {s: (statistics.median(xs), statistics.median(ys)) for s, ms in measurements.items()
if (xs := [m["x"] for m in ms if a <= m["t"] < b and m["x"] is not None])
and (ys := [m["y"] for m in ms if a <= m["t"] < b and m["y"] is not None])}
return homes if any(abs(x) > MARGIN_PX or abs(y) > MARGIN_PX for x, y in homes.values()) else {}
def references(measurements, a, b):
"""Sensors of the compensation over the range: those with a reproj_err (references only)."""
return {s for s, ms in measurements.items() if any(m["reproj_err"] is not None for m in ms if a <= m["t"] < b)}
# --- the images of the Data page
def images(station, start, end):
"""The images of [start, end), the median positions of the sensors, the references, and for each circled sensor its
corr_x, corr_y at the time of each image: the nearest valid measurement, within 30 min, otherwise 0."""
d, (a, b) = folder(station), bounds(start, end)
views = [(t, im) for im in sorted(glob.glob(str(d / "images" / "*"))) if (t := data.image_time(im)) and a <= t < b]
if not views:
raise Refusal(f"no image of {station} in this range")
measurements = data.load(d)
homes, refs = positions(measurements, a, b), references(measurements, a, b)
# ponytail: a single measurement at each bound, noisy; a median over a window if the arrows shake too much
margin, corr = timedelta(minutes=30), {}
for s in homes:
series = [(m["t"], m["corr_x"], m["corr_y"]) for m in measurements[s] if m["corr_x"] is not None and m["corr_y"] is not None]
times, corr[s] = [m[0] for m in series], []
for t, _ in views:
k = bisect.bisect_left(times, t)
k = min((n for n in (k - 1, k) if 0 <= n < len(series)), key=lambda n: abs(times[n] - t), default=None)
corr[s].append([round(series[k][1], 4), round(series[k][2], 4)] if k is not None and abs(times[k] - t) <= margin else 0)
return {"images": [{"t": data.iso(t).replace("T", " ").rstrip("Z"), "name": os.path.basename(im)} for t, im in views],
"positions": {s: [round(x, 1), round(y, 1)] for s, (x, y) in sorted(homes.items())},
"references": sorted(refs), "corr": corr}
def image(station, name):
f = (folder(station) / "images" / name).resolve()
if f.parent != (DATA / station / "images").resolve() or not f.is_file():
raise Refusal("unknown image")
return f
# --- the curves of the Data page
def metrics(d, a, b):
"""The numeric system metrics of metrics/ over [a, b) -> {name: {t: value or None}} (an empty value: None, the gap
shows), without the text metrics, the identifiers, or those with no value at all."""
station, series, text = d.name + "_", {}, set()
for p in sorted(glob.glob(str(d / "metrics" / "*.csv"))):
if not (m := METRIC.match(os.path.basename(p))) or IDENTIFIER.match(name := m.group(1).removeprefix(station)):
continue
with open(p, newline="") as f:
for r in csv.DictReader(f):
t, v = data.parse_time(r["time_utc"]), (r.get("metric") or "").strip("[]'\" ")
if not a <= t < b:
continue
try:
series.setdefault(name, {})[t] = float(v) if v else None
except ValueError:
text.add(name) # link, mode…: no curve
return {n: dict(sorted(s.items())) for n, s in series.items()
if n not in text and any(v is not None for v in s.values())}
def span(measurements, start, end):
"""The range read: from start to end (excluded), otherwise from the first measurement to the last, plus one step."""
every = sorted({m["t"] for ms in measurements.values() for m in ms})
a = data.parse_time(start) if start else every[0]
b = data.parse_time(end) if end else every[-1] + (every[-1] - every[-2] if len(every) > 1 else timedelta(hours=1))
return a, b
def curves(station, start, end):
"""-> {A, B, T, plots}: T, the times of each series, once (key: the sensor, "" for the camera, "system:" and its
name for a metric); plot: title and traces (name, t, y, line, mode), without empty plots or sensors."""
d = folder(station)
measurements = data.load(d)
if not measurements:
raise Refusal(f"no \"advanced\" export for {station}")
a, b = span(measurements, start, end)
poses = [p for p in data.load_camera(d) if a <= p["t"] < b]
hour = lambda t: t.strftime("%Y-%m-%d %H:%M") # noqa: E731 (UTC, without time zone: as Plotly shows it)
values = lambda s, c: [None if m[c] is None else round(m[c], 5) for m in s] # noqa: E731
inside = {s: [m for m in measurements[s] if a <= m["t"] < b] for s in sorted(measurements)}
refs, out = references(measurements, a, b), []
def per_sensor(columns):
for col, title in columns:
traces = [{"name": s + " (ref.)" * (s in refs), "t": s, "y": y,
"line": {"width": 1.2, "color": COLORS[i % len(COLORS)], "dash": "dot" if s in refs else "solid"}}
for i, (s, y) in enumerate((s, values(ms, col)) for s, ms in inside.items()) if any(v is not None for v in y)]
if traces:
out.append({"title": title, "traces": traces})
per_sensor(PLOTS)
if poses:
out.append({"title": "Camera movement (deg)", "traces": [{"name": c, "t": "", "y": values(poses, c),
"line": {"width": 1.2, "color": col}} for c, col in POSE]})
per_sensor(QUALITY)
T = {"": [hour(p["t"]) for p in poses], **{s: [hour(m["t"]) for m in ms] for s, ms in inside.items()}}
system = metrics(d, a, b)
for name in sorted(system, key=str.lower):
T["system:" + name] = [hour(t) for t in system[name]]
out.append({"title": "System: " + name, "traces": [{"name": name, "t": "system:" + name, "mode": "lines+markers",
"y": [None if v is None else round(v, 5) for v in system[name].values()],
"line": {"width": 1.2, "color": COLORS[0]}}]})
if not out:
raise Refusal(f"no measurement of {station} in this range")
return {"A": data.iso(a), "B": data.iso(b), "T": T, "plots": out}
# --- the Data page, "Detectors" labels: the events of the journal, read-only
HISTORY = timedelta(days=3) # days before given to the detectors, as categories.py does (rule of 2026-10-24)
# ponytail: kept until the server restarts (a minute or more per range); restart it after an /improve run outside the app
@functools.lru_cache(maxsize=32)
def detectors(station, start, end, targets=""):
"""-> the detectors' events over the range, in the label format (start, end, family, description); plus kind,
sensors, ignore (greyed area of the journal). targets: the sensors kept, comma-separated (empty: all); the others are
removed before the detectors, as if they did not exist."""
d = folder(station)
every = data.load(d)
if targets:
keep = set(targets.split(","))
every = {s: ms for s, ms in every.items() if s in keep}
if not every:
raise Refusal(f"no \"advanced\" export for {station}" + " with these targets" * bool(targets))
a, b = span(every, start, end)
measurements = {s: k for s, ms in every.items() if (k := [m for m in ms if a - HISTORY <= m["t"] < b])}
evs = [v for v in events(measurements, b, str(d)) if v["end"] >= data.iso(a)] if measurements else []
refs = {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"}
return [{"id": f"D-{i + 1}", "start": v["start"], "end": v["end"], "family": v["category"], "kind": v["episodes"][0]["label"],
"description": f"{v['episodes'][0]['label']}. {v['text']}", "sensors": v["sensors"], "ignore": v["ignore"],
"targets": [s for s in v["sensors"] if s not in refs], "references": [s for s in v["sensors"] if s in refs],
"message": message(v["text"]), "action": action(v)}
for i, v in enumerate(evs)]
def message(text):
"""What happens: the first sentence of the journal (detectors/scripts/categories.py, sentence), up to its ";"; the rest
says what to do about it, except the sentence of the absorbed jumps, kept. ". " only: "1.5×" or "0.30 px" do not cut."""
same = re.search(r" (The jumps? of .* same cause\.)$", text)
return re.split(r"\. |; ", text, maxsplit=1)[0].rstrip(".") + "." + (" " + same.group(1) if same else "")
# What to do about it: the "For the client" column of labeling/families.md, per family; refined per kind (the label of the
# main episode, with its level) after the journal sentences validated on 2026-10-24 (detectors/scripts/categories.py, sentence)
ACTION_FAMILY = {
"Visibility": "Do not trust the curves in this period: this is not a movement.",
"Obstruction": "Ignore the curves of the hidden targets in this period.",
"Thermal displacement": "Normal, nothing to do.",
"Displacement": "To check: look at the images, then keep watching.",
"Target issue": "Act on the target (prism, mount); ignore its curve in the meantime.",
"No data": "The curves are missing in this period; its cause tells whether to act.",
}
ACTION_KIND = {
"Poor visibility reduced": "Use the data in this period with care: they are less precise.",
"Heat shimmer": "Nothing to do: the small fast wiggles are not movement.",
"Camera thermal movement": "Nothing to do: this is not a movement of the structure.",
"Brief obstruction": "Ignore this single point on the curves.",
"Sudden jump, back to normal": "Look at the images: the cause is not established.",
"Lasting shift": "Check whether the structure moved: look at the images.",
"Jump, not back yet": "Check whether the structure moved: look at the images.",
"Large shift": "Check whether the structure moved: look at the images.",
"Slow drift": "Keep watching it.",
}
def action(v):
ep = v["episodes"][0]
return ACTION_KIND.get(f"{ep['label']} {ep.get('level') or ''}".strip(), ACTION_KIND.get(ep["label"])) \
or ACTION_FAMILY.get(v["category"], "")
# --- the labels
def label_file(station, source):
if source not in FOLDERS:
raise Refusal("unknown label folder")
folder(station)
return LABELS / source / f"{station}.jsonl"
def read(station, source):
f = label_file(station, source)
# split and not splitlines: the latter would also cut on the Unicode separators the text may contain
return [json.loads(l) for l in f.read_text(encoding="utf-8").split("\n") if l.strip()] if f.exists() else []
def assign_ids(station, labels):
"""As labeling/scripts/ids.py does (D-131): an id for the labels without one, the next of the station, as first field."""
n = max((int(e["id"].rsplit("-", 1)[1]) for e in labels if "id" in e), default=0)
for e in labels:
if "id" not in e:
n += 1
e["id"] = f"{station}-{n}"
if next(iter(e)) != "id":
rest = {k: v for k, v in e.items() if k != "id"}
ident = e["id"]
e.clear()
e.update({"id": ident}, **rest)
def _validate(body, start, end):
description = (body.get("description") or "").strip()
try:
empty = datetime.strptime(start, FORMAT) >= datetime.strptime(end, FORMAT)
except (TypeError, ValueError):
raise Refusal("unreadable bounds (YYYY-MM-DDTHH:MMZ)") from None
if not description or empty:
raise Refusal("empty text or empty range")
family = body.get("family") or ""
if family and family not in FAMILIES:
raise Refusal("unknown family")
return description, family
def add(body):
station, source = body.get("station"), body.get("folder", "manual")
f = label_file(station, source)
description, family = _validate(body, body.get("start"), body.get("end"))
label = {"start": body["start"], "end": body["end"]} | ({"family": family} if family else {}) | {"description": description}
if source == "auto": # an automatic label carries a stable id, the next of its station (D-131)
assign_ids(station, read(station, "auto") + [label])
f.parent.mkdir(parents=True, exist_ok=True)
before = f.read_bytes() if f.exists() else b"" # a line removed by hand may leave the file without a final newline
with open(f, "ab") as out:
out.write(b"\n" * bool(before and not before.endswith(b"\n")) + json.dumps(label, ensure_ascii=False).encode() + b"\n")
return label
def update(body):
before = body.get("before") or {}
start, end = body.get("start", before.get("start")), body.get("end", before.get("end"))
description, family = _validate(body, start, end)
label = dict.fromkeys(("id", "start", "end", "family")) | before | {"start": start, "end": end, "family": family,
"description": description}
if label["id"] is None:
del label["id"]
if not family:
del label["family"]
label.pop("review", None) # changed: no longer reviewed
return replace(body.get("station"), body.get("folder", "manual"), before, label)
def delete(body):
replace(body.get("station"), body.get("folder", "manual"), body.get("before"), None)
return body.get("before")
def replace(station, source, before, label):
"""Replaces the label before by label (None: removes it), found by its content, not by its rank: the file may have
changed since the page was loaded. Writes next to it then replaces: an interruption does not damage the file."""
f = label_file(station, source)
lines = f.read_text(encoding="utf-8").split("\n") if f.exists() else []
for i, line in enumerate(lines):
if line.strip() and json.loads(line) == before:
lines[i:i + 1] = [] if label is None else [json.dumps(label, ensure_ascii=False)]
temp = f.with_name(f.name + ".tmp")
temp.write_text("\n".join(lines), encoding="utf-8")
os.replace(temp, f)
return label
raise Refusal("this label is no longer in the file: reload the page")