Files
aimsight-app/app/server/station.py
T

338 lines
17 KiB
Python
Raw Normal View History

"""The data of the Data page: a station's images, its curves, its labels.
Data are read as the agents read them, tools/scripts/data.py (the tools repository, through the tools/ link); the detectors'
events are those of detectors/scripts/categories.py (events). Read-only on data/; writes only in labeling/{auto,manual}/.
"""
import bisect
import functools
import csv
import glob
import json
import os
import re
import statistics
import sys
from datetime import datetime, timedelta
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(ROOT / "tools" / "scripts"), str(ROOT / "detectors" / "scripts")]
import data # noqa: E402
from categories import events # noqa: E402
# data/ and labeling/ are links to ~/aimsight-data/data and ~/aimsight-labeling: no resolve(), the agents see data/ and labeling/
DATA = Path(os.environ.get("AIMSIGHT_DATA", ROOT / "data")).absolute()
LABELS = Path(os.environ.get("AIMSIGHT_LABELS", ROOT / "labeling")).absolute()
TOOLS = ROOT / "tools" # the reading harness (~/aimsight-tools), read by the analyst and written by the upgrader (Optimizer)
def links():
"""--add-dir for claude -p: the real target of the data/, labeling/ and tools/ links, outside the repository."""
return [a for d in (DATA, LABELS, TOOLS) for a in ("--add-dir", str(d.resolve()))]
FOLDERS = ("auto", "manual") # those of the agents and of the page (D-105), those of the expert (D-102)
FAMILIES = ("Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data", "None") # D-109
FORMAT = "%Y-%m-%dT%H:%MZ"
MARGIN_PX = 150 # Support-Agent detecteurs.MARGE_PX: x, y are positions in the image only far from the corner
COLORS = ["#2a78d6", "#eb6834", "#1baf7a", "#8a5cd6", "#d6a12a", "#d63b8a", "#3bb3d6", "#6b6b66"] # one per sensor
POSE = (("pan", "#1a1a19"), ("tilt", "#8a8a85"), ("rotation", "#b5651d"))
# The Advanced menu of Explore, in its order (Support-Agent labeling.PLOTS, QUALITE)
PLOTS = (("error_value", "Error value"), ("x", "X [px]"), ("y", "Y [px]"),
("relative_x", "Relative Horizontal Shift"), ("relative_y", "Relative Vertical Shift"),
("corr_x", "Relative Corrected Horizontal Shift"), ("corr_y", "Relative Corrected Vertical Shift"),
("reproj_err", "References (reproj. error)"))
QUALITY = (("peak", "Quality: Peak"), ("flux", "Quality: Flux"), ("sharpness", "Quality: Sharpness"),
("fwhm", "Quality: FWHM [px]"), ("dominance", "Quality: Dominance"), ("snr", "Quality: SNR"))
IDENTIFIER = re.compile(r"(SIM_CCID|ip(_.*)?)$") # never read (CLAUDE.md, "data/")
METRIC = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
class Refusal(ValueError):
"""A request to refuse (400), with its reason."""
def stations():
return sorted(d.name for d in DATA.iterdir() if d.is_dir())
def folder(station):
if station not in stations(): # the name becomes a path: a folder of data/, nothing else
raise Refusal(f"unknown station: {station}")
return DATA / station
def bounds(start, end):
return data.parse_time(start or "0001-01-01"), data.parse_time(end or "9999-01-01")
# --- the positions of the sensors in the image
def positions(measurements, a, b):
"""The median position of each sensor in the image over [a, b) ({} if x, y are not positions there)."""
homes = {s: (statistics.median(xs), statistics.median(ys)) for s, ms in measurements.items()
if (xs := [m["x"] for m in ms if a <= m["t"] < b and m["x"] is not None])
and (ys := [m["y"] for m in ms if a <= m["t"] < b and m["y"] is not None])}
return homes if any(abs(x) > MARGIN_PX or abs(y) > MARGIN_PX for x, y in homes.values()) else {}
def references(measurements, a, b):
"""Sensors of the compensation over the range: those with a reproj_err (references only)."""
return {s for s, ms in measurements.items() if any(m["reproj_err"] is not None for m in ms if a <= m["t"] < b)}
# --- the images of the Data page
def images(station, start, end):
"""The images of [start, end), the median positions of the sensors, the references, and for each circled sensor its
corr_x, corr_y at the time of each image: the nearest valid measurement, within 30 min, otherwise 0."""
d, (a, b) = folder(station), bounds(start, end)
views = [(t, im) for im in sorted(glob.glob(str(d / "images" / "*"))) if (t := data.image_time(im)) and a <= t < b]
if not views:
raise Refusal(f"no image of {station} in this range")
measurements = data.load(d)
homes, refs = positions(measurements, a, b), references(measurements, a, b)
# ponytail: a single measurement at each bound, noisy; a median over a window if the arrows shake too much
margin, corr = timedelta(minutes=30), {}
for s in homes:
series = [(m["t"], m["corr_x"], m["corr_y"]) for m in measurements[s] if m["corr_x"] is not None and m["corr_y"] is not None]
times, corr[s] = [m[0] for m in series], []
for t, _ in views:
k = bisect.bisect_left(times, t)
k = min((n for n in (k - 1, k) if 0 <= n < len(series)), key=lambda n: abs(times[n] - t), default=None)
corr[s].append([round(series[k][1], 4), round(series[k][2], 4)] if k is not None and abs(times[k] - t) <= margin else 0)
return {"images": [{"t": data.iso(t).replace("T", " ").rstrip("Z"), "name": os.path.basename(im)} for t, im in views],
"positions": {s: [round(x, 1), round(y, 1)] for s, (x, y) in sorted(homes.items())},
"references": sorted(refs), "corr": corr}
def image(station, name):
f = (folder(station) / "images" / name).resolve()
if f.parent != (DATA / station / "images").resolve() or not f.is_file():
raise Refusal("unknown image")
return f
# --- the curves of the Data page
def metrics(d, a, b):
"""The numeric system metrics of metrics/ over [a, b) -> {name: {t: value or None}} (an empty value: None, the gap
shows), without the text metrics, the identifiers, or those with no value at all."""
station, series, text = d.name + "_", {}, set()
for p in sorted(glob.glob(str(d / "metrics" / "*.csv"))):
if not (m := METRIC.match(os.path.basename(p))) or IDENTIFIER.match(name := m.group(1).removeprefix(station)):
continue
with open(p, newline="") as f:
for r in csv.DictReader(f):
t, v = data.parse_time(r["time_utc"]), (r.get("metric") or "").strip("[]'\" ")
if not a <= t < b:
continue
try:
series.setdefault(name, {})[t] = float(v) if v else None
except ValueError:
text.add(name) # link, mode…: no curve
return {n: dict(sorted(s.items())) for n, s in series.items()
if n not in text and any(v is not None for v in s.values())}
def span(measurements, start, end):
"""The range read: from start to end (excluded), otherwise from the first measurement to the last, plus one step."""
every = sorted({m["t"] for ms in measurements.values() for m in ms})
a = data.parse_time(start) if start else every[0]
b = data.parse_time(end) if end else every[-1] + (every[-1] - every[-2] if len(every) > 1 else timedelta(hours=1))
return a, b
def curves(station, start, end):
"""-> {A, B, T, plots}: T, the times of each series, once (key: the sensor, "" for the camera, "system:" and its
name for a metric); plot: title and traces (name, t, y, line, mode), without empty plots or sensors."""
d = folder(station)
measurements = data.load(d)
if not measurements:
raise Refusal(f"no \"advanced\" export for {station}")
a, b = span(measurements, start, end)
poses = [p for p in data.load_camera(d) if a <= p["t"] < b]
hour = lambda t: t.strftime("%Y-%m-%d %H:%M") # noqa: E731 (UTC, without time zone: as Plotly shows it)
values = lambda s, c: [None if m[c] is None else round(m[c], 5) for m in s] # noqa: E731
inside = {s: [m for m in measurements[s] if a <= m["t"] < b] for s in sorted(measurements)}
refs, out = references(measurements, a, b), []
def per_sensor(columns):
for col, title in columns:
traces = [{"name": s + " (ref.)" * (s in refs), "t": s, "y": y,
"line": {"width": 1.2, "color": COLORS[i % len(COLORS)], "dash": "dot" if s in refs else "solid"}}
for i, (s, y) in enumerate((s, values(ms, col)) for s, ms in inside.items()) if any(v is not None for v in y)]
if traces:
out.append({"title": title, "traces": traces})
per_sensor(PLOTS)
if poses:
out.append({"title": "Camera movement (deg)", "traces": [{"name": c, "t": "", "y": values(poses, c),
"line": {"width": 1.2, "color": col}} for c, col in POSE]})
per_sensor(QUALITY)
T = {"": [hour(p["t"]) for p in poses], **{s: [hour(m["t"]) for m in ms] for s, ms in inside.items()}}
system = metrics(d, a, b)
for name in sorted(system, key=str.lower):
T["system:" + name] = [hour(t) for t in system[name]]
out.append({"title": "System: " + name, "traces": [{"name": name, "t": "system:" + name, "mode": "lines+markers",
"y": [None if v is None else round(v, 5) for v in system[name].values()],
"line": {"width": 1.2, "color": COLORS[0]}}]})
if not out:
raise Refusal(f"no measurement of {station} in this range")
return {"A": data.iso(a), "B": data.iso(b), "T": T, "plots": out}
# --- the Data page, "Detectors" labels: the events of the journal, read-only
HISTORY = timedelta(days=3) # days before given to the detectors, as categories.py does (rule of 2026-10-24)
# ponytail: kept until the server restarts (a minute or more per range); restart it after an /optimize run outside the app
@functools.lru_cache(maxsize=32)
def detectors(station, start, end, targets=""):
"""-> the detectors' events over the range, in the label format (start, end, family, description); plus kind,
sensors, ignore (greyed area of the journal). targets: the sensors kept, comma-separated (empty: all); the others are
removed before the detectors, as if they did not exist."""
d = folder(station)
every = data.load(d)
if targets:
keep = set(targets.split(","))
every = {s: ms for s, ms in every.items() if s in keep}
if not every:
raise Refusal(f"no \"advanced\" export for {station}" + " with these targets" * bool(targets))
a, b = span(every, start, end)
measurements = {s: k for s, ms in every.items() if (k := [m for m in ms if a - HISTORY <= m["t"] < b])}
evs = [v for v in events(measurements, b, str(d)) if v["end"] >= data.iso(a)] if measurements else []
refs = {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"}
return [{"id": f"D-{i + 1}", "start": v["start"], "end": v["end"], "family": v["category"], "kind": v["episodes"][0]["label"],
"description": f"{v['episodes'][0]['label']}. {v['text']}", "sensors": v["sensors"], "ignore": v["ignore"],
"targets": [s for s in v["sensors"] if s not in refs], "references": [s for s in v["sensors"] if s in refs],
"message": message(v["text"]), "action": action(v)}
for i, v in enumerate(evs)]
def message(text):
"""What happens: the first sentence of the journal (detectors/scripts/categories.py, sentence), up to its ";"; the rest
says what to do about it, except the sentence of the absorbed jumps, kept. ". " only: "1.5×" or "0.30 px" do not cut."""
same = re.search(r" (The jumps? of .* same cause\.)$", text)
return re.split(r"\. |; ", text, maxsplit=1)[0].rstrip(".") + "." + (" " + same.group(1) if same else "")
# What to do about it: the "For the client" column of labeling/families.md, per family; refined per kind (the label of the
# main episode, with its level) after the journal sentences validated on 2026-10-24 (detectors/scripts/categories.py, sentence)
ACTION_FAMILY = {
"Visibility": "Do not trust the curves in this period: this is not a movement.",
"Obstruction": "Ignore the curves of the hidden targets in this period.",
"Thermal displacement": "Normal, nothing to do.",
"Displacement": "To check: look at the images, then keep watching.",
"Target issue": "Act on the target (prism, mount); ignore its curve in the meantime.",
"No data": "The curves are missing in this period; its cause tells whether to act.",
}
ACTION_KIND = {
"Poor visibility reduced": "Use the data in this period with care: they are less precise.",
"Heat shimmer": "Nothing to do: the small fast wiggles are not movement.",
"Camera thermal movement": "Nothing to do: this is not a movement of the structure.",
"Brief obstruction": "Ignore this single point on the curves.",
"Sudden jump, back to normal": "Look at the images: the cause is not established.",
"Lasting shift": "Check whether the structure moved: look at the images.",
"Jump, not back yet": "Check whether the structure moved: look at the images.",
"Large shift": "Check whether the structure moved: look at the images.",
"Slow drift": "Keep watching it.",
}
def action(v):
ep = v["episodes"][0]
return ACTION_KIND.get(f"{ep['label']} {ep.get('level') or ''}".strip(), ACTION_KIND.get(ep["label"])) \
or ACTION_FAMILY.get(v["category"], "")
# --- the labels
def label_file(station, source):
if source not in FOLDERS:
raise Refusal("unknown label folder")
folder(station)
return LABELS / source / f"{station}.jsonl"
def read(station, source):
f = label_file(station, source)
# split and not splitlines: the latter would also cut on the Unicode separators the text may contain
return [json.loads(l) for l in f.read_text(encoding="utf-8").split("\n") if l.strip()] if f.exists() else []
def assign_ids(station, labels):
"""As labeling/scripts/ids.py does (D-131): an id for the labels without one, the next of the station, as first field."""
n = max((int(e["id"].rsplit("-", 1)[1]) for e in labels if "id" in e), default=0)
for e in labels:
if "id" not in e:
n += 1
e["id"] = f"{station}-{n}"
if next(iter(e)) != "id":
rest = {k: v for k, v in e.items() if k != "id"}
ident = e["id"]
e.clear()
e.update({"id": ident}, **rest)
def _validate(body, start, end):
description = (body.get("description") or "").strip()
try:
empty = datetime.strptime(start, FORMAT) >= datetime.strptime(end, FORMAT)
except (TypeError, ValueError):
raise Refusal("unreadable bounds (YYYY-MM-DDTHH:MMZ)") from None
if not description or empty:
raise Refusal("empty text or empty range")
family = body.get("family") or ""
if family and family not in FAMILIES:
raise Refusal("unknown family")
return description, family
def add(body):
station, source = body.get("station"), body.get("folder", "manual")
f = label_file(station, source)
description, family = _validate(body, body.get("start"), body.get("end"))
label = {"start": body["start"], "end": body["end"]} | ({"family": family} if family else {}) | {"description": description}
if source == "auto": # an automatic label carries a stable id, the next of its station (D-131)
assign_ids(station, read(station, "auto") + [label])
f.parent.mkdir(parents=True, exist_ok=True)
before = f.read_bytes() if f.exists() else b"" # a line removed by hand may leave the file without a final newline
with open(f, "ab") as out:
out.write(b"\n" * bool(before and not before.endswith(b"\n")) + json.dumps(label, ensure_ascii=False).encode() + b"\n")
return label
def update(body):
before = body.get("before") or {}
start, end = body.get("start", before.get("start")), body.get("end", before.get("end"))
description, family = _validate(body, start, end)
label = dict.fromkeys(("id", "start", "end", "family")) | before | {"start": start, "end": end, "family": family,
"description": description}
if label["id"] is None:
del label["id"]
if not family:
del label["family"]
label.pop("review", None) # changed: no longer reviewed
return replace(body.get("station"), body.get("folder", "manual"), before, label)
def delete(body):
replace(body.get("station"), body.get("folder", "manual"), body.get("before"), None)
return body.get("before")
def replace(station, source, before, label):
"""Replaces the label before by label (None: removes it), found by its content, not by its rank: the file may have
changed since the page was loaded. Writes next to it then replaces: an interruption does not damage the file."""
f = label_file(station, source)
lines = f.read_text(encoding="utf-8").split("\n") if f.exists() else []
for i, line in enumerate(lines):
if line.strip() and json.loads(line) == before:
lines[i:i + 1] = [] if label is None else [json.dumps(label, ensure_ascii=False)]
temp = f.with_name(f.name + ".tmp")
temp.write_text("\n".join(lines), encoding="utf-8")
os.replace(temp, f)
return label
raise Refusal("this label is no longer in the file: reload the page")