English: the app (server/, analysis.py, routes and JSON fields), instructions/analyst.md, docs; the table of shared renames
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1d80a77cb1
commit
a3e6ade845
24 files changed
+1959
-1835
No files matched your search
@@ -0,0 +1,317 @@
|
||||
"""The Analysis tab of the Data page: the analyst of the agents repository (agents/, a plugin: --agent agents:analyst), launched
|
||||
by `claude -p`, non-interactive, on a range chosen in the page, with the user's optional hints, its instructions
|
||||
(instructions/analyst.md) and without the detectors (DETECTORS: no: its labels will be used to improve them). It writes in
|
||||
labeling/auto/analyses/<STATION>_<start>_<end>/ its labels (labels.jsonl) and a diagnostic report (report.md, its figures
|
||||
next to it); at the end, the server submits its labels to labeling/auto/<STATION>.jsonl, marked with their analysis (field
|
||||
`analysis`). Then the user can write to it: the same session resumes (`claude -p --resume`), and its labels, if they
|
||||
change, replace those it had submitted.
|
||||
On a finished analysis, "Optimize" launches `/improve` in Optimizer (optimizer/, the skill that improves the detectors on the
|
||||
range of the analysis, its labels as truth), then the tool-builder of the agents repository, which brings the analyst's
|
||||
scripts into the reading tools (tools/); one optimization at a time, no analysis or message meanwhile; the summary of the
|
||||
session (optimizer/sessions/<analysis>_<time>/summary.md) and that of the tools (<analysis>/tools.md) show under the report.
|
||||
"""
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
import markdown
|
||||
|
||||
import station
|
||||
|
||||
FOLDER = station.LABELS / "auto" / "analyses"
|
||||
DETECTORS = station.ROOT / "detectors"
|
||||
AGENTS = station.ROOT / "agents" # the plugin of the generic agents (~/agents)
|
||||
OPTIMIZER = station.ROOT / "optimizer" # the Optimizer repository (~/Optimizer), where /improve runs
|
||||
SESSIONS = OPTIMIZER / "sessions"
|
||||
# the request of /improve (optimizer/README.md), paths relative to the root of Optimizer (its links data, labeling, detectors)
|
||||
IMPROVE = """/improve DETECTORS: detectors/. STATION: data/{station}. FROM: {start}. TO: {end}. \
|
||||
LABELS: labeling/auto/{station}.jsonl. REFERENCE: --labels labeling/auto --data data. FAMILIES: labeling/families.md. \
|
||||
CONTEXT: labeling/auto/analyses/{name}/report.md. OUT: sessions/{name}_{time}/. Caller's context: Diagnostic-app, through \
|
||||
Optimizer; read-only on data/ and labeling/; commit only in detectors/, on its current branch; do not commit OUT."""
|
||||
REQUEST = """Station: data/{station}/. Range: {start} → {end} (UTC). Hints: {hints}. \
|
||||
Output folder: {out}/. DETECTORS: no. INSTRUCTIONS: instructions/analyst.md."""
|
||||
|
||||
# name -> {"state": "running" | "done" | "error", "lines": [...], "submitted": n}; one analysis at a time per range
|
||||
analyses, lock = {}, threading.Lock()
|
||||
# name of the analysis -> {"state": "running" | "done" | "error", "lines": [...]}; one optimization at a time, all analyses
|
||||
optimizations = {}
|
||||
BUSY = "an optimization of the detectors is running"
|
||||
|
||||
|
||||
ISOLATED = ["--setting-sources", "project,local", "--strict-mcp-config"] # no plugins, hooks or MCP servers of the workstation
|
||||
|
||||
|
||||
def _env():
|
||||
"""The environment of claude -p, without an API key: the agents run on the Claude subscription, never on a key."""
|
||||
return {k: v for k, v in os.environ.items() if k not in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")}
|
||||
|
||||
|
||||
def _optimizing():
|
||||
return any(o["state"] == "running" for o in optimizations.values())
|
||||
|
||||
|
||||
def launch(body):
|
||||
st, start, end = body.get("station"), body.get("start"), body.get("end")
|
||||
station._validate({"description": "-"}, start, end)
|
||||
station.folder(st)
|
||||
if not (AGENTS / ".claude-plugin" / "plugin.json").exists():
|
||||
raise station.Refusal("no agents in agents/ (the link: README.md)")
|
||||
name = f"{st}_{start[:16].replace(':', '')}_{end[:16].replace(':', '')}"
|
||||
with lock:
|
||||
if _optimizing():
|
||||
raise station.Refusal(BUSY)
|
||||
if analyses.get(name, {}).get("state") == "running":
|
||||
raise station.Refusal("this range is already being analyzed")
|
||||
out = FOLDER / name
|
||||
if out.exists(): # a new analysis of the same range: the old one stays, next to it
|
||||
name += datetime.now(timezone.utc).strftime("_%H%M%S")
|
||||
out = FOLDER / name
|
||||
hints = (body.get("hints") or "").strip()
|
||||
request = {"station": st, "start": start, "end": end, "hints": hints}
|
||||
analyses[name] = {"state": "running", "lines": [], "submitted": 0, "station": st, "request": request}
|
||||
(out / "work").mkdir(parents=True)
|
||||
(out / "request.json").write_text(json.dumps(request, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
prompt = REQUEST.format(station=st, start=start, end=end, out=out.relative_to(station.ROOT), hints=hints or "none")
|
||||
threading.Thread(target=work, args=(name, out, prompt), daemon=True).start()
|
||||
return {"name": name}
|
||||
|
||||
|
||||
def work(name, out, prompt, resume=None):
|
||||
"""A turn of the analyst: the request, or a message of the user in the same session (resume: its id)."""
|
||||
a = analyses[name]
|
||||
try:
|
||||
p = subprocess.Popen(["claude", "-p", prompt, "--plugin-dir", str(AGENTS), "--agent", "agents:analyst", *ISOLATED,
|
||||
"--output-format", "stream-json", "--verbose",
|
||||
"--allowedTools", "Read,Write,Bash", "--permission-mode", "acceptEdits", *station.links()]
|
||||
+ (["--resume", resume] if resume else []),
|
||||
cwd=station.ROOT, env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
|
||||
for line in p.stdout:
|
||||
log.write(line)
|
||||
a["lines"] += digest(line)
|
||||
if '"subtype":"init"' in line: # the session, for the next messages
|
||||
(out / "work" / "session").write_text(json.loads(line)["session_id"])
|
||||
if p.wait():
|
||||
raise RuntimeError(f"claude ended with code {p.returncode}")
|
||||
a["submitted"] = submit(a["station"], name, out)
|
||||
a["state"] = "done"
|
||||
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
|
||||
a["lines"].append(f"Error: {e}")
|
||||
a["state"] = "error"
|
||||
|
||||
|
||||
def message(name, body):
|
||||
"""A message of the user to the analyst, after its report: the same session resumes (claude -p --resume)."""
|
||||
text, out = (body.get("text") or "").strip(), FOLDER / name
|
||||
state(name) # the name checked, the state read again after a restart
|
||||
a, session = analyses[name], out / "work" / "session"
|
||||
if not text or not session.exists():
|
||||
raise station.Refusal("empty message, or the analysis has no session to resume")
|
||||
with lock:
|
||||
if _optimizing():
|
||||
raise station.Refusal(BUSY)
|
||||
if a["state"] == "running":
|
||||
raise station.Refusal("the analyst is still working")
|
||||
a["state"] = "running"
|
||||
line = json.dumps({"type": "page", "text": text}, ensure_ascii=False) + "\n"
|
||||
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
|
||||
log.write(line)
|
||||
a["lines"] += digest(line)
|
||||
prompt = (f"{text}\n\n(If you change your labels, rewrite {out.relative_to(station.ROOT)}/labels.jsonl in full, the "
|
||||
"server will submit them again; if the diagnosis changes, update report.md.)")
|
||||
threading.Thread(target=work, args=(name, out, prompt, session.read_text()), daemon=True).start()
|
||||
return {}
|
||||
|
||||
|
||||
def digest(line):
|
||||
"""A line of the log -> what the page shows of it: the user's messages ("› "), the analyst's texts, its tools and their
|
||||
command, one line each."""
|
||||
try:
|
||||
m = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
return [line.strip()[:200]] if line.strip() else []
|
||||
if m.get("type") == "page":
|
||||
return [f"› {m['text']}"]
|
||||
if m.get("type") != "assistant":
|
||||
return []
|
||||
out = []
|
||||
for c in m["message"].get("content", []):
|
||||
if c.get("type") == "text" and c["text"].strip():
|
||||
out.append(c["text"].strip()[:300])
|
||||
elif c.get("type") == "tool_use":
|
||||
i = c.get("input", {})
|
||||
out.append(f"{c['name']}: {i.get('command') or i.get('file_path') or ''}"[:200])
|
||||
return out
|
||||
|
||||
|
||||
def submit(st, name, out):
|
||||
"""The labels of the analysis into labeling/auto/<STATION>.jsonl, with their id and the name of the analysis; if the
|
||||
analyst changed them since the last submission, they replace those it had submitted -> their number."""
|
||||
labels, copy = out / "labels.jsonl", out / "work" / "submitted.jsonl"
|
||||
text = labels.read_text(encoding="utf-8") if labels.exists() else ""
|
||||
new = []
|
||||
for l in text.split("\n"):
|
||||
if l.strip():
|
||||
e = json.loads(l)
|
||||
station._validate(e, e.get("start"), e.get("end"))
|
||||
new.append({k: e[k] for k in ("start", "end", "family", "description") if k in e} | {"analysis": name})
|
||||
if copy.exists() and copy.read_text(encoding="utf-8") == text:
|
||||
return len(new)
|
||||
# ponytail: removed then put back, the labels of the analysis lose the edits made in the page; keeping them would
|
||||
# require matching them one by one
|
||||
with lock:
|
||||
f = station.label_file(st, "auto") # the other lines stay as they are
|
||||
kept = [l for l in (f.read_text(encoding="utf-8").split("\n") if f.exists() else [])
|
||||
if l.strip() and json.loads(l).get("analysis") != name]
|
||||
station.assign_ids(st, [json.loads(l) for l in kept] + new)
|
||||
temp = f.with_name(f.name + ".tmp")
|
||||
temp.write_text("".join(l + "\n" for l in kept + [json.dumps(e, ensure_ascii=False) for e in new]), encoding="utf-8")
|
||||
os.replace(temp, f)
|
||||
copy.write_text(text, encoding="utf-8")
|
||||
return len(new)
|
||||
|
||||
|
||||
def state(name):
|
||||
out = FOLDER / name
|
||||
if "/" in name or name.startswith(".") or not out.is_dir():
|
||||
raise station.Refusal("unknown analysis")
|
||||
if name not in analyses: # after a restart of the server, only the folder is left: the thread is read again in the log
|
||||
log = out / "work" / "log.jsonl"
|
||||
lines = [r for l in open(log, encoding="utf-8") for r in digest(l)] if log.exists() else []
|
||||
request = json.loads((out / "request.json").read_text(encoding="utf-8"))
|
||||
analyses[name] = {"state": "done" if (out / "report.md").exists() else "interrupted", "lines": lines,
|
||||
"submitted": None, "station": request["station"], "request": request}
|
||||
a, r = analyses[name], out / "report.md"
|
||||
return {"name": name, "state": a["state"], "lines": a["lines"], "submitted": a["submitted"],
|
||||
"report": int(r.stat().st_mtime) if r.exists() else None,
|
||||
"dialogue": (out / "work" / "session").exists(), "request": a["request"], "optimization": optimization(name)}
|
||||
|
||||
|
||||
def list_analyses():
|
||||
"""The history of the analyses (the sidebar), the most recent first: those of the server with their state, the others
|
||||
read on disk (done with a report, otherwise interrupted: an analysis stops with the server)."""
|
||||
out = []
|
||||
for d in sorted(FOLDER.glob("*/request.json"), key=lambda f: f.parent.stat().st_mtime, reverse=True):
|
||||
name = d.parent.name
|
||||
a = analyses.get(name) or {"state": "done" if (d.parent / "report.md").exists() else "interrupted",
|
||||
"request": json.loads(d.read_text(encoding="utf-8"))}
|
||||
out.append({"name": name, "state": a["state"]} | a["request"])
|
||||
return out
|
||||
|
||||
|
||||
def file(name, path):
|
||||
"""A file of the folder of the analysis (the report, its figures), never outside."""
|
||||
f = (FOLDER / name / path).resolve()
|
||||
if not f.is_file() or not f.is_relative_to(FOLDER.resolve()):
|
||||
raise station.Refusal("unknown file")
|
||||
return f
|
||||
|
||||
|
||||
STYLE = """body{font:14px/1.55 Roboto,sans-serif;margin:0;padding:4px 2px;color:#212529;background:transparent}
|
||||
h1{font-size:20px}h2{font-size:16px;margin-top:22px}img{max-width:100%;border:1px solid #dee2e6;border-radius:4px}
|
||||
code{font-size:12px;background:#f1f3f5;padding:1px 4px;border-radius:3px}pre{overflow-x:auto}
|
||||
table{border-collapse:collapse}td,th{border:1px solid #dee2e6;padding:3px 8px;text-align:left}
|
||||
body.dark{color:#c9c9c9}body.dark code{background:#2e2e2e}body.dark td,body.dark th,body.dark img{border-color:#424242}"""
|
||||
|
||||
|
||||
def _page(text, title, dark):
|
||||
"""A markdown in HTML, in the style of the page; its relative images resolve in the folder of the address."""
|
||||
body = markdown.markdown(text, extensions=["tables", "fenced_code"])
|
||||
return (f"<!doctype html><meta charset=utf-8><title>{html.escape(title)}</title><style>{STYLE}</style>"
|
||||
f"<body class='{'dark' if dark else ''}'>{body}</body>")
|
||||
|
||||
|
||||
def report(name, dark):
|
||||
"""report.md in HTML, served in the folder of the analysis: its relative images are found there."""
|
||||
return _page(file(name, "report.md").read_text(encoding="utf-8"), name, dark)
|
||||
|
||||
|
||||
# --- "Optimize": /improve in Optimizer on the analysis, then the tool-builder, by claude -p
|
||||
|
||||
def _summary_file(name):
|
||||
"""The summary of the last session of this analysis, or None. The time pattern rules out the sessions of another
|
||||
analysis of the same range (<name>_<HHMMSS>_<time>)."""
|
||||
summaries = sorted(SESSIONS.glob(f"{name}_????????T????Z/summary.md")) # the order of the name is that of time
|
||||
return summaries[-1] if summaries else None
|
||||
|
||||
|
||||
def optimization(name):
|
||||
"""-> None (never optimized) or {state, lines, summary: mtime of the last summary}. After a restart, the state is read
|
||||
again in the stream: its last line {"type": "end"} says done or error; without it, the optimization was cut
|
||||
(interrupted)."""
|
||||
stream, s = FOLDER / name / "work" / "optimization.jsonl", _summary_file(name)
|
||||
o = optimizations.get(name)
|
||||
if o is None:
|
||||
if not stream.exists():
|
||||
return None
|
||||
lines = stream.read_text(encoding="utf-8").splitlines()
|
||||
end = json.loads(lines[-1]) if lines and lines[-1].startswith('{"type": "end"') else None
|
||||
o = {"state": "interrupted" if end is None else end["state"], "lines": [r for l in lines for r in digest(l)]}
|
||||
optimizations[name] = o
|
||||
return {"state": o["state"], "lines": o["lines"], "summary": int(s.stat().st_mtime) if s else None}
|
||||
|
||||
|
||||
def optimize(name):
|
||||
"""The "Optimize" button on a finished analysis: the session in a thread; refused if an optimization or an analysis runs."""
|
||||
if state(name)["state"] != "done":
|
||||
raise station.Refusal("only a finished analysis can be optimized")
|
||||
if not ((OPTIMIZER / ".claude" / "skills" / "improve" / "SKILL.md").exists()
|
||||
and all((OPTIMIZER / link).exists() for link in ("data", "labeling", "detectors"))
|
||||
and (AGENTS / ".claude-plugin" / "plugin.json").exists()):
|
||||
raise station.Refusal("no Optimizer in optimizer/ or no agents in agents/ (the links: README.md)")
|
||||
with lock:
|
||||
if _optimizing():
|
||||
raise station.Refusal(BUSY)
|
||||
if any(a["state"] == "running" for a in analyses.values()):
|
||||
raise station.Refusal("an analysis is running: the optimization waits for its end")
|
||||
optimizations[name] = {"state": "running", "lines": []}
|
||||
threading.Thread(target=optimize_thread, args=(name,), daemon=True).start()
|
||||
return {}
|
||||
|
||||
|
||||
def optimize_thread(name):
|
||||
"""The two turns of the optimization, one after the other: the detectors (/improve, in Optimizer), then the reading
|
||||
tools (the tool-builder, even if the detectors did not change)."""
|
||||
o, out = optimizations[name], FOLDER / name
|
||||
rel = os.path.relpath(out, station.ROOT)
|
||||
(out / "work").mkdir(exist_ok=True)
|
||||
with open(out / "work" / "optimization.jsonl", "a", encoding="utf-8") as stream:
|
||||
try:
|
||||
d = json.loads((out / "request.json").read_text(encoding="utf-8"))
|
||||
request = IMPROVE.format( # TO excluded: the day after the end of the range, as in the page
|
||||
station=d["station"], start=d["start"][:10], end=date.fromisoformat(d["end"][:10]) + timedelta(days=1),
|
||||
name=name, time=f"{datetime.now(timezone.utc):%Y%m%dT%H%MZ}")
|
||||
# ponytail: Optimizer reads data/ and labeling/ through its own links; AIMSIGHT_DATA / AIMSIGHT_LABELS do not follow
|
||||
turns = [(OPTIMIZER.resolve(), [request, "--plugin-dir", str(AGENTS.resolve()),
|
||||
"--add-dir", str(DETECTORS.resolve()), # the target of the link, where the session writes
|
||||
"--allowedTools", "Read,Write,Edit,Bash,Agent"]),
|
||||
(station.ROOT, [f"Analysis: {rel}/. Tools: tools/. Write {rel}/tools.md.", "--plugin-dir", str(AGENTS),
|
||||
"--agent", "agents:tool-builder", "--allowedTools", "Read,Write,Edit,Bash"])]
|
||||
for cwd, (prompt, *options) in turns:
|
||||
p = subprocess.Popen(["claude", "-p", prompt, *ISOLATED, "--output-format", "stream-json", "--verbose",
|
||||
"--permission-mode", "acceptEdits", *station.links(), *options],
|
||||
cwd=cwd, env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
for line in p.stdout:
|
||||
stream.write(line)
|
||||
o["lines"] += digest(line)
|
||||
if p.wait():
|
||||
raise RuntimeError(f"claude ended with code {p.returncode}")
|
||||
o["state"] = "done"
|
||||
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
|
||||
o["lines"].append(f"Error: {e}")
|
||||
o["state"] = "error"
|
||||
finally:
|
||||
stream.write(json.dumps({"type": "end", "state": o["state"]}) + "\n")
|
||||
station.detectors.cache_clear() # the detectors may have changed: the Data page computes them again
|
||||
|
||||
|
||||
def summary(name, dark):
|
||||
"""The summary of the detectors' session, then that of the tools (<analysis>/tools.md), in one page."""
|
||||
s, tools = _summary_file(name), FOLDER / name / "tools.md"
|
||||
if s is None and not tools.exists():
|
||||
raise station.Refusal("no summary for this analysis")
|
||||
text = "\n\n".join(f.read_text(encoding="utf-8") for f in (s, tools) if f and f.exists())
|
||||
return _page(text, name, dark)
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Checks of the web app, without calling claude: .venv/bin/python app/server/check.py [name]"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
import uvicorn # noqa: E402
|
||||
import server # noqa: E402
|
||||
import station # noqa: E402
|
||||
|
||||
|
||||
def temp_folder():
|
||||
return Path(tempfile.mkdtemp())
|
||||
|
||||
|
||||
def check_server():
|
||||
# the real server on a free port: the guard (Host, declared JSON), a read
|
||||
with socket.socket() as so:
|
||||
so.bind(("127.0.0.1", 0))
|
||||
port = so.getsockname()[1]
|
||||
srv = uvicorn.Server(uvicorn.Config(server.app, host="127.0.0.1", port=port, log_level="error"))
|
||||
threading.Thread(target=srv.run, daemon=True).start()
|
||||
while not srv.started:
|
||||
time.sleep(0.05)
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
|
||||
def call(path, body=None, headers=None, method=None):
|
||||
r = urllib.request.Request(base + path, json.dumps(body).encode() if body is not None else None,
|
||||
{"Content-Type": "application/json", **(headers or {})}, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(r) as rep:
|
||||
return rep.status
|
||||
except HTTPError as e:
|
||||
return e.code
|
||||
try:
|
||||
assert call("/api/stations", headers={"Host": "evil.example:80"}) == 403
|
||||
assert call("/api/labels", {"station": "x"}, {"Content-Type": "text/plain"}) == 403
|
||||
assert call("/api/stations") == 200
|
||||
assert call("/api/analyses") == 200
|
||||
assert call("/api/analyses/UNKNOWN/optimize", {}) == 400
|
||||
finally:
|
||||
srv.should_exit = True
|
||||
|
||||
|
||||
def refuses(f, *a):
|
||||
try:
|
||||
f(*a)
|
||||
except station.Refusal as e:
|
||||
return str(e)
|
||||
raise AssertionError(f"{f.__name__}{a} did not refuse")
|
||||
|
||||
|
||||
def check_station():
|
||||
# the data of the Data page, on the real data; the labels in a temporary folder
|
||||
assert "NIEDERHORN" in station.stations()
|
||||
im = station.images("NIEDERHORN", "2026-09-25", "2026-09-26")
|
||||
assert im["images"] and im["positions"] and set(im["corr"]) == set(im["positions"]), list(im)
|
||||
assert all(len(c) == len(im["images"]) for c in im["corr"].values())
|
||||
assert station.image("NIEDERHORN", im["images"][0]["name"]).is_file()
|
||||
refuses(station.image, "NIEDERHORN", "../../../.env")
|
||||
refuses(station.images, "../API", "", "")
|
||||
assert station.images("LENZSPITZE", "2026-10-26", "2026-10-27")["images"]
|
||||
c = station.curves("LENZSPITZE", "2026-10-26", "2026-10-27")
|
||||
titles = [p["title"] for p in c["plots"]]
|
||||
assert "Relative Corrected Vertical Shift" in titles and any(t.startswith("System: ") for t in titles), titles
|
||||
ev = station.detectors("LENZSPITZE", "2026-10-26", "2026-10-27") # the detectors of the journal, ~10 s
|
||||
assert ev and {"start", "end", "family", "description", "sensors"} <= set(ev[0]), ev[:1]
|
||||
assert all(e["message"] and e["action"] and set(e["targets"] + e["references"]) == set(e["sensors"]) for e in ev), ev[:1]
|
||||
assert station.message("A moved; nothing to do. The jump of 2 targets in this period has the same cause.") \
|
||||
== "A moved. The jump of 2 targets in this period has the same cause."
|
||||
assert all(e["end"] >= "2026-10-26T00:00Z" and e["start"] < "2026-10-27T00:00Z" for e in ev), [(e["start"], e["end"]) for e in ev]
|
||||
assert not any("SIM_CCID" in t or t.startswith("System: ip") for t in titles)
|
||||
assert all(len(c["T"][tr["t"]]) == len(tr["y"]) for p in c["plots"] for tr in p["traces"])
|
||||
before, station.LABELS = station.LABELS, temp_folder()
|
||||
try:
|
||||
e = station.add({"station": "NIEDERHORN", "folder": "auto", "start": "2026-09-25T01:00Z",
|
||||
"end": "2026-09-25T02:00Z", "family": "Visibility", "description": "fog"})
|
||||
assert e["id"] == "NIEDERHORN-1" and next(iter(e)) == "id", e
|
||||
e2 = station.update({"station": "NIEDERHORN", "folder": "auto", "before": e,
|
||||
"start": e["start"], "end": e["end"], "family": "", "description": "rain"})
|
||||
assert e2 == {"id": "NIEDERHORN-1", "start": e["start"], "end": e["end"], "description": "rain"}, e2
|
||||
assert station.read("NIEDERHORN", "auto") == [e2]
|
||||
assert "no longer in the file" in refuses(station.delete, {"station": "NIEDERHORN", "folder": "auto", "before": e})
|
||||
station.delete({"station": "NIEDERHORN", "folder": "auto", "before": e2})
|
||||
assert station.read("NIEDERHORN", "auto") == []
|
||||
refuses(station.add, {"station": "NIEDERHORN", "folder": "../x", "start": "a", "end": "b", "description": "x"})
|
||||
refuses(station.add, {"station": "NIEDERHORN", "start": "2026-09-25T02:00Z", "end": "2026-09-25T01:00Z", "description": "x"})
|
||||
finally:
|
||||
shutil.rmtree(station.LABELS)
|
||||
station.LABELS = before
|
||||
|
||||
|
||||
def check_optimization():
|
||||
# the Optimize button (analysis.optimize), without claude: a fake process that ends only on a signal, a finished
|
||||
# analysis copied into a temporary folder; the lock (one optimization at a time, no analysis or message meanwhile), the
|
||||
# state read again after a restart
|
||||
import analysis
|
||||
end = threading.Event()
|
||||
|
||||
launched = []
|
||||
|
||||
class FakeProcess:
|
||||
returncode = 0
|
||||
stdout = iter(lambda: end.wait() and "", "") # nothing, until the signal
|
||||
|
||||
def __init__(self, args, *a, **k):
|
||||
launched.append((args, k))
|
||||
|
||||
def wait(self):
|
||||
return 0
|
||||
|
||||
before = analysis.FOLDER, analysis.subprocess.Popen, analysis.OPTIMIZER, analysis.SESSIONS, analysis.AGENTS
|
||||
key = os.environ.get("ANTHROPIC_API_KEY")
|
||||
os.environ["ANTHROPIC_API_KEY"] = "test-key"
|
||||
root = temp_folder()
|
||||
analysis.FOLDER, analysis.OPTIMIZER, analysis.AGENTS = root / "analyses", root / "optimizer", root / "agents"
|
||||
analysis.SESSIONS = analysis.OPTIMIZER / "sessions"
|
||||
analysis.subprocess.Popen = FakeProcess
|
||||
try:
|
||||
for name in ("NORDEND_a", "NORDEND_b"):
|
||||
(analysis.FOLDER / name / "work").mkdir(parents=True)
|
||||
(analysis.FOLDER / name / "request.json").write_text(json.dumps(
|
||||
{"station": "NORDEND", "start": "2026-11-04T21:12Z", "end": "2026-11-07T14:12Z", "hints": ""}))
|
||||
(analysis.FOLDER / name / "report.md").write_text("# report")
|
||||
assert analysis.state("NORDEND_a")["optimization"] is None
|
||||
# without Optimizer, then without agents: a clear refusal, not an error trace
|
||||
assert "optimizer/" in refuses(analysis.optimize, "NORDEND_a")
|
||||
(analysis.OPTIMIZER / ".claude" / "skills" / "improve").mkdir(parents=True)
|
||||
(analysis.OPTIMIZER / ".claude" / "skills" / "improve" / "SKILL.md").write_text("improve")
|
||||
assert "agents/" in refuses(analysis.optimize, "NORDEND_a")
|
||||
(analysis.AGENTS / ".claude-plugin").mkdir(parents=True)
|
||||
(analysis.AGENTS / ".claude-plugin" / "plugin.json").write_text("{}")
|
||||
assert "optimizer/" in refuses(analysis.optimize, "NORDEND_a") # without the data, labeling, detectors links of Optimizer
|
||||
for link in ("data", "labeling", "detectors"):
|
||||
(analysis.OPTIMIZER / link).mkdir()
|
||||
assert analysis.optimize("NORDEND_a") == {}
|
||||
assert analysis.state("NORDEND_a")["optimization"]["state"] == "running"
|
||||
refuses(analysis.optimize, "NORDEND_a")
|
||||
refuses(analysis.optimize, "NORDEND_b")
|
||||
refuses(analysis.launch, {"station": "NORDEND", "start": "2026-11-04T00:00Z", "end": "2026-11-05T00:00Z"})
|
||||
refuses(analysis.message, "NORDEND_b", {"text": "continue"})
|
||||
end.set()
|
||||
while analysis.state("NORDEND_a")["optimization"]["state"] == "running":
|
||||
time.sleep(0.05)
|
||||
assert analysis.state("NORDEND_a")["optimization"]["state"] == "done"
|
||||
args, k = launched[0] # /improve in Optimizer, on the range of the analysis, TO excluded
|
||||
request = args[2]
|
||||
assert request.startswith("/improve ") and "STATION: data/NORDEND. FROM: 2026-11-04. TO: 2026-11-08." in request \
|
||||
and "OUT: sessions/NORDEND_a_" in request and "FAMILIES: labeling/families.md." in request \
|
||||
and "CONTEXT: labeling/auto/analyses/NORDEND_a/report.md." in request, request
|
||||
assert k["cwd"] == analysis.OPTIMIZER.resolve() and "--plugin-dir" in args, (args, k)
|
||||
assert all("ANTHROPIC_API_KEY" not in kw["env"] for _, kw in launched), "an API key reaches claude"
|
||||
# isolated from the settings of the workstation (the user's plugins, hooks, MCP servers)
|
||||
assert all(a[a.index("--setting-sources") + 1] == "project,local" and "--strict-mcp-config" in a for a, _ in launched)
|
||||
args, k = launched[1] # then the tool-builder of the agents repository, on the tools repository
|
||||
assert args[args.index("--agent") + 1] == "agents:tool-builder" and "Tools: tools/" in args[2] \
|
||||
and args[2].endswith("/tools.md."), args
|
||||
analysis.analyses["NORDEND_b"]["state"] = "running" # a running analysis blocks the optimization
|
||||
refuses(analysis.optimize, "NORDEND_a")
|
||||
# after a restart: the stream ends with {"type": "end"} -> done; without that line -> interrupted; the summary of
|
||||
# the detectors and that of the tools in one page
|
||||
stream = analysis.FOLDER / "NORDEND_a" / "work" / "optimization.jsonl"
|
||||
assert stream.read_text().endswith('{"type": "end", "state": "done"}\n'), stream.read_text()
|
||||
analysis.analyses.clear()
|
||||
analysis.optimizations.clear()
|
||||
assert analysis.state("NORDEND_a")["optimization"]["state"] == "done"
|
||||
stream.write_text('{"type": "assistant", "message": {"content": []}}\n')
|
||||
analysis.optimizations.clear()
|
||||
assert analysis.state("NORDEND_a")["optimization"]["state"] == "interrupted"
|
||||
(analysis.SESSIONS / "NORDEND_a_20261113T1000Z").mkdir(parents=True)
|
||||
(analysis.SESSIONS / "NORDEND_a_20261113T1000Z" / "summary.md").write_text("# improve\n\n**Result**: none")
|
||||
# a new analysis of the same range (NORDEND_a_143000): its session is not the summary of NORDEND_a
|
||||
(analysis.SESSIONS / "NORDEND_a_143000_20261113T1100Z").mkdir(parents=True)
|
||||
(analysis.SESSIONS / "NORDEND_a_143000_20261113T1100Z" / "summary.md").write_text("# other analysis")
|
||||
assert analysis._summary_file("NORDEND_a").parent.name == "NORDEND_a_20261113T1000Z", analysis._summary_file("NORDEND_a")
|
||||
(analysis.FOLDER / "NORDEND_a" / "tools.md").write_text("# Tools\n\n**Tools changed**: none")
|
||||
assert analysis.state("NORDEND_a")["optimization"]["summary"]
|
||||
page = analysis.summary("NORDEND_a", False)
|
||||
assert "<strong>Result</strong>" in page and "<strong>Tools changed</strong>" in page, page
|
||||
assert analysis.state("NORDEND_b")["optimization"] is None
|
||||
finally:
|
||||
end.set()
|
||||
os.environ.pop("ANTHROPIC_API_KEY")
|
||||
if key is not None:
|
||||
os.environ["ANTHROPIC_API_KEY"] = key
|
||||
shutil.rmtree(root)
|
||||
analysis.FOLDER, analysis.subprocess.Popen, analysis.OPTIMIZER, analysis.SESSIONS, analysis.AGENTS = before
|
||||
analysis.analyses.clear()
|
||||
analysis.optimizations.clear()
|
||||
|
||||
|
||||
def check_analysis():
|
||||
# "Analyze the range" then a message (analysis.launch, analysis.message), without claude: the analyst of the agents
|
||||
# repository, its parameters, and the resumption of its session with the same agent
|
||||
import analysis
|
||||
launched = []
|
||||
|
||||
class FakeProcess:
|
||||
returncode = 0
|
||||
|
||||
def __init__(self, args, *a, **k):
|
||||
launched.append((args, k))
|
||||
self.stdout = iter([])
|
||||
|
||||
def wait(self):
|
||||
return 0
|
||||
|
||||
(station.ROOT / "out").mkdir(exist_ok=True) # under the root: the request cites the output folder relative to it
|
||||
before = analysis.FOLDER, analysis.subprocess.Popen, analysis.submit
|
||||
analysis.FOLDER = Path(tempfile.mkdtemp(dir=station.ROOT / "out"))
|
||||
analysis.subprocess.Popen = FakeProcess
|
||||
analysis.submit = lambda *a: 0 # otherwise the test rewrites the real labeling/auto/NORDEND.jsonl
|
||||
key = os.environ.get("ANTHROPIC_API_KEY")
|
||||
os.environ["ANTHROPIC_API_KEY"] = "test-key" # the agent runs on the subscription, never on an API key
|
||||
try:
|
||||
name = analysis.launch({"station": "NORDEND", "start": "2026-11-05T06:00Z", "end": "2026-11-05T12:00Z"})["name"]
|
||||
while analysis.analyses[name]["state"] == "running":
|
||||
time.sleep(0.05)
|
||||
assert analysis.analyses[name]["state"] == "done", analysis.analyses[name]
|
||||
assert json.loads((analysis.FOLDER / name / "request.json").read_text()) == \
|
||||
{"station": "NORDEND", "start": "2026-11-05T06:00Z", "end": "2026-11-05T12:00Z", "hints": ""}
|
||||
args, k = launched[0]
|
||||
assert args[args.index("--agent") + 1] == "agents:analyst" and "--plugin-dir" in args, args
|
||||
assert "ANTHROPIC_API_KEY" not in k["env"] and "PATH" in k["env"], sorted(k.get("env") or {})
|
||||
assert args[2].startswith("Station: data/NORDEND/. Range: 2026-11-05T06:00Z → 2026-11-05T12:00Z (UTC). Hints: none. ") \
|
||||
and "DETECTORS: no" in args[2] and "INSTRUCTIONS: instructions/analyst.md" in args[2], args[2]
|
||||
assert args[args.index("--setting-sources") + 1] == "project,local" and "--strict-mcp-config" in args, args
|
||||
(analysis.FOLDER / name / "work" / "session").write_text("s-1")
|
||||
analysis.message(name, {"text": "continue"})
|
||||
while analysis.analyses[name]["state"] == "running":
|
||||
time.sleep(0.05)
|
||||
args, k = launched[1]
|
||||
assert "ANTHROPIC_API_KEY" not in k["env"], sorted(k.get("env") or {})
|
||||
assert args[args.index("--resume") + 1] == "s-1" and args[args.index("--agent") + 1] == "agents:analyst" \
|
||||
and "--plugin-dir" in args, args
|
||||
agents, analysis.AGENTS = analysis.AGENTS, analysis.FOLDER / "no-agents" # the agents/ link is missing: clear refusal
|
||||
try:
|
||||
assert "agents/" in refuses(analysis.launch, {"station": "NORDEND", "start": "2026-11-06T06:00Z", "end": "2026-11-06T12:00Z"})
|
||||
finally:
|
||||
analysis.AGENTS = agents
|
||||
finally:
|
||||
os.environ.pop("ANTHROPIC_API_KEY")
|
||||
if key is not None:
|
||||
os.environ["ANTHROPIC_API_KEY"] = key
|
||||
shutil.rmtree(analysis.FOLDER)
|
||||
analysis.FOLDER, analysis.subprocess.Popen, analysis.submit = before
|
||||
analysis.analyses.clear()
|
||||
|
||||
|
||||
CHECKS = [check_server, check_station, check_optimization, check_analysis]
|
||||
|
||||
if __name__ == "__main__":
|
||||
name = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
for f in CHECKS:
|
||||
if name in (None, f.__name__):
|
||||
f()
|
||||
print("ok", f.__name__)
|
||||
@@ -0,0 +1,165 @@
|
||||
"""The web app: .venv/bin/python app/server/server.py → http://127.0.0.1:8770
|
||||
|
||||
The FastAPI server serves the built React page (app/web/dist, `npm run build` in app/web) and the API under /api.
|
||||
The Data page reads data/ and the labels of labeling/ (station.py); its Analysis tab launches the analyst, and the
|
||||
optimization of the detectors (analysis.py).
|
||||
Settings: AIMSIGHT_DATA (data), AIMSIGHT_LABELS (labels), AIMSIGHT_PORT (port, 8770).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import webbrowser
|
||||
from pathlib import Path
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Query, Request
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
|
||||
import analysis
|
||||
import station
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
APP = Path(__file__).resolve().parents[1]
|
||||
|
||||
PORT = int(os.environ.get("AIMSIGHT_PORT", 8770))
|
||||
HOSTS = {"127.0.0.1", "localhost"} # against DNS rebinding: another site does not read the data
|
||||
PAGE = APP / "web" / "dist"
|
||||
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
def error(code: int, text: str) -> JSONResponse:
|
||||
return JSONResponse({"error": text}, code)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def guard(request: Request, call_next):
|
||||
"""Only the page itself talks to the server: right Host; in POST, a declared JSON (a page of another site cannot send
|
||||
it without a preflight request, to which the server does not answer)."""
|
||||
host = request.headers.get("host", "").rsplit(":", 1)[0]
|
||||
if host not in HOSTS or (request.method in ("POST", "PUT", "DELETE")
|
||||
and not request.headers.get("content-type", "").startswith("application/json")):
|
||||
return error(403, "request refused")
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# --- the Data page (station.py)
|
||||
|
||||
@app.exception_handler(station.Refusal)
|
||||
def refusal(_, e):
|
||||
return error(400, str(e))
|
||||
|
||||
|
||||
@app.get("/api/stations")
|
||||
def stations():
|
||||
return station.stations()
|
||||
|
||||
|
||||
@app.get("/api/images")
|
||||
def images(station_: str = Query(alias="station"), start: str = "", end: str = ""):
|
||||
return station.images(station_, start, end)
|
||||
|
||||
|
||||
@app.get("/api/image/{station_}/{name}")
|
||||
def image(station_: str, name: str):
|
||||
return FileResponse(station.image(station_, name), headers={"Cache-Control": "max-age=86400"})
|
||||
|
||||
|
||||
@app.get("/api/curves")
|
||||
def curves(station_: str = Query(alias="station"), start: str = "", end: str = ""):
|
||||
return station.curves(station_, start, end)
|
||||
|
||||
|
||||
@app.get("/api/detectors")
|
||||
def detectors(station_: str = Query(alias="station"), start: str = "", end: str = "", targets: str = ""):
|
||||
return station.detectors(station_, start, end, targets)
|
||||
|
||||
|
||||
@app.get("/api/labels")
|
||||
def labels(station_: str = Query(alias="station"), folder: str = "manual"):
|
||||
try:
|
||||
return station.read(station_, folder)
|
||||
except json.JSONDecodeError as e: # a line damaged by hand: say so, without touching the file
|
||||
return error(500, f"labeling/{folder}/{station_}.jsonl unreadable: {e}")
|
||||
|
||||
|
||||
@app.post("/api/labels")
|
||||
def add_label(body: dict):
|
||||
return station.add(body)
|
||||
|
||||
|
||||
@app.put("/api/labels")
|
||||
def update_label(body: dict):
|
||||
return station.update(body)
|
||||
|
||||
|
||||
@app.delete("/api/labels")
|
||||
def delete_label(body: dict):
|
||||
return station.delete(body)
|
||||
|
||||
|
||||
# --- the Analysis tab of the Data page (analysis.py): the analyst launched by claude -p on a range
|
||||
|
||||
@app.post("/api/analyses")
|
||||
def launch_analysis(body: dict):
|
||||
return analysis.launch(body)
|
||||
|
||||
|
||||
@app.get("/api/analyses")
|
||||
def analyses():
|
||||
return analysis.list_analyses()
|
||||
|
||||
|
||||
@app.get("/api/analyses/{name}")
|
||||
def analysis_state(name: str):
|
||||
return analysis.state(name)
|
||||
|
||||
|
||||
@app.post("/api/analyses/{name}/message")
|
||||
def analysis_message(name: str, body: dict):
|
||||
return analysis.message(name, body)
|
||||
|
||||
|
||||
@app.get("/api/analyses/{name}/report.html")
|
||||
def analysis_report(name: str, dark: bool = False):
|
||||
return HTMLResponse(analysis.report(name, dark))
|
||||
|
||||
|
||||
@app.post("/api/analyses/{name}/optimize")
|
||||
def optimize_analysis(name: str):
|
||||
return analysis.optimize(name)
|
||||
|
||||
|
||||
@app.get("/api/analyses/{name}/summary.html")
|
||||
def analysis_summary(name: str, dark: bool = False):
|
||||
return HTMLResponse(analysis.summary(name, dark))
|
||||
|
||||
|
||||
@app.get("/api/analyses/{name}/{path:path}")
|
||||
def analysis_file(name: str, path: str):
|
||||
return FileResponse(analysis.file(name, path))
|
||||
|
||||
|
||||
if PAGE.is_dir():
|
||||
app.mount("/", StaticFiles(directory=PAGE, html=True), name="page")
|
||||
else:
|
||||
@app.get("/")
|
||||
def no_page():
|
||||
return error(503, "the page is not built: cd app/web && npm install && npm run build")
|
||||
|
||||
|
||||
def open_browser(url: str) -> None:
|
||||
if "microsoft" in platform.uname().release.lower(): # WSL: the Windows browser
|
||||
subprocess.Popen(["explorer.exe", url], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
else:
|
||||
webbrowser.open(url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
url = f"http://127.0.0.1:{PORT}"
|
||||
print(f"AIMsight diagnostic: {url}")
|
||||
if "--no-browser" not in sys.argv:
|
||||
threading.Timer(1, open_browser, [url]).start()
|
||||
uvicorn.run(app, host="127.0.0.1", port=PORT, log_level="warning", timeout_graceful_shutdown=1)
|
||||
@@ -0,0 +1,338 @@
|
||||
"""The data of the Data page: a station's images, its curves, its labels.
|
||||
|
||||
Data are read as the agents read them, tools/scripts/data.py (the tools repository, through the tools/ link); the detectors'
|
||||
events are those of detectors/scripts/categories.py (events). Read-only on data/; writes only in labeling/{auto,manual}/.
|
||||
"""
|
||||
import bisect
|
||||
import functools
|
||||
import csv
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path[:0] = [str(ROOT / "tools" / "scripts"), str(ROOT / "detectors" / "scripts")]
|
||||
import data # noqa: E402
|
||||
from categories import events # noqa: E402
|
||||
|
||||
# data/ and labeling/ are links to ~/aimsight-data/data and ~/labeling: no resolve(), the agents see data/ and labeling/
|
||||
DATA = Path(os.environ.get("AIMSIGHT_DATA", ROOT / "data")).absolute()
|
||||
LABELS = Path(os.environ.get("AIMSIGHT_LABELS", ROOT / "labeling")).absolute()
|
||||
TOOLS = ROOT / "tools" # the reading harness (~/tools), read by the analyst and written by the tool-builder
|
||||
|
||||
|
||||
def links():
|
||||
"""--add-dir for claude -p: the real target of the data/, labeling/ and tools/ links, outside the repository."""
|
||||
return [a for d in (DATA, LABELS, TOOLS) for a in ("--add-dir", str(d.resolve()))]
|
||||
|
||||
|
||||
FOLDERS = ("auto", "manual") # those of the agents and of the page (D-105), those of the expert (D-102)
|
||||
FAMILIES = ("Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data", "None") # D-109
|
||||
FORMAT = "%Y-%m-%dT%H:%MZ"
|
||||
MARGIN_PX = 150 # Support-Agent detecteurs.MARGE_PX: x, y are positions in the image only far from the corner
|
||||
COLORS = ["#2a78d6", "#eb6834", "#1baf7a", "#8a5cd6", "#d6a12a", "#d63b8a", "#3bb3d6", "#6b6b66"] # one per sensor
|
||||
POSE = (("pan", "#1a1a19"), ("tilt", "#8a8a85"), ("rotation", "#b5651d"))
|
||||
# The Advanced menu of Explore, in its order (Support-Agent labeling.PLOTS, QUALITE)
|
||||
PLOTS = (("error_value", "Error value"), ("x", "X [px]"), ("y", "Y [px]"),
|
||||
("relative_x", "Relative Horizontal Shift"), ("relative_y", "Relative Vertical Shift"),
|
||||
("corr_x", "Relative Corrected Horizontal Shift"), ("corr_y", "Relative Corrected Vertical Shift"),
|
||||
("reproj_err", "References (reproj. error)"))
|
||||
QUALITY = (("peak", "Quality: Peak"), ("flux", "Quality: Flux"), ("sharpness", "Quality: Sharpness"),
|
||||
("fwhm", "Quality: FWHM [px]"), ("dominance", "Quality: Dominance"), ("snr", "Quality: SNR"))
|
||||
IDENTIFIER = re.compile(r"(SIM_CCID|ip(_.*)?)$") # never read (CLAUDE.md, "data/")
|
||||
METRIC = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
|
||||
|
||||
|
||||
class Refusal(ValueError):
|
||||
"""A request to refuse (400), with its reason."""
|
||||
|
||||
|
||||
def stations():
|
||||
return sorted(d.name for d in DATA.iterdir() if d.is_dir())
|
||||
|
||||
|
||||
def folder(station):
|
||||
if station not in stations(): # the name becomes a path: a folder of data/, nothing else
|
||||
raise Refusal(f"unknown station: {station}")
|
||||
return DATA / station
|
||||
|
||||
|
||||
def bounds(start, end):
|
||||
return data.parse_time(start or "0001-01-01"), data.parse_time(end or "9999-01-01")
|
||||
|
||||
|
||||
# --- the positions of the sensors in the image
|
||||
|
||||
def positions(measurements, a, b):
|
||||
"""The median position of each sensor in the image over [a, b) ({} if x, y are not positions there)."""
|
||||
homes = {s: (statistics.median(xs), statistics.median(ys)) for s, ms in measurements.items()
|
||||
if (xs := [m["x"] for m in ms if a <= m["t"] < b and m["x"] is not None])
|
||||
and (ys := [m["y"] for m in ms if a <= m["t"] < b and m["y"] is not None])}
|
||||
return homes if any(abs(x) > MARGIN_PX or abs(y) > MARGIN_PX for x, y in homes.values()) else {}
|
||||
|
||||
|
||||
def references(measurements, a, b):
|
||||
"""Sensors of the compensation over the range: those with a reproj_err (references only)."""
|
||||
return {s for s, ms in measurements.items() if any(m["reproj_err"] is not None for m in ms if a <= m["t"] < b)}
|
||||
|
||||
|
||||
# --- the images of the Data page
|
||||
|
||||
def images(station, start, end):
|
||||
"""The images of [start, end), the median positions of the sensors, the references, and for each circled sensor its
|
||||
corr_x, corr_y at the time of each image: the nearest valid measurement, within 30 min, otherwise 0."""
|
||||
d, (a, b) = folder(station), bounds(start, end)
|
||||
views = [(t, im) for im in sorted(glob.glob(str(d / "images" / "*"))) if (t := data.image_time(im)) and a <= t < b]
|
||||
if not views:
|
||||
raise Refusal(f"no image of {station} in this range")
|
||||
measurements = data.load(d)
|
||||
homes, refs = positions(measurements, a, b), references(measurements, a, b)
|
||||
# ponytail: a single measurement at each bound, noisy; a median over a window if the arrows shake too much
|
||||
margin, corr = timedelta(minutes=30), {}
|
||||
for s in homes:
|
||||
series = [(m["t"], m["corr_x"], m["corr_y"]) for m in measurements[s] if m["corr_x"] is not None and m["corr_y"] is not None]
|
||||
times, corr[s] = [m[0] for m in series], []
|
||||
for t, _ in views:
|
||||
k = bisect.bisect_left(times, t)
|
||||
k = min((n for n in (k - 1, k) if 0 <= n < len(series)), key=lambda n: abs(times[n] - t), default=None)
|
||||
corr[s].append([round(series[k][1], 4), round(series[k][2], 4)] if k is not None and abs(times[k] - t) <= margin else 0)
|
||||
return {"images": [{"t": data.iso(t).replace("T", " ").rstrip("Z"), "name": os.path.basename(im)} for t, im in views],
|
||||
"positions": {s: [round(x, 1), round(y, 1)] for s, (x, y) in sorted(homes.items())},
|
||||
"references": sorted(refs), "corr": corr}
|
||||
|
||||
|
||||
def image(station, name):
|
||||
f = (folder(station) / "images" / name).resolve()
|
||||
if f.parent != (DATA / station / "images").resolve() or not f.is_file():
|
||||
raise Refusal("unknown image")
|
||||
return f
|
||||
|
||||
|
||||
# --- the curves of the Data page
|
||||
|
||||
def metrics(d, a, b):
|
||||
"""The numeric system metrics of metrics/ over [a, b) -> {name: {t: value or None}} (an empty value: None, the gap
|
||||
shows), without the text metrics, the identifiers, or those with no value at all."""
|
||||
station, series, text = d.name + "_", {}, set()
|
||||
for p in sorted(glob.glob(str(d / "metrics" / "*.csv"))):
|
||||
if not (m := METRIC.match(os.path.basename(p))) or IDENTIFIER.match(name := m.group(1).removeprefix(station)):
|
||||
continue
|
||||
with open(p, newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
t, v = data.parse_time(r["time_utc"]), (r.get("metric") or "").strip("[]'\" ")
|
||||
if not a <= t < b:
|
||||
continue
|
||||
try:
|
||||
series.setdefault(name, {})[t] = float(v) if v else None
|
||||
except ValueError:
|
||||
text.add(name) # link, mode…: no curve
|
||||
return {n: dict(sorted(s.items())) for n, s in series.items()
|
||||
if n not in text and any(v is not None for v in s.values())}
|
||||
|
||||
|
||||
def span(measurements, start, end):
|
||||
"""The range read: from start to end (excluded), otherwise from the first measurement to the last, plus one step."""
|
||||
every = sorted({m["t"] for ms in measurements.values() for m in ms})
|
||||
a = data.parse_time(start) if start else every[0]
|
||||
b = data.parse_time(end) if end else every[-1] + (every[-1] - every[-2] if len(every) > 1 else timedelta(hours=1))
|
||||
return a, b
|
||||
|
||||
|
||||
def curves(station, start, end):
|
||||
"""-> {A, B, T, plots}: T, the times of each series, once (key: the sensor, "" for the camera, "system:" and its
|
||||
name for a metric); plot: title and traces (name, t, y, line, mode), without empty plots or sensors."""
|
||||
d = folder(station)
|
||||
measurements = data.load(d)
|
||||
if not measurements:
|
||||
raise Refusal(f"no \"advanced\" export for {station}")
|
||||
a, b = span(measurements, start, end)
|
||||
poses = [p for p in data.load_camera(d) if a <= p["t"] < b]
|
||||
hour = lambda t: t.strftime("%Y-%m-%d %H:%M") # noqa: E731 (UTC, without time zone: as Plotly shows it)
|
||||
values = lambda s, c: [None if m[c] is None else round(m[c], 5) for m in s] # noqa: E731
|
||||
inside = {s: [m for m in measurements[s] if a <= m["t"] < b] for s in sorted(measurements)}
|
||||
refs, out = references(measurements, a, b), []
|
||||
|
||||
def per_sensor(columns):
|
||||
for col, title in columns:
|
||||
traces = [{"name": s + " (ref.)" * (s in refs), "t": s, "y": y,
|
||||
"line": {"width": 1.2, "color": COLORS[i % len(COLORS)], "dash": "dot" if s in refs else "solid"}}
|
||||
for i, (s, y) in enumerate((s, values(ms, col)) for s, ms in inside.items()) if any(v is not None for v in y)]
|
||||
if traces:
|
||||
out.append({"title": title, "traces": traces})
|
||||
|
||||
per_sensor(PLOTS)
|
||||
if poses:
|
||||
out.append({"title": "Camera movement (deg)", "traces": [{"name": c, "t": "", "y": values(poses, c),
|
||||
"line": {"width": 1.2, "color": col}} for c, col in POSE]})
|
||||
per_sensor(QUALITY)
|
||||
T = {"": [hour(p["t"]) for p in poses], **{s: [hour(m["t"]) for m in ms] for s, ms in inside.items()}}
|
||||
system = metrics(d, a, b)
|
||||
for name in sorted(system, key=str.lower):
|
||||
T["system:" + name] = [hour(t) for t in system[name]]
|
||||
out.append({"title": "System: " + name, "traces": [{"name": name, "t": "system:" + name, "mode": "lines+markers",
|
||||
"y": [None if v is None else round(v, 5) for v in system[name].values()],
|
||||
"line": {"width": 1.2, "color": COLORS[0]}}]})
|
||||
if not out:
|
||||
raise Refusal(f"no measurement of {station} in this range")
|
||||
return {"A": data.iso(a), "B": data.iso(b), "T": T, "plots": out}
|
||||
|
||||
|
||||
# --- the Data page, "Detectors" labels: the events of the journal, read-only
|
||||
|
||||
HISTORY = timedelta(days=3) # days before given to the detectors, as categories.py does (rule of 2026-10-24)
|
||||
|
||||
|
||||
# ponytail: kept until the server restarts (a minute or more per range); restart it after an /improve run outside the app
|
||||
@functools.lru_cache(maxsize=32)
|
||||
def detectors(station, start, end, targets=""):
|
||||
"""-> the detectors' events over the range, in the label format (start, end, family, description); plus kind,
|
||||
sensors, ignore (greyed area of the journal). targets: the sensors kept, comma-separated (empty: all); the others are
|
||||
removed before the detectors, as if they did not exist."""
|
||||
d = folder(station)
|
||||
every = data.load(d)
|
||||
if targets:
|
||||
keep = set(targets.split(","))
|
||||
every = {s: ms for s, ms in every.items() if s in keep}
|
||||
if not every:
|
||||
raise Refusal(f"no \"advanced\" export for {station}" + " with these targets" * bool(targets))
|
||||
a, b = span(every, start, end)
|
||||
measurements = {s: k for s, ms in every.items() if (k := [m for m in ms if a - HISTORY <= m["t"] < b])}
|
||||
evs = [v for v in events(measurements, b, str(d)) if v["end"] >= data.iso(a)] if measurements else []
|
||||
refs = {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"}
|
||||
return [{"id": f"D-{i + 1}", "start": v["start"], "end": v["end"], "family": v["category"], "kind": v["episodes"][0]["label"],
|
||||
"description": f"{v['episodes'][0]['label']}. {v['text']}", "sensors": v["sensors"], "ignore": v["ignore"],
|
||||
"targets": [s for s in v["sensors"] if s not in refs], "references": [s for s in v["sensors"] if s in refs],
|
||||
"message": message(v["text"]), "action": action(v)}
|
||||
for i, v in enumerate(evs)]
|
||||
|
||||
|
||||
def message(text):
|
||||
"""What happens: the first sentence of the journal (detectors/scripts/categories.py, sentence), up to its ";"; the rest
|
||||
says what to do about it, except the sentence of the absorbed jumps, kept. ". " only: "1.5×" or "0.30 px" do not cut."""
|
||||
same = re.search(r" (The jumps? of .* same cause\.)$", text)
|
||||
return re.split(r"\. |; ", text, maxsplit=1)[0].rstrip(".") + "." + (" " + same.group(1) if same else "")
|
||||
|
||||
|
||||
# What to do about it: the "For the client" column of labeling/families.md, per family; refined per kind (the label of the
|
||||
# main episode, with its level) after the journal sentences validated on 2026-10-24 (detectors/scripts/categories.py, sentence)
|
||||
ACTION_FAMILY = {
|
||||
"Visibility": "Do not trust the curves in this period: this is not a movement.",
|
||||
"Obstruction": "Ignore the curves of the hidden targets in this period.",
|
||||
"Thermal displacement": "Normal, nothing to do.",
|
||||
"Displacement": "To check: look at the images, then keep watching.",
|
||||
"Target issue": "Act on the target (prism, mount); ignore its curve in the meantime.",
|
||||
"No data": "The curves are missing in this period; its cause tells whether to act.",
|
||||
}
|
||||
ACTION_KIND = {
|
||||
"Poor visibility reduced": "Use the data in this period with care: they are less precise.",
|
||||
"Heat shimmer": "Nothing to do: the small fast wiggles are not movement.",
|
||||
"Camera thermal movement": "Nothing to do: this is not a movement of the structure.",
|
||||
"Brief obstruction": "Ignore this single point on the curves.",
|
||||
"Sudden jump, back to normal": "Look at the images: the cause is not established.",
|
||||
"Lasting shift": "Check whether the structure moved: look at the images.",
|
||||
"Jump, not back yet": "Check whether the structure moved: look at the images.",
|
||||
"Large shift": "Check whether the structure moved: look at the images.",
|
||||
"Slow drift": "Keep watching it.",
|
||||
}
|
||||
|
||||
|
||||
def action(v):
|
||||
ep = v["episodes"][0]
|
||||
return ACTION_KIND.get(f"{ep['label']} {ep.get('level') or ''}".strip(), ACTION_KIND.get(ep["label"])) \
|
||||
or ACTION_FAMILY.get(v["category"], "")
|
||||
|
||||
|
||||
# --- the labels
|
||||
|
||||
def label_file(station, source):
|
||||
if source not in FOLDERS:
|
||||
raise Refusal("unknown label folder")
|
||||
folder(station)
|
||||
return LABELS / source / f"{station}.jsonl"
|
||||
|
||||
|
||||
def read(station, source):
|
||||
f = label_file(station, source)
|
||||
# split and not splitlines: the latter would also cut on the Unicode separators the text may contain
|
||||
return [json.loads(l) for l in f.read_text(encoding="utf-8").split("\n") if l.strip()] if f.exists() else []
|
||||
|
||||
|
||||
def assign_ids(station, labels):
|
||||
"""As labeling/ids.py does (D-131): an id for the labels without one, the next of the station, as first field."""
|
||||
n = max((int(e["id"].rsplit("-", 1)[1]) for e in labels if "id" in e), default=0)
|
||||
for e in labels:
|
||||
if "id" not in e:
|
||||
n += 1
|
||||
e["id"] = f"{station}-{n}"
|
||||
if next(iter(e)) != "id":
|
||||
rest = {k: v for k, v in e.items() if k != "id"}
|
||||
ident = e["id"]
|
||||
e.clear()
|
||||
e.update({"id": ident}, **rest)
|
||||
|
||||
|
||||
def _validate(body, start, end):
|
||||
description = (body.get("description") or "").strip()
|
||||
try:
|
||||
empty = datetime.strptime(start, FORMAT) >= datetime.strptime(end, FORMAT)
|
||||
except (TypeError, ValueError):
|
||||
raise Refusal("unreadable bounds (YYYY-MM-DDTHH:MMZ)") from None
|
||||
if not description or empty:
|
||||
raise Refusal("empty text or empty range")
|
||||
family = body.get("family") or ""
|
||||
if family and family not in FAMILIES:
|
||||
raise Refusal("unknown family")
|
||||
return description, family
|
||||
|
||||
|
||||
def add(body):
|
||||
station, source = body.get("station"), body.get("folder", "manual")
|
||||
f = label_file(station, source)
|
||||
description, family = _validate(body, body.get("start"), body.get("end"))
|
||||
label = {"start": body["start"], "end": body["end"]} | ({"family": family} if family else {}) | {"description": description}
|
||||
if source == "auto": # an automatic label carries a stable id, the next of its station (D-131)
|
||||
assign_ids(station, read(station, "auto") + [label])
|
||||
f.parent.mkdir(parents=True, exist_ok=True)
|
||||
before = f.read_bytes() if f.exists() else b"" # a line removed by hand may leave the file without a final newline
|
||||
with open(f, "ab") as out:
|
||||
out.write(b"\n" * bool(before and not before.endswith(b"\n")) + json.dumps(label, ensure_ascii=False).encode() + b"\n")
|
||||
return label
|
||||
|
||||
|
||||
def update(body):
|
||||
before = body.get("before") or {}
|
||||
start, end = body.get("start", before.get("start")), body.get("end", before.get("end"))
|
||||
description, family = _validate(body, start, end)
|
||||
label = dict.fromkeys(("id", "start", "end", "family")) | before | {"start": start, "end": end, "family": family,
|
||||
"description": description}
|
||||
if label["id"] is None:
|
||||
del label["id"]
|
||||
if not family:
|
||||
del label["family"]
|
||||
label.pop("review", None) # changed: no longer reviewed
|
||||
return replace(body.get("station"), body.get("folder", "manual"), before, label)
|
||||
|
||||
|
||||
def delete(body):
|
||||
replace(body.get("station"), body.get("folder", "manual"), body.get("before"), None)
|
||||
return body.get("before")
|
||||
|
||||
|
||||
def replace(station, source, before, label):
|
||||
"""Replaces the label before by label (None: removes it), found by its content, not by its rank: the file may have
|
||||
changed since the page was loaded. Writes next to it then replaces: an interruption does not damage the file."""
|
||||
f = label_file(station, source)
|
||||
lines = f.read_text(encoding="utf-8").split("\n") if f.exists() else []
|
||||
for i, line in enumerate(lines):
|
||||
if line.strip() and json.loads(line) == before:
|
||||
lines[i:i + 1] = [] if label is None else [json.dumps(label, ensure_ascii=False)]
|
||||
temp = f.with_name(f.name + ".tmp")
|
||||
temp.write_text("\n".join(lines), encoding="utf-8")
|
||||
os.replace(temp, f)
|
||||
return label
|
||||
raise Refusal("this label is no longer in the file: reload the page")
|
||||
Reference in new issue
Block a user