English: the app (server/, analysis.py, routes and JSON fields), instructions/analyst.md, docs; the table of shared renames

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-11-16 12:40:03 +01:00
1 parent 1d80a77cb1
commit a3e6ade845
24 files changed
+1959 -1835

No files matched your search

+317
View File
@@ -0,0 +1,317 @@
"""The Analysis tab of the Data page: the analyst of the agents repository (agents/, a plugin: --agent agents:analyst), launched
by `claude -p`, non-interactive, on a range chosen in the page, with the user's optional hints, its instructions
(instructions/analyst.md) and without the detectors (DETECTORS: no: its labels will be used to improve them). It writes in
labeling/auto/analyses/<STATION>_<start>_<end>/ its labels (labels.jsonl) and a diagnostic report (report.md, its figures
next to it); at the end, the server submits its labels to labeling/auto/<STATION>.jsonl, marked with their analysis (field
`analysis`). Then the user can write to it: the same session resumes (`claude -p --resume`), and its labels, if they
change, replace those it had submitted.
On a finished analysis, "Optimize" launches `/improve` in Optimizer (optimizer/, the skill that improves the detectors on the
range of the analysis, its labels as truth), then the tool-builder of the agents repository, which brings the analyst's
scripts into the reading tools (tools/); one optimization at a time, no analysis or message meanwhile; the summary of the
session (optimizer/sessions/<analysis>_<time>/summary.md) and that of the tools (<analysis>/tools.md) show under the report.
"""
import html
import json
import os
import subprocess
import threading
from datetime import date, datetime, timedelta, timezone
import markdown
import station
FOLDER = station.LABELS / "auto" / "analyses"
DETECTORS = station.ROOT / "detectors"
AGENTS = station.ROOT / "agents" # the plugin of the generic agents (~/agents)
OPTIMIZER = station.ROOT / "optimizer" # the Optimizer repository (~/Optimizer), where /improve runs
SESSIONS = OPTIMIZER / "sessions"
# the request of /improve (optimizer/README.md), paths relative to the root of Optimizer (its links data, labeling, detectors)
IMPROVE = """/improve DETECTORS: detectors/. STATION: data/{station}. FROM: {start}. TO: {end}. \
LABELS: labeling/auto/{station}.jsonl. REFERENCE: --labels labeling/auto --data data. FAMILIES: labeling/families.md. \
CONTEXT: labeling/auto/analyses/{name}/report.md. OUT: sessions/{name}_{time}/. Caller's context: Diagnostic-app, through \
Optimizer; read-only on data/ and labeling/; commit only in detectors/, on its current branch; do not commit OUT."""
REQUEST = """Station: data/{station}/. Range: {start} → {end} (UTC). Hints: {hints}. \
Output folder: {out}/. DETECTORS: no. INSTRUCTIONS: instructions/analyst.md."""
# name -> {"state": "running" | "done" | "error", "lines": [...], "submitted": n}; one analysis at a time per range
analyses, lock = {}, threading.Lock()
# name of the analysis -> {"state": "running" | "done" | "error", "lines": [...]}; one optimization at a time, all analyses
optimizations = {}
BUSY = "an optimization of the detectors is running"
ISOLATED = ["--setting-sources", "project,local", "--strict-mcp-config"] # no plugins, hooks or MCP servers of the workstation
def _env():
"""The environment of claude -p, without an API key: the agents run on the Claude subscription, never on a key."""
return {k: v for k, v in os.environ.items() if k not in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")}
def _optimizing():
return any(o["state"] == "running" for o in optimizations.values())
def launch(body):
st, start, end = body.get("station"), body.get("start"), body.get("end")
station._validate({"description": "-"}, start, end)
station.folder(st)
if not (AGENTS / ".claude-plugin" / "plugin.json").exists():
raise station.Refusal("no agents in agents/ (the link: README.md)")
name = f"{st}_{start[:16].replace(':', '')}_{end[:16].replace(':', '')}"
with lock:
if _optimizing():
raise station.Refusal(BUSY)
if analyses.get(name, {}).get("state") == "running":
raise station.Refusal("this range is already being analyzed")
out = FOLDER / name
if out.exists(): # a new analysis of the same range: the old one stays, next to it
name += datetime.now(timezone.utc).strftime("_%H%M%S")
out = FOLDER / name
hints = (body.get("hints") or "").strip()
request = {"station": st, "start": start, "end": end, "hints": hints}
analyses[name] = {"state": "running", "lines": [], "submitted": 0, "station": st, "request": request}
(out / "work").mkdir(parents=True)
(out / "request.json").write_text(json.dumps(request, ensure_ascii=False, indent=1), encoding="utf-8")
prompt = REQUEST.format(station=st, start=start, end=end, out=out.relative_to(station.ROOT), hints=hints or "none")
threading.Thread(target=work, args=(name, out, prompt), daemon=True).start()
return {"name": name}
def work(name, out, prompt, resume=None):
"""A turn of the analyst: the request, or a message of the user in the same session (resume: its id)."""
a = analyses[name]
try:
p = subprocess.Popen(["claude", "-p", prompt, "--plugin-dir", str(AGENTS), "--agent", "agents:analyst", *ISOLATED,
"--output-format", "stream-json", "--verbose",
"--allowedTools", "Read,Write,Bash", "--permission-mode", "acceptEdits", *station.links()]
+ (["--resume", resume] if resume else []),
cwd=station.ROOT, env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
for line in p.stdout:
log.write(line)
a["lines"] += digest(line)
if '"subtype":"init"' in line: # the session, for the next messages
(out / "work" / "session").write_text(json.loads(line)["session_id"])
if p.wait():
raise RuntimeError(f"claude ended with code {p.returncode}")
a["submitted"] = submit(a["station"], name, out)
a["state"] = "done"
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
a["lines"].append(f"Error: {e}")
a["state"] = "error"
def message(name, body):
"""A message of the user to the analyst, after its report: the same session resumes (claude -p --resume)."""
text, out = (body.get("text") or "").strip(), FOLDER / name
state(name) # the name checked, the state read again after a restart
a, session = analyses[name], out / "work" / "session"
if not text or not session.exists():
raise station.Refusal("empty message, or the analysis has no session to resume")
with lock:
if _optimizing():
raise station.Refusal(BUSY)
if a["state"] == "running":
raise station.Refusal("the analyst is still working")
a["state"] = "running"
line = json.dumps({"type": "page", "text": text}, ensure_ascii=False) + "\n"
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
log.write(line)
a["lines"] += digest(line)
prompt = (f"{text}\n\n(If you change your labels, rewrite {out.relative_to(station.ROOT)}/labels.jsonl in full, the "
"server will submit them again; if the diagnosis changes, update report.md.)")
threading.Thread(target=work, args=(name, out, prompt, session.read_text()), daemon=True).start()
return {}
def digest(line):
"""A line of the log -> what the page shows of it: the user's messages ("› "), the analyst's texts, its tools and their
command, one line each."""
try:
m = json.loads(line)
except json.JSONDecodeError:
return [line.strip()[:200]] if line.strip() else []
if m.get("type") == "page":
return [f"› {m['text']}"]
if m.get("type") != "assistant":
return []
out = []
for c in m["message"].get("content", []):
if c.get("type") == "text" and c["text"].strip():
out.append(c["text"].strip()[:300])
elif c.get("type") == "tool_use":
i = c.get("input", {})
out.append(f"{c['name']}: {i.get('command') or i.get('file_path') or ''}"[:200])
return out
def submit(st, name, out):
"""The labels of the analysis into labeling/auto/<STATION>.jsonl, with their id and the name of the analysis; if the
analyst changed them since the last submission, they replace those it had submitted -> their number."""
labels, copy = out / "labels.jsonl", out / "work" / "submitted.jsonl"
text = labels.read_text(encoding="utf-8") if labels.exists() else ""
new = []
for l in text.split("\n"):
if l.strip():
e = json.loads(l)
station._validate(e, e.get("start"), e.get("end"))
new.append({k: e[k] for k in ("start", "end", "family", "description") if k in e} | {"analysis": name})
if copy.exists() and copy.read_text(encoding="utf-8") == text:
return len(new)
# ponytail: removed then put back, the labels of the analysis lose the edits made in the page; keeping them would
# require matching them one by one
with lock:
f = station.label_file(st, "auto") # the other lines stay as they are
kept = [l for l in (f.read_text(encoding="utf-8").split("\n") if f.exists() else [])
if l.strip() and json.loads(l).get("analysis") != name]
station.assign_ids(st, [json.loads(l) for l in kept] + new)
temp = f.with_name(f.name + ".tmp")
temp.write_text("".join(l + "\n" for l in kept + [json.dumps(e, ensure_ascii=False) for e in new]), encoding="utf-8")
os.replace(temp, f)
copy.write_text(text, encoding="utf-8")
return len(new)
def state(name):
out = FOLDER / name
if "/" in name or name.startswith(".") or not out.is_dir():
raise station.Refusal("unknown analysis")
if name not in analyses: # after a restart of the server, only the folder is left: the thread is read again in the log
log = out / "work" / "log.jsonl"
lines = [r for l in open(log, encoding="utf-8") for r in digest(l)] if log.exists() else []
request = json.loads((out / "request.json").read_text(encoding="utf-8"))
analyses[name] = {"state": "done" if (out / "report.md").exists() else "interrupted", "lines": lines,
"submitted": None, "station": request["station"], "request": request}
a, r = analyses[name], out / "report.md"
return {"name": name, "state": a["state"], "lines": a["lines"], "submitted": a["submitted"],
"report": int(r.stat().st_mtime) if r.exists() else None,
"dialogue": (out / "work" / "session").exists(), "request": a["request"], "optimization": optimization(name)}
def list_analyses():
"""The history of the analyses (the sidebar), the most recent first: those of the server with their state, the others
read on disk (done with a report, otherwise interrupted: an analysis stops with the server)."""
out = []
for d in sorted(FOLDER.glob("*/request.json"), key=lambda f: f.parent.stat().st_mtime, reverse=True):
name = d.parent.name
a = analyses.get(name) or {"state": "done" if (d.parent / "report.md").exists() else "interrupted",
"request": json.loads(d.read_text(encoding="utf-8"))}
out.append({"name": name, "state": a["state"]} | a["request"])
return out
def file(name, path):
"""A file of the folder of the analysis (the report, its figures), never outside."""
f = (FOLDER / name / path).resolve()
if not f.is_file() or not f.is_relative_to(FOLDER.resolve()):
raise station.Refusal("unknown file")
return f
STYLE = """body{font:14px/1.55 Roboto,sans-serif;margin:0;padding:4px 2px;color:#212529;background:transparent}
h1{font-size:20px}h2{font-size:16px;margin-top:22px}img{max-width:100%;border:1px solid #dee2e6;border-radius:4px}
code{font-size:12px;background:#f1f3f5;padding:1px 4px;border-radius:3px}pre{overflow-x:auto}
table{border-collapse:collapse}td,th{border:1px solid #dee2e6;padding:3px 8px;text-align:left}
body.dark{color:#c9c9c9}body.dark code{background:#2e2e2e}body.dark td,body.dark th,body.dark img{border-color:#424242}"""
def _page(text, title, dark):
"""A markdown in HTML, in the style of the page; its relative images resolve in the folder of the address."""
body = markdown.markdown(text, extensions=["tables", "fenced_code"])
return (f"<!doctype html><meta charset=utf-8><title>{html.escape(title)}</title><style>{STYLE}</style>"
f"<body class='{'dark' if dark else ''}'>{body}</body>")
def report(name, dark):
"""report.md in HTML, served in the folder of the analysis: its relative images are found there."""
return _page(file(name, "report.md").read_text(encoding="utf-8"), name, dark)
# --- "Optimize": /improve in Optimizer on the analysis, then the tool-builder, by claude -p
def _summary_file(name):
"""The summary of the last session of this analysis, or None. The time pattern rules out the sessions of another
analysis of the same range (<name>_<HHMMSS>_<time>)."""
summaries = sorted(SESSIONS.glob(f"{name}_????????T????Z/summary.md")) # the order of the name is that of time
return summaries[-1] if summaries else None
def optimization(name):
"""-> None (never optimized) or {state, lines, summary: mtime of the last summary}. After a restart, the state is read
again in the stream: its last line {"type": "end"} says done or error; without it, the optimization was cut
(interrupted)."""
stream, s = FOLDER / name / "work" / "optimization.jsonl", _summary_file(name)
o = optimizations.get(name)
if o is None:
if not stream.exists():
return None
lines = stream.read_text(encoding="utf-8").splitlines()
end = json.loads(lines[-1]) if lines and lines[-1].startswith('{"type": "end"') else None
o = {"state": "interrupted" if end is None else end["state"], "lines": [r for l in lines for r in digest(l)]}
optimizations[name] = o
return {"state": o["state"], "lines": o["lines"], "summary": int(s.stat().st_mtime) if s else None}
def optimize(name):
"""The "Optimize" button on a finished analysis: the session in a thread; refused if an optimization or an analysis runs."""
if state(name)["state"] != "done":
raise station.Refusal("only a finished analysis can be optimized")
if not ((OPTIMIZER / ".claude" / "skills" / "improve" / "SKILL.md").exists()
and all((OPTIMIZER / link).exists() for link in ("data", "labeling", "detectors"))
and (AGENTS / ".claude-plugin" / "plugin.json").exists()):
raise station.Refusal("no Optimizer in optimizer/ or no agents in agents/ (the links: README.md)")
with lock:
if _optimizing():
raise station.Refusal(BUSY)
if any(a["state"] == "running" for a in analyses.values()):
raise station.Refusal("an analysis is running: the optimization waits for its end")
optimizations[name] = {"state": "running", "lines": []}
threading.Thread(target=optimize_thread, args=(name,), daemon=True).start()
return {}
def optimize_thread(name):
"""The two turns of the optimization, one after the other: the detectors (/improve, in Optimizer), then the reading
tools (the tool-builder, even if the detectors did not change)."""
o, out = optimizations[name], FOLDER / name
rel = os.path.relpath(out, station.ROOT)
(out / "work").mkdir(exist_ok=True)
with open(out / "work" / "optimization.jsonl", "a", encoding="utf-8") as stream:
try:
d = json.loads((out / "request.json").read_text(encoding="utf-8"))
request = IMPROVE.format( # TO excluded: the day after the end of the range, as in the page
station=d["station"], start=d["start"][:10], end=date.fromisoformat(d["end"][:10]) + timedelta(days=1),
name=name, time=f"{datetime.now(timezone.utc):%Y%m%dT%H%MZ}")
# ponytail: Optimizer reads data/ and labeling/ through its own links; AIMSIGHT_DATA / AIMSIGHT_LABELS do not follow
turns = [(OPTIMIZER.resolve(), [request, "--plugin-dir", str(AGENTS.resolve()),
"--add-dir", str(DETECTORS.resolve()), # the target of the link, where the session writes
"--allowedTools", "Read,Write,Edit,Bash,Agent"]),
(station.ROOT, [f"Analysis: {rel}/. Tools: tools/. Write {rel}/tools.md.", "--plugin-dir", str(AGENTS),
"--agent", "agents:tool-builder", "--allowedTools", "Read,Write,Edit,Bash"])]
for cwd, (prompt, *options) in turns:
p = subprocess.Popen(["claude", "-p", prompt, *ISOLATED, "--output-format", "stream-json", "--verbose",
"--permission-mode", "acceptEdits", *station.links(), *options],
cwd=cwd, env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
for line in p.stdout:
stream.write(line)
o["lines"] += digest(line)
if p.wait():
raise RuntimeError(f"claude ended with code {p.returncode}")
o["state"] = "done"
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
o["lines"].append(f"Error: {e}")
o["state"] = "error"
finally:
stream.write(json.dumps({"type": "end", "state": o["state"]}) + "\n")
station.detectors.cache_clear() # the detectors may have changed: the Data page computes them again
def summary(name, dark):
"""The summary of the detectors' session, then that of the tools (<analysis>/tools.md), in one page."""
s, tools = _summary_file(name), FOLDER / name / "tools.md"
if s is None and not tools.exists():
raise station.Refusal("no summary for this analysis")
text = "\n\n".join(f.read_text(encoding="utf-8") for f in (s, tools) if f and f.exists())
return _page(text, name, dark)
+266
View File
@@ -0,0 +1,266 @@
"""Checks of the web app, without calling claude: .venv/bin/python app/server/check.py [name]"""
import json
import os
import shutil
import socket
import sys
import tempfile
import threading
import time
import urllib.request
from pathlib import Path
from urllib.error import HTTPError
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import uvicorn # noqa: E402
import server # noqa: E402
import station # noqa: E402
def temp_folder():
return Path(tempfile.mkdtemp())
def check_server():
# the real server on a free port: the guard (Host, declared JSON), a read
with socket.socket() as so:
so.bind(("127.0.0.1", 0))
port = so.getsockname()[1]
srv = uvicorn.Server(uvicorn.Config(server.app, host="127.0.0.1", port=port, log_level="error"))
threading.Thread(target=srv.run, daemon=True).start()
while not srv.started:
time.sleep(0.05)
base = f"http://127.0.0.1:{port}"
def call(path, body=None, headers=None, method=None):
r = urllib.request.Request(base + path, json.dumps(body).encode() if body is not None else None,
{"Content-Type": "application/json", **(headers or {})}, method=method)
try:
with urllib.request.urlopen(r) as rep:
return rep.status
except HTTPError as e:
return e.code
try:
assert call("/api/stations", headers={"Host": "evil.example:80"}) == 403
assert call("/api/labels", {"station": "x"}, {"Content-Type": "text/plain"}) == 403
assert call("/api/stations") == 200
assert call("/api/analyses") == 200
assert call("/api/analyses/UNKNOWN/optimize", {}) == 400
finally:
srv.should_exit = True
def refuses(f, *a):
try:
f(*a)
except station.Refusal as e:
return str(e)
raise AssertionError(f"{f.__name__}{a} did not refuse")
def check_station():
# the data of the Data page, on the real data; the labels in a temporary folder
assert "NIEDERHORN" in station.stations()
im = station.images("NIEDERHORN", "2026-09-25", "2026-09-26")
assert im["images"] and im["positions"] and set(im["corr"]) == set(im["positions"]), list(im)
assert all(len(c) == len(im["images"]) for c in im["corr"].values())
assert station.image("NIEDERHORN", im["images"][0]["name"]).is_file()
refuses(station.image, "NIEDERHORN", "../../../.env")
refuses(station.images, "../API", "", "")
assert station.images("LENZSPITZE", "2026-10-26", "2026-10-27")["images"]
c = station.curves("LENZSPITZE", "2026-10-26", "2026-10-27")
titles = [p["title"] for p in c["plots"]]
assert "Relative Corrected Vertical Shift" in titles and any(t.startswith("System: ") for t in titles), titles
ev = station.detectors("LENZSPITZE", "2026-10-26", "2026-10-27") # the detectors of the journal, ~10 s
assert ev and {"start", "end", "family", "description", "sensors"} <= set(ev[0]), ev[:1]
assert all(e["message"] and e["action"] and set(e["targets"] + e["references"]) == set(e["sensors"]) for e in ev), ev[:1]
assert station.message("A moved; nothing to do. The jump of 2 targets in this period has the same cause.") \
== "A moved. The jump of 2 targets in this period has the same cause."
assert all(e["end"] >= "2026-10-26T00:00Z" and e["start"] < "2026-10-27T00:00Z" for e in ev), [(e["start"], e["end"]) for e in ev]
assert not any("SIM_CCID" in t or t.startswith("System: ip") for t in titles)
assert all(len(c["T"][tr["t"]]) == len(tr["y"]) for p in c["plots"] for tr in p["traces"])
before, station.LABELS = station.LABELS, temp_folder()
try:
e = station.add({"station": "NIEDERHORN", "folder": "auto", "start": "2026-09-25T01:00Z",
"end": "2026-09-25T02:00Z", "family": "Visibility", "description": "fog"})
assert e["id"] == "NIEDERHORN-1" and next(iter(e)) == "id", e
e2 = station.update({"station": "NIEDERHORN", "folder": "auto", "before": e,
"start": e["start"], "end": e["end"], "family": "", "description": "rain"})
assert e2 == {"id": "NIEDERHORN-1", "start": e["start"], "end": e["end"], "description": "rain"}, e2
assert station.read("NIEDERHORN", "auto") == [e2]
assert "no longer in the file" in refuses(station.delete, {"station": "NIEDERHORN", "folder": "auto", "before": e})
station.delete({"station": "NIEDERHORN", "folder": "auto", "before": e2})
assert station.read("NIEDERHORN", "auto") == []
refuses(station.add, {"station": "NIEDERHORN", "folder": "../x", "start": "a", "end": "b", "description": "x"})
refuses(station.add, {"station": "NIEDERHORN", "start": "2026-09-25T02:00Z", "end": "2026-09-25T01:00Z", "description": "x"})
finally:
shutil.rmtree(station.LABELS)
station.LABELS = before
def check_optimization():
# the Optimize button (analysis.optimize), without claude: a fake process that ends only on a signal, a finished
# analysis copied into a temporary folder; the lock (one optimization at a time, no analysis or message meanwhile), the
# state read again after a restart
import analysis
end = threading.Event()
launched = []
class FakeProcess:
returncode = 0
stdout = iter(lambda: end.wait() and "", "") # nothing, until the signal
def __init__(self, args, *a, **k):
launched.append((args, k))
def wait(self):
return 0
before = analysis.FOLDER, analysis.subprocess.Popen, analysis.OPTIMIZER, analysis.SESSIONS, analysis.AGENTS
key = os.environ.get("ANTHROPIC_API_KEY")
os.environ["ANTHROPIC_API_KEY"] = "test-key"
root = temp_folder()
analysis.FOLDER, analysis.OPTIMIZER, analysis.AGENTS = root / "analyses", root / "optimizer", root / "agents"
analysis.SESSIONS = analysis.OPTIMIZER / "sessions"
analysis.subprocess.Popen = FakeProcess
try:
for name in ("NORDEND_a", "NORDEND_b"):
(analysis.FOLDER / name / "work").mkdir(parents=True)
(analysis.FOLDER / name / "request.json").write_text(json.dumps(
{"station": "NORDEND", "start": "2026-11-04T21:12Z", "end": "2026-11-07T14:12Z", "hints": ""}))
(analysis.FOLDER / name / "report.md").write_text("# report")
assert analysis.state("NORDEND_a")["optimization"] is None
# without Optimizer, then without agents: a clear refusal, not an error trace
assert "optimizer/" in refuses(analysis.optimize, "NORDEND_a")
(analysis.OPTIMIZER / ".claude" / "skills" / "improve").mkdir(parents=True)
(analysis.OPTIMIZER / ".claude" / "skills" / "improve" / "SKILL.md").write_text("improve")
assert "agents/" in refuses(analysis.optimize, "NORDEND_a")
(analysis.AGENTS / ".claude-plugin").mkdir(parents=True)
(analysis.AGENTS / ".claude-plugin" / "plugin.json").write_text("{}")
assert "optimizer/" in refuses(analysis.optimize, "NORDEND_a") # without the data, labeling, detectors links of Optimizer
for link in ("data", "labeling", "detectors"):
(analysis.OPTIMIZER / link).mkdir()
assert analysis.optimize("NORDEND_a") == {}
assert analysis.state("NORDEND_a")["optimization"]["state"] == "running"
refuses(analysis.optimize, "NORDEND_a")
refuses(analysis.optimize, "NORDEND_b")
refuses(analysis.launch, {"station": "NORDEND", "start": "2026-11-04T00:00Z", "end": "2026-11-05T00:00Z"})
refuses(analysis.message, "NORDEND_b", {"text": "continue"})
end.set()
while analysis.state("NORDEND_a")["optimization"]["state"] == "running":
time.sleep(0.05)
assert analysis.state("NORDEND_a")["optimization"]["state"] == "done"
args, k = launched[0] # /improve in Optimizer, on the range of the analysis, TO excluded
request = args[2]
assert request.startswith("/improve ") and "STATION: data/NORDEND. FROM: 2026-11-04. TO: 2026-11-08." in request \
and "OUT: sessions/NORDEND_a_" in request and "FAMILIES: labeling/families.md." in request \
and "CONTEXT: labeling/auto/analyses/NORDEND_a/report.md." in request, request
assert k["cwd"] == analysis.OPTIMIZER.resolve() and "--plugin-dir" in args, (args, k)
assert all("ANTHROPIC_API_KEY" not in kw["env"] for _, kw in launched), "an API key reaches claude"
# isolated from the settings of the workstation (the user's plugins, hooks, MCP servers)
assert all(a[a.index("--setting-sources") + 1] == "project,local" and "--strict-mcp-config" in a for a, _ in launched)
args, k = launched[1] # then the tool-builder of the agents repository, on the tools repository
assert args[args.index("--agent") + 1] == "agents:tool-builder" and "Tools: tools/" in args[2] \
and args[2].endswith("/tools.md."), args
analysis.analyses["NORDEND_b"]["state"] = "running" # a running analysis blocks the optimization
refuses(analysis.optimize, "NORDEND_a")
# after a restart: the stream ends with {"type": "end"} -> done; without that line -> interrupted; the summary of
# the detectors and that of the tools in one page
stream = analysis.FOLDER / "NORDEND_a" / "work" / "optimization.jsonl"
assert stream.read_text().endswith('{"type": "end", "state": "done"}\n'), stream.read_text()
analysis.analyses.clear()
analysis.optimizations.clear()
assert analysis.state("NORDEND_a")["optimization"]["state"] == "done"
stream.write_text('{"type": "assistant", "message": {"content": []}}\n')
analysis.optimizations.clear()
assert analysis.state("NORDEND_a")["optimization"]["state"] == "interrupted"
(analysis.SESSIONS / "NORDEND_a_20261113T1000Z").mkdir(parents=True)
(analysis.SESSIONS / "NORDEND_a_20261113T1000Z" / "summary.md").write_text("# improve\n\n**Result**: none")
# a new analysis of the same range (NORDEND_a_143000): its session is not the summary of NORDEND_a
(analysis.SESSIONS / "NORDEND_a_143000_20261113T1100Z").mkdir(parents=True)
(analysis.SESSIONS / "NORDEND_a_143000_20261113T1100Z" / "summary.md").write_text("# other analysis")
assert analysis._summary_file("NORDEND_a").parent.name == "NORDEND_a_20261113T1000Z", analysis._summary_file("NORDEND_a")
(analysis.FOLDER / "NORDEND_a" / "tools.md").write_text("# Tools\n\n**Tools changed**: none")
assert analysis.state("NORDEND_a")["optimization"]["summary"]
page = analysis.summary("NORDEND_a", False)
assert "<strong>Result</strong>" in page and "<strong>Tools changed</strong>" in page, page
assert analysis.state("NORDEND_b")["optimization"] is None
finally:
end.set()
os.environ.pop("ANTHROPIC_API_KEY")
if key is not None:
os.environ["ANTHROPIC_API_KEY"] = key
shutil.rmtree(root)
analysis.FOLDER, analysis.subprocess.Popen, analysis.OPTIMIZER, analysis.SESSIONS, analysis.AGENTS = before
analysis.analyses.clear()
analysis.optimizations.clear()
def check_analysis():
# "Analyze the range" then a message (analysis.launch, analysis.message), without claude: the analyst of the agents
# repository, its parameters, and the resumption of its session with the same agent
import analysis
launched = []
class FakeProcess:
returncode = 0
def __init__(self, args, *a, **k):
launched.append((args, k))
self.stdout = iter([])
def wait(self):
return 0
(station.ROOT / "out").mkdir(exist_ok=True) # under the root: the request cites the output folder relative to it
before = analysis.FOLDER, analysis.subprocess.Popen, analysis.submit
analysis.FOLDER = Path(tempfile.mkdtemp(dir=station.ROOT / "out"))
analysis.subprocess.Popen = FakeProcess
analysis.submit = lambda *a: 0 # otherwise the test rewrites the real labeling/auto/NORDEND.jsonl
key = os.environ.get("ANTHROPIC_API_KEY")
os.environ["ANTHROPIC_API_KEY"] = "test-key" # the agent runs on the subscription, never on an API key
try:
name = analysis.launch({"station": "NORDEND", "start": "2026-11-05T06:00Z", "end": "2026-11-05T12:00Z"})["name"]
while analysis.analyses[name]["state"] == "running":
time.sleep(0.05)
assert analysis.analyses[name]["state"] == "done", analysis.analyses[name]
assert json.loads((analysis.FOLDER / name / "request.json").read_text()) == \
{"station": "NORDEND", "start": "2026-11-05T06:00Z", "end": "2026-11-05T12:00Z", "hints": ""}
args, k = launched[0]
assert args[args.index("--agent") + 1] == "agents:analyst" and "--plugin-dir" in args, args
assert "ANTHROPIC_API_KEY" not in k["env"] and "PATH" in k["env"], sorted(k.get("env") or {})
assert args[2].startswith("Station: data/NORDEND/. Range: 2026-11-05T06:00Z → 2026-11-05T12:00Z (UTC). Hints: none. ") \
and "DETECTORS: no" in args[2] and "INSTRUCTIONS: instructions/analyst.md" in args[2], args[2]
assert args[args.index("--setting-sources") + 1] == "project,local" and "--strict-mcp-config" in args, args
(analysis.FOLDER / name / "work" / "session").write_text("s-1")
analysis.message(name, {"text": "continue"})
while analysis.analyses[name]["state"] == "running":
time.sleep(0.05)
args, k = launched[1]
assert "ANTHROPIC_API_KEY" not in k["env"], sorted(k.get("env") or {})
assert args[args.index("--resume") + 1] == "s-1" and args[args.index("--agent") + 1] == "agents:analyst" \
and "--plugin-dir" in args, args
agents, analysis.AGENTS = analysis.AGENTS, analysis.FOLDER / "no-agents" # the agents/ link is missing: clear refusal
try:
assert "agents/" in refuses(analysis.launch, {"station": "NORDEND", "start": "2026-11-06T06:00Z", "end": "2026-11-06T12:00Z"})
finally:
analysis.AGENTS = agents
finally:
os.environ.pop("ANTHROPIC_API_KEY")
if key is not None:
os.environ["ANTHROPIC_API_KEY"] = key
shutil.rmtree(analysis.FOLDER)
analysis.FOLDER, analysis.subprocess.Popen, analysis.submit = before
analysis.analyses.clear()
CHECKS = [check_server, check_station, check_optimization, check_analysis]
if __name__ == "__main__":
name = sys.argv[1] if len(sys.argv) > 1 else None
for f in CHECKS:
if name in (None, f.__name__):
f()
print("ok", f.__name__)
+165
View File
@@ -0,0 +1,165 @@
"""The web app: .venv/bin/python app/server/server.py → http://127.0.0.1:8770
The FastAPI server serves the built React page (app/web/dist, `npm run build` in app/web) and the API under /api.
The Data page reads data/ and the labels of labeling/ (station.py); its Analysis tab launches the analyst, and the
optimization of the detectors (analysis.py).
Settings: AIMSIGHT_DATA (data), AIMSIGHT_LABELS (labels), AIMSIGHT_PORT (port, 8770).
"""
import json
import os
import platform
import subprocess
import sys
import threading
import webbrowser
from pathlib import Path
import uvicorn
from fastapi import FastAPI, Query, Request
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
import analysis
import station
from fastapi.staticfiles import StaticFiles
APP = Path(__file__).resolve().parents[1]
PORT = int(os.environ.get("AIMSIGHT_PORT", 8770))
HOSTS = {"127.0.0.1", "localhost"} # against DNS rebinding: another site does not read the data
PAGE = APP / "web" / "dist"
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
def error(code: int, text: str) -> JSONResponse:
return JSONResponse({"error": text}, code)
@app.middleware("http")
async def guard(request: Request, call_next):
"""Only the page itself talks to the server: right Host; in POST, a declared JSON (a page of another site cannot send
it without a preflight request, to which the server does not answer)."""
host = request.headers.get("host", "").rsplit(":", 1)[0]
if host not in HOSTS or (request.method in ("POST", "PUT", "DELETE")
and not request.headers.get("content-type", "").startswith("application/json")):
return error(403, "request refused")
return await call_next(request)
# --- the Data page (station.py)
@app.exception_handler(station.Refusal)
def refusal(_, e):
return error(400, str(e))
@app.get("/api/stations")
def stations():
return station.stations()
@app.get("/api/images")
def images(station_: str = Query(alias="station"), start: str = "", end: str = ""):
return station.images(station_, start, end)
@app.get("/api/image/{station_}/{name}")
def image(station_: str, name: str):
return FileResponse(station.image(station_, name), headers={"Cache-Control": "max-age=86400"})
@app.get("/api/curves")
def curves(station_: str = Query(alias="station"), start: str = "", end: str = ""):
return station.curves(station_, start, end)
@app.get("/api/detectors")
def detectors(station_: str = Query(alias="station"), start: str = "", end: str = "", targets: str = ""):
return station.detectors(station_, start, end, targets)
@app.get("/api/labels")
def labels(station_: str = Query(alias="station"), folder: str = "manual"):
try:
return station.read(station_, folder)
except json.JSONDecodeError as e: # a line damaged by hand: say so, without touching the file
return error(500, f"labeling/{folder}/{station_}.jsonl unreadable: {e}")
@app.post("/api/labels")
def add_label(body: dict):
return station.add(body)
@app.put("/api/labels")
def update_label(body: dict):
return station.update(body)
@app.delete("/api/labels")
def delete_label(body: dict):
return station.delete(body)
# --- the Analysis tab of the Data page (analysis.py): the analyst launched by claude -p on a range
@app.post("/api/analyses")
def launch_analysis(body: dict):
return analysis.launch(body)
@app.get("/api/analyses")
def analyses():
return analysis.list_analyses()
@app.get("/api/analyses/{name}")
def analysis_state(name: str):
return analysis.state(name)
@app.post("/api/analyses/{name}/message")
def analysis_message(name: str, body: dict):
return analysis.message(name, body)
@app.get("/api/analyses/{name}/report.html")
def analysis_report(name: str, dark: bool = False):
return HTMLResponse(analysis.report(name, dark))
@app.post("/api/analyses/{name}/optimize")
def optimize_analysis(name: str):
return analysis.optimize(name)
@app.get("/api/analyses/{name}/summary.html")
def analysis_summary(name: str, dark: bool = False):
return HTMLResponse(analysis.summary(name, dark))
@app.get("/api/analyses/{name}/{path:path}")
def analysis_file(name: str, path: str):
return FileResponse(analysis.file(name, path))
if PAGE.is_dir():
app.mount("/", StaticFiles(directory=PAGE, html=True), name="page")
else:
@app.get("/")
def no_page():
return error(503, "the page is not built: cd app/web && npm install && npm run build")
def open_browser(url: str) -> None:
if "microsoft" in platform.uname().release.lower(): # WSL: the Windows browser
subprocess.Popen(["explorer.exe", url], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
else:
webbrowser.open(url)
if __name__ == "__main__":
url = f"http://127.0.0.1:{PORT}"
print(f"AIMsight diagnostic: {url}")
if "--no-browser" not in sys.argv:
threading.Timer(1, open_browser, [url]).start()
uvicorn.run(app, host="127.0.0.1", port=PORT, log_level="warning", timeout_graceful_shutdown=1)
+338
View File
@@ -0,0 +1,338 @@
"""The data of the Data page: a station's images, its curves, its labels.
Data are read as the agents read them, tools/scripts/data.py (the tools repository, through the tools/ link); the detectors'
events are those of detectors/scripts/categories.py (events). Read-only on data/; writes only in labeling/{auto,manual}/.
"""
import bisect
import functools
import csv
import glob
import json
import os
import re
import statistics
import sys
from datetime import datetime, timedelta
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(ROOT / "tools" / "scripts"), str(ROOT / "detectors" / "scripts")]
import data # noqa: E402
from categories import events # noqa: E402
# data/ and labeling/ are links to ~/aimsight-data/data and ~/labeling: no resolve(), the agents see data/ and labeling/
DATA = Path(os.environ.get("AIMSIGHT_DATA", ROOT / "data")).absolute()
LABELS = Path(os.environ.get("AIMSIGHT_LABELS", ROOT / "labeling")).absolute()
TOOLS = ROOT / "tools" # the reading harness (~/tools), read by the analyst and written by the tool-builder
def links():
"""--add-dir for claude -p: the real target of the data/, labeling/ and tools/ links, outside the repository."""
return [a for d in (DATA, LABELS, TOOLS) for a in ("--add-dir", str(d.resolve()))]
FOLDERS = ("auto", "manual") # those of the agents and of the page (D-105), those of the expert (D-102)
FAMILIES = ("Visibility", "Obstruction", "Thermal displacement", "Displacement", "Target issue", "No data", "None") # D-109
FORMAT = "%Y-%m-%dT%H:%MZ"
MARGIN_PX = 150 # Support-Agent detecteurs.MARGE_PX: x, y are positions in the image only far from the corner
COLORS = ["#2a78d6", "#eb6834", "#1baf7a", "#8a5cd6", "#d6a12a", "#d63b8a", "#3bb3d6", "#6b6b66"] # one per sensor
POSE = (("pan", "#1a1a19"), ("tilt", "#8a8a85"), ("rotation", "#b5651d"))
# The Advanced menu of Explore, in its order (Support-Agent labeling.PLOTS, QUALITE)
PLOTS = (("error_value", "Error value"), ("x", "X [px]"), ("y", "Y [px]"),
("relative_x", "Relative Horizontal Shift"), ("relative_y", "Relative Vertical Shift"),
("corr_x", "Relative Corrected Horizontal Shift"), ("corr_y", "Relative Corrected Vertical Shift"),
("reproj_err", "References (reproj. error)"))
QUALITY = (("peak", "Quality: Peak"), ("flux", "Quality: Flux"), ("sharpness", "Quality: Sharpness"),
("fwhm", "Quality: FWHM [px]"), ("dominance", "Quality: Dominance"), ("snr", "Quality: SNR"))
IDENTIFIER = re.compile(r"(SIM_CCID|ip(_.*)?)$") # never read (CLAUDE.md, "data/")
METRIC = re.compile(r"^(.+?)_\d{4}_\d{2}_\d{2}_")
class Refusal(ValueError):
"""A request to refuse (400), with its reason."""
def stations():
return sorted(d.name for d in DATA.iterdir() if d.is_dir())
def folder(station):
if station not in stations(): # the name becomes a path: a folder of data/, nothing else
raise Refusal(f"unknown station: {station}")
return DATA / station
def bounds(start, end):
return data.parse_time(start or "0001-01-01"), data.parse_time(end or "9999-01-01")
# --- the positions of the sensors in the image
def positions(measurements, a, b):
"""The median position of each sensor in the image over [a, b) ({} if x, y are not positions there)."""
homes = {s: (statistics.median(xs), statistics.median(ys)) for s, ms in measurements.items()
if (xs := [m["x"] for m in ms if a <= m["t"] < b and m["x"] is not None])
and (ys := [m["y"] for m in ms if a <= m["t"] < b and m["y"] is not None])}
return homes if any(abs(x) > MARGIN_PX or abs(y) > MARGIN_PX for x, y in homes.values()) else {}
def references(measurements, a, b):
"""Sensors of the compensation over the range: those with a reproj_err (references only)."""
return {s for s, ms in measurements.items() if any(m["reproj_err"] is not None for m in ms if a <= m["t"] < b)}
# --- the images of the Data page
def images(station, start, end):
"""The images of [start, end), the median positions of the sensors, the references, and for each circled sensor its
corr_x, corr_y at the time of each image: the nearest valid measurement, within 30 min, otherwise 0."""
d, (a, b) = folder(station), bounds(start, end)
views = [(t, im) for im in sorted(glob.glob(str(d / "images" / "*"))) if (t := data.image_time(im)) and a <= t < b]
if not views:
raise Refusal(f"no image of {station} in this range")
measurements = data.load(d)
homes, refs = positions(measurements, a, b), references(measurements, a, b)
# ponytail: a single measurement at each bound, noisy; a median over a window if the arrows shake too much
margin, corr = timedelta(minutes=30), {}
for s in homes:
series = [(m["t"], m["corr_x"], m["corr_y"]) for m in measurements[s] if m["corr_x"] is not None and m["corr_y"] is not None]
times, corr[s] = [m[0] for m in series], []
for t, _ in views:
k = bisect.bisect_left(times, t)
k = min((n for n in (k - 1, k) if 0 <= n < len(series)), key=lambda n: abs(times[n] - t), default=None)
corr[s].append([round(series[k][1], 4), round(series[k][2], 4)] if k is not None and abs(times[k] - t) <= margin else 0)
return {"images": [{"t": data.iso(t).replace("T", " ").rstrip("Z"), "name": os.path.basename(im)} for t, im in views],
"positions": {s: [round(x, 1), round(y, 1)] for s, (x, y) in sorted(homes.items())},
"references": sorted(refs), "corr": corr}
def image(station, name):
f = (folder(station) / "images" / name).resolve()
if f.parent != (DATA / station / "images").resolve() or not f.is_file():
raise Refusal("unknown image")
return f
# --- the curves of the Data page
def metrics(d, a, b):
"""The numeric system metrics of metrics/ over [a, b) -> {name: {t: value or None}} (an empty value: None, the gap
shows), without the text metrics, the identifiers, or those with no value at all."""
station, series, text = d.name + "_", {}, set()
for p in sorted(glob.glob(str(d / "metrics" / "*.csv"))):
if not (m := METRIC.match(os.path.basename(p))) or IDENTIFIER.match(name := m.group(1).removeprefix(station)):
continue
with open(p, newline="") as f:
for r in csv.DictReader(f):
t, v = data.parse_time(r["time_utc"]), (r.get("metric") or "").strip("[]'\" ")
if not a <= t < b:
continue
try:
series.setdefault(name, {})[t] = float(v) if v else None
except ValueError:
text.add(name) # link, mode…: no curve
return {n: dict(sorted(s.items())) for n, s in series.items()
if n not in text and any(v is not None for v in s.values())}
def span(measurements, start, end):
"""The range read: from start to end (excluded), otherwise from the first measurement to the last, plus one step."""
every = sorted({m["t"] for ms in measurements.values() for m in ms})
a = data.parse_time(start) if start else every[0]
b = data.parse_time(end) if end else every[-1] + (every[-1] - every[-2] if len(every) > 1 else timedelta(hours=1))
return a, b
def curves(station, start, end):
"""-> {A, B, T, plots}: T, the times of each series, once (key: the sensor, "" for the camera, "system:" and its
name for a metric); plot: title and traces (name, t, y, line, mode), without empty plots or sensors."""
d = folder(station)
measurements = data.load(d)
if not measurements:
raise Refusal(f"no \"advanced\" export for {station}")
a, b = span(measurements, start, end)
poses = [p for p in data.load_camera(d) if a <= p["t"] < b]
hour = lambda t: t.strftime("%Y-%m-%d %H:%M") # noqa: E731 (UTC, without time zone: as Plotly shows it)
values = lambda s, c: [None if m[c] is None else round(m[c], 5) for m in s] # noqa: E731
inside = {s: [m for m in measurements[s] if a <= m["t"] < b] for s in sorted(measurements)}
refs, out = references(measurements, a, b), []
def per_sensor(columns):
for col, title in columns:
traces = [{"name": s + " (ref.)" * (s in refs), "t": s, "y": y,
"line": {"width": 1.2, "color": COLORS[i % len(COLORS)], "dash": "dot" if s in refs else "solid"}}
for i, (s, y) in enumerate((s, values(ms, col)) for s, ms in inside.items()) if any(v is not None for v in y)]
if traces:
out.append({"title": title, "traces": traces})
per_sensor(PLOTS)
if poses:
out.append({"title": "Camera movement (deg)", "traces": [{"name": c, "t": "", "y": values(poses, c),
"line": {"width": 1.2, "color": col}} for c, col in POSE]})
per_sensor(QUALITY)
T = {"": [hour(p["t"]) for p in poses], **{s: [hour(m["t"]) for m in ms] for s, ms in inside.items()}}
system = metrics(d, a, b)
for name in sorted(system, key=str.lower):
T["system:" + name] = [hour(t) for t in system[name]]
out.append({"title": "System: " + name, "traces": [{"name": name, "t": "system:" + name, "mode": "lines+markers",
"y": [None if v is None else round(v, 5) for v in system[name].values()],
"line": {"width": 1.2, "color": COLORS[0]}}]})
if not out:
raise Refusal(f"no measurement of {station} in this range")
return {"A": data.iso(a), "B": data.iso(b), "T": T, "plots": out}
# --- the Data page, "Detectors" labels: the events of the journal, read-only
HISTORY = timedelta(days=3) # days before given to the detectors, as categories.py does (rule of 2026-10-24)
# ponytail: kept until the server restarts (a minute or more per range); restart it after an /improve run outside the app
@functools.lru_cache(maxsize=32)
def detectors(station, start, end, targets=""):
"""-> the detectors' events over the range, in the label format (start, end, family, description); plus kind,
sensors, ignore (greyed area of the journal). targets: the sensors kept, comma-separated (empty: all); the others are
removed before the detectors, as if they did not exist."""
d = folder(station)
every = data.load(d)
if targets:
keep = set(targets.split(","))
every = {s: ms for s, ms in every.items() if s in keep}
if not every:
raise Refusal(f"no \"advanced\" export for {station}" + " with these targets" * bool(targets))
a, b = span(every, start, end)
measurements = {s: k for s, ms in every.items() if (k := [m for m in ms if a - HISTORY <= m["t"] < b])}
evs = [v for v in events(measurements, b, str(d)) if v["end"] >= data.iso(a)] if measurements else []
refs = {s for s, ms in measurements.items() if ms[0]["pattern_type"] == "global_ref"}
return [{"id": f"D-{i + 1}", "start": v["start"], "end": v["end"], "family": v["category"], "kind": v["episodes"][0]["label"],
"description": f"{v['episodes'][0]['label']}. {v['text']}", "sensors": v["sensors"], "ignore": v["ignore"],
"targets": [s for s in v["sensors"] if s not in refs], "references": [s for s in v["sensors"] if s in refs],
"message": message(v["text"]), "action": action(v)}
for i, v in enumerate(evs)]
def message(text):
"""What happens: the first sentence of the journal (detectors/scripts/categories.py, sentence), up to its ";"; the rest
says what to do about it, except the sentence of the absorbed jumps, kept. ". " only: "1.5×" or "0.30 px" do not cut."""
same = re.search(r" (The jumps? of .* same cause\.)$", text)
return re.split(r"\. |; ", text, maxsplit=1)[0].rstrip(".") + "." + (" " + same.group(1) if same else "")
# What to do about it: the "For the client" column of labeling/families.md, per family; refined per kind (the label of the
# main episode, with its level) after the journal sentences validated on 2026-10-24 (detectors/scripts/categories.py, sentence)
ACTION_FAMILY = {
"Visibility": "Do not trust the curves in this period: this is not a movement.",
"Obstruction": "Ignore the curves of the hidden targets in this period.",
"Thermal displacement": "Normal, nothing to do.",
"Displacement": "To check: look at the images, then keep watching.",
"Target issue": "Act on the target (prism, mount); ignore its curve in the meantime.",
"No data": "The curves are missing in this period; its cause tells whether to act.",
}
ACTION_KIND = {
"Poor visibility reduced": "Use the data in this period with care: they are less precise.",
"Heat shimmer": "Nothing to do: the small fast wiggles are not movement.",
"Camera thermal movement": "Nothing to do: this is not a movement of the structure.",
"Brief obstruction": "Ignore this single point on the curves.",
"Sudden jump, back to normal": "Look at the images: the cause is not established.",
"Lasting shift": "Check whether the structure moved: look at the images.",
"Jump, not back yet": "Check whether the structure moved: look at the images.",
"Large shift": "Check whether the structure moved: look at the images.",
"Slow drift": "Keep watching it.",
}
def action(v):
ep = v["episodes"][0]
return ACTION_KIND.get(f"{ep['label']} {ep.get('level') or ''}".strip(), ACTION_KIND.get(ep["label"])) \
or ACTION_FAMILY.get(v["category"], "")
# --- the labels
def label_file(station, source):
if source not in FOLDERS:
raise Refusal("unknown label folder")
folder(station)
return LABELS / source / f"{station}.jsonl"
def read(station, source):
f = label_file(station, source)
# split and not splitlines: the latter would also cut on the Unicode separators the text may contain
return [json.loads(l) for l in f.read_text(encoding="utf-8").split("\n") if l.strip()] if f.exists() else []
def assign_ids(station, labels):
"""As labeling/ids.py does (D-131): an id for the labels without one, the next of the station, as first field."""
n = max((int(e["id"].rsplit("-", 1)[1]) for e in labels if "id" in e), default=0)
for e in labels:
if "id" not in e:
n += 1
e["id"] = f"{station}-{n}"
if next(iter(e)) != "id":
rest = {k: v for k, v in e.items() if k != "id"}
ident = e["id"]
e.clear()
e.update({"id": ident}, **rest)
def _validate(body, start, end):
description = (body.get("description") or "").strip()
try:
empty = datetime.strptime(start, FORMAT) >= datetime.strptime(end, FORMAT)
except (TypeError, ValueError):
raise Refusal("unreadable bounds (YYYY-MM-DDTHH:MMZ)") from None
if not description or empty:
raise Refusal("empty text or empty range")
family = body.get("family") or ""
if family and family not in FAMILIES:
raise Refusal("unknown family")
return description, family
def add(body):
station, source = body.get("station"), body.get("folder", "manual")
f = label_file(station, source)
description, family = _validate(body, body.get("start"), body.get("end"))
label = {"start": body["start"], "end": body["end"]} | ({"family": family} if family else {}) | {"description": description}
if source == "auto": # an automatic label carries a stable id, the next of its station (D-131)
assign_ids(station, read(station, "auto") + [label])
f.parent.mkdir(parents=True, exist_ok=True)
before = f.read_bytes() if f.exists() else b"" # a line removed by hand may leave the file without a final newline
with open(f, "ab") as out:
out.write(b"\n" * bool(before and not before.endswith(b"\n")) + json.dumps(label, ensure_ascii=False).encode() + b"\n")
return label
def update(body):
before = body.get("before") or {}
start, end = body.get("start", before.get("start")), body.get("end", before.get("end"))
description, family = _validate(body, start, end)
label = dict.fromkeys(("id", "start", "end", "family")) | before | {"start": start, "end": end, "family": family,
"description": description}
if label["id"] is None:
del label["id"]
if not family:
del label["family"]
label.pop("review", None) # changed: no longer reviewed
return replace(body.get("station"), body.get("folder", "manual"), before, label)
def delete(body):
replace(body.get("station"), body.get("folder", "manual"), body.get("before"), None)
return body.get("before")
def replace(station, source, before, label):
"""Replaces the label before by label (None: removes it), found by its content, not by its rank: the file may have
changed since the page was loaded. Writes next to it then replaces: an interruption does not damage the file."""
f = label_file(station, source)
lines = f.read_text(encoding="utf-8").split("\n") if f.exists() else []
for i, line in enumerate(lines):
if line.strip() and json.loads(line) == before:
lines[i:i + 1] = [] if label is None else [json.dumps(label, ensure_ascii=False)]
temp = f.with_name(f.name + ".tmp")
temp.write_text("\n".join(lines), encoding="utf-8")
os.replace(temp, f)
return label
raise Refusal("this label is no longer in the file: reload the page")