Files
aimsight-app/app/server/analysis.py
T

317 lines
18 KiB
Python
Raw Normal View History

"""The Analysis tab of the Data page: the analyst of the agents repository (agents/, a plugin: --agent agents:analyst), launched
by `claude -p`, non-interactive, on a range chosen in the page, with the user's optional hints, its instructions
(instructions/analyst.md) and without the detectors (DETECTORS: no: its labels will be used to improve them). It writes in
labeling/auto/analyses/<STATION>_<start>_<end>/ its labels (labels.jsonl) and a diagnostic report (report.md, its figures
next to it); at the end, the server submits its labels to labeling/auto/<STATION>.jsonl, marked with their analysis (field
`analysis`). Then the user can write to it: the same session resumes (`claude -p --resume`), and its labels, if they
change, replace those it had submitted.
On a finished analysis, "Optimize" launches `/improve` in Optimizer (optimizer/, the skill that improves the detectors on the
range of the analysis, its labels as truth), then the tool-builder of the agents repository, which brings the analyst's
scripts into the reading tools (tools/); one optimization at a time, no analysis or message meanwhile; the summary of the
session (optimizer/sessions/<analysis>_<time>/summary.md) and that of the tools (<analysis>/tools.md) show under the report.
"""
import html
import json
import os
import subprocess
import threading
from datetime import date, datetime, timedelta, timezone
import markdown
import station
FOLDER = station.LABELS / "auto" / "analyses"
DETECTORS = station.ROOT / "detectors"
AGENTS = station.ROOT / "agents" # the plugin of the generic agents (~/agents)
OPTIMIZER = station.ROOT / "optimizer" # the Optimizer repository (~/Optimizer), where /improve runs
SESSIONS = OPTIMIZER / "sessions"
# the request of /improve (optimizer/README.md), paths relative to the root of Optimizer (its links data, labeling, detectors)
IMPROVE = """/improve DETECTORS: detectors/. STATION: data/{station}. FROM: {start}. TO: {end}. \
LABELS: labeling/auto/{station}.jsonl. REFERENCE: --labels labeling/auto --data data. FAMILIES: labeling/families.md. \
CONTEXT: labeling/auto/analyses/{name}/report.md. OUT: sessions/{name}_{time}/. Caller's context: Diagnostic-app, through \
Optimizer; read-only on data/ and labeling/; commit only in detectors/, on its current branch; do not commit OUT."""
REQUEST = """Station: data/{station}/. Range: {start} → {end} (UTC). Hints: {hints}. \
Output folder: {out}/. DETECTORS: no. INSTRUCTIONS: instructions/analyst.md."""
# name -> {"state": "running" | "done" | "error", "lines": [...], "submitted": n}; one analysis at a time per range
analyses, lock = {}, threading.Lock()
# name of the analysis -> {"state": "running" | "done" | "error", "lines": [...]}; one optimization at a time, all analyses
optimizations = {}
BUSY = "an optimization of the detectors is running"
ISOLATED = ["--setting-sources", "project,local", "--strict-mcp-config"] # no plugins, hooks or MCP servers of the workstation
def _env():
"""The environment of claude -p, without an API key: the agents run on the Claude subscription, never on a key."""
return {k: v for k, v in os.environ.items() if k not in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")}
def _optimizing():
return any(o["state"] == "running" for o in optimizations.values())
def launch(body):
st, start, end = body.get("station"), body.get("start"), body.get("end")
station._validate({"description": "-"}, start, end)
station.folder(st)
if not (AGENTS / ".claude-plugin" / "plugin.json").exists():
raise station.Refusal("no agents in agents/ (the link: README.md)")
name = f"{st}_{start[:16].replace(':', '')}_{end[:16].replace(':', '')}"
with lock:
if _optimizing():
raise station.Refusal(BUSY)
if analyses.get(name, {}).get("state") == "running":
raise station.Refusal("this range is already being analyzed")
out = FOLDER / name
if out.exists(): # a new analysis of the same range: the old one stays, next to it
name += datetime.now(timezone.utc).strftime("_%H%M%S")
out = FOLDER / name
hints = (body.get("hints") or "").strip()
request = {"station": st, "start": start, "end": end, "hints": hints}
analyses[name] = {"state": "running", "lines": [], "submitted": 0, "station": st, "request": request}
(out / "work").mkdir(parents=True)
(out / "request.json").write_text(json.dumps(request, ensure_ascii=False, indent=1), encoding="utf-8")
prompt = REQUEST.format(station=st, start=start, end=end, out=out.relative_to(station.ROOT), hints=hints or "none")
threading.Thread(target=work, args=(name, out, prompt), daemon=True).start()
return {"name": name}
def work(name, out, prompt, resume=None):
"""A turn of the analyst: the request, or a message of the user in the same session (resume: its id)."""
a = analyses[name]
try:
p = subprocess.Popen(["claude", "-p", prompt, "--plugin-dir", str(AGENTS), "--agent", "agents:analyst", *ISOLATED,
"--output-format", "stream-json", "--verbose",
"--allowedTools", "Read,Write,Bash", "--permission-mode", "acceptEdits", *station.links()]
+ (["--resume", resume] if resume else []),
cwd=station.ROOT, env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
for line in p.stdout:
log.write(line)
a["lines"] += digest(line)
if '"subtype":"init"' in line: # the session, for the next messages
(out / "work" / "session").write_text(json.loads(line)["session_id"])
if p.wait():
raise RuntimeError(f"claude ended with code {p.returncode}")
a["submitted"] = submit(a["station"], name, out)
a["state"] = "done"
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
a["lines"].append(f"Error: {e}")
a["state"] = "error"
def message(name, body):
"""A message of the user to the analyst, after its report: the same session resumes (claude -p --resume)."""
text, out = (body.get("text") or "").strip(), FOLDER / name
state(name) # the name checked, the state read again after a restart
a, session = analyses[name], out / "work" / "session"
if not text or not session.exists():
raise station.Refusal("empty message, or the analysis has no session to resume")
with lock:
if _optimizing():
raise station.Refusal(BUSY)
if a["state"] == "running":
raise station.Refusal("the analyst is still working")
a["state"] = "running"
line = json.dumps({"type": "page", "text": text}, ensure_ascii=False) + "\n"
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
log.write(line)
a["lines"] += digest(line)
prompt = (f"{text}\n\n(If you change your labels, rewrite {out.relative_to(station.ROOT)}/labels.jsonl in full, the "
"server will submit them again; if the diagnosis changes, update report.md.)")
threading.Thread(target=work, args=(name, out, prompt, session.read_text()), daemon=True).start()
return {}
def digest(line):
"""A line of the log -> what the page shows of it: the user's messages ("› "), the analyst's texts, its tools and their
command, one line each."""
try:
m = json.loads(line)
except json.JSONDecodeError:
return [line.strip()[:200]] if line.strip() else []
if m.get("type") == "page":
return [f"› {m['text']}"]
if m.get("type") != "assistant":
return []
out = []
for c in m["message"].get("content", []):
if c.get("type") == "text" and c["text"].strip():
out.append(c["text"].strip()[:300])
elif c.get("type") == "tool_use":
i = c.get("input", {})
out.append(f"{c['name']}: {i.get('command') or i.get('file_path') or ''}"[:200])
return out
def submit(st, name, out):
"""The labels of the analysis into labeling/auto/<STATION>.jsonl, with their id and the name of the analysis; if the
analyst changed them since the last submission, they replace those it had submitted -> their number."""
labels, copy = out / "labels.jsonl", out / "work" / "submitted.jsonl"
text = labels.read_text(encoding="utf-8") if labels.exists() else ""
new = []
for l in text.split("\n"):
if l.strip():
e = json.loads(l)
station._validate(e, e.get("start"), e.get("end"))
new.append({k: e[k] for k in ("start", "end", "family", "description") if k in e} | {"analysis": name})
if copy.exists() and copy.read_text(encoding="utf-8") == text:
return len(new)
# ponytail: removed then put back, the labels of the analysis lose the edits made in the page; keeping them would
# require matching them one by one
with lock:
f = station.label_file(st, "auto") # the other lines stay as they are
kept = [l for l in (f.read_text(encoding="utf-8").split("\n") if f.exists() else [])
if l.strip() and json.loads(l).get("analysis") != name]
station.assign_ids(st, [json.loads(l) for l in kept] + new)
temp = f.with_name(f.name + ".tmp")
temp.write_text("".join(l + "\n" for l in kept + [json.dumps(e, ensure_ascii=False) for e in new]), encoding="utf-8")
os.replace(temp, f)
copy.write_text(text, encoding="utf-8")
return len(new)
def state(name):
out = FOLDER / name
if "/" in name or name.startswith(".") or not out.is_dir():
raise station.Refusal("unknown analysis")
if name not in analyses: # after a restart of the server, only the folder is left: the thread is read again in the log
log = out / "work" / "log.jsonl"
lines = [r for l in open(log, encoding="utf-8") for r in digest(l)] if log.exists() else []
request = json.loads((out / "request.json").read_text(encoding="utf-8"))
analyses[name] = {"state": "done" if (out / "report.md").exists() else "interrupted", "lines": lines,
"submitted": None, "station": request["station"], "request": request}
a, r = analyses[name], out / "report.md"
return {"name": name, "state": a["state"], "lines": a["lines"], "submitted": a["submitted"],
"report": int(r.stat().st_mtime) if r.exists() else None,
"dialogue": (out / "work" / "session").exists(), "request": a["request"], "optimization": optimization(name)}
def list_analyses():
"""The history of the analyses (the sidebar), the most recent first: those of the server with their state, the others
read on disk (done with a report, otherwise interrupted: an analysis stops with the server)."""
out = []
for d in sorted(FOLDER.glob("*/request.json"), key=lambda f: f.parent.stat().st_mtime, reverse=True):
name = d.parent.name
a = analyses.get(name) or {"state": "done" if (d.parent / "report.md").exists() else "interrupted",
"request": json.loads(d.read_text(encoding="utf-8"))}
out.append({"name": name, "state": a["state"]} | a["request"])
return out
def file(name, path):
"""A file of the folder of the analysis (the report, its figures), never outside."""
f = (FOLDER / name / path).resolve()
if not f.is_file() or not f.is_relative_to(FOLDER.resolve()):
raise station.Refusal("unknown file")
return f
STYLE = """body{font:14px/1.55 Roboto,sans-serif;margin:0;padding:4px 2px;color:#212529;background:transparent}
h1{font-size:20px}h2{font-size:16px;margin-top:22px}img{max-width:100%;border:1px solid #dee2e6;border-radius:4px}
code{font-size:12px;background:#f1f3f5;padding:1px 4px;border-radius:3px}pre{overflow-x:auto}
table{border-collapse:collapse}td,th{border:1px solid #dee2e6;padding:3px 8px;text-align:left}
body.dark{color:#c9c9c9}body.dark code{background:#2e2e2e}body.dark td,body.dark th,body.dark img{border-color:#424242}"""
def _page(text, title, dark):
"""A markdown in HTML, in the style of the page; its relative images resolve in the folder of the address."""
body = markdown.markdown(text, extensions=["tables", "fenced_code"])
return (f"<!doctype html><meta charset=utf-8><title>{html.escape(title)}</title><style>{STYLE}</style>"
f"<body class='{'dark' if dark else ''}'>{body}</body>")
def report(name, dark):
"""report.md in HTML, served in the folder of the analysis: its relative images are found there."""
return _page(file(name, "report.md").read_text(encoding="utf-8"), name, dark)
# --- "Optimize": /improve in Optimizer on the analysis, then the tool-builder, by claude -p
def _summary_file(name):
"""The summary of the last session of this analysis, or None. The time pattern rules out the sessions of another
analysis of the same range (<name>_<HHMMSS>_<time>)."""
summaries = sorted(SESSIONS.glob(f"{name}_????????T????Z/summary.md")) # the order of the name is that of time
return summaries[-1] if summaries else None
def optimization(name):
"""-> None (never optimized) or {state, lines, summary: mtime of the last summary}. After a restart, the state is read
again in the stream: its last line {"type": "end"} says done or error; without it, the optimization was cut
(interrupted)."""
stream, s = FOLDER / name / "work" / "optimization.jsonl", _summary_file(name)
o = optimizations.get(name)
if o is None:
if not stream.exists():
return None
lines = stream.read_text(encoding="utf-8").splitlines()
end = json.loads(lines[-1]) if lines and lines[-1].startswith('{"type": "end"') else None
o = {"state": "interrupted" if end is None else end["state"], "lines": [r for l in lines for r in digest(l)]}
optimizations[name] = o
return {"state": o["state"], "lines": o["lines"], "summary": int(s.stat().st_mtime) if s else None}
def optimize(name):
"""The "Optimize" button on a finished analysis: the session in a thread; refused if an optimization or an analysis runs."""
if state(name)["state"] != "done":
raise station.Refusal("only a finished analysis can be optimized")
if not ((OPTIMIZER / ".claude" / "skills" / "improve" / "SKILL.md").exists()
and all((OPTIMIZER / link).exists() for link in ("data", "labeling", "detectors"))
and (AGENTS / ".claude-plugin" / "plugin.json").exists()):
raise station.Refusal("no Optimizer in optimizer/ or no agents in agents/ (the links: README.md)")
with lock:
if _optimizing():
raise station.Refusal(BUSY)
if any(a["state"] == "running" for a in analyses.values()):
raise station.Refusal("an analysis is running: the optimization waits for its end")
optimizations[name] = {"state": "running", "lines": []}
threading.Thread(target=optimize_thread, args=(name,), daemon=True).start()
return {}
def optimize_thread(name):
"""The two turns of the optimization, one after the other: the detectors (/improve, in Optimizer), then the reading
tools (the tool-builder, even if the detectors did not change)."""
o, out = optimizations[name], FOLDER / name
rel = os.path.relpath(out, station.ROOT)
(out / "work").mkdir(exist_ok=True)
with open(out / "work" / "optimization.jsonl", "a", encoding="utf-8") as stream:
try:
d = json.loads((out / "request.json").read_text(encoding="utf-8"))
request = IMPROVE.format( # TO excluded: the day after the end of the range, as in the page
station=d["station"], start=d["start"][:10], end=date.fromisoformat(d["end"][:10]) + timedelta(days=1),
name=name, time=f"{datetime.now(timezone.utc):%Y%m%dT%H%MZ}")
# ponytail: Optimizer reads data/ and labeling/ through its own links; AIMSIGHT_DATA / AIMSIGHT_LABELS do not follow
turns = [(OPTIMIZER.resolve(), [request, "--plugin-dir", str(AGENTS.resolve()),
"--add-dir", str(DETECTORS.resolve()), # the target of the link, where the session writes
"--allowedTools", "Read,Write,Edit,Bash,Agent"]),
(station.ROOT, [f"Analysis: {rel}/. Tools: tools/. Write {rel}/tools.md.", "--plugin-dir", str(AGENTS),
"--agent", "agents:tool-builder", "--allowedTools", "Read,Write,Edit,Bash"])]
for cwd, (prompt, *options) in turns:
p = subprocess.Popen(["claude", "-p", prompt, *ISOLATED, "--output-format", "stream-json", "--verbose",
"--permission-mode", "acceptEdits", *station.links(), *options],
cwd=cwd, env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
for line in p.stdout:
stream.write(line)
o["lines"] += digest(line)
if p.wait():
raise RuntimeError(f"claude ended with code {p.returncode}")
o["state"] = "done"
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
o["lines"].append(f"Error: {e}")
o["state"] = "error"
finally:
stream.write(json.dumps({"type": "end", "state": o["state"]}) + "\n")
station.detectors.cache_clear() # the detectors may have changed: the Data page computes them again
def summary(name, dark):
"""The summary of the detectors' session, then that of the tools (<analysis>/tools.md), in one page."""
s, tools = _summary_file(name), FOLDER / name / "tools.md"
if s is None and not tools.exists():
raise station.Refusal("no summary for this analysis")
text = "\n\n".join(f.read_text(encoding="utf-8") for f in (s, tools) if f and f.exists())
return _page(text, name, dark)