Files
aimsight-app/app/server/analysis.py
T

311 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""The Analysis tab of the Data page: the analyst of the agents repository (agents/, a plugin: --agent agents:analyst), launched
by `claude -p`, non-interactive, on a range chosen in the page, with the user's optional hints, its instructions
(instructions/analyst.md) and without the detectors (DETECTORS: no: its labels will be used to improve them). It writes in
labeling/auto/analyses/<STATION>_<start>_<end>/ its labels (labels.jsonl) and a diagnostic report (report.md, its figures
next to it); at the end, the server submits its labels to labeling/auto/<STATION>.jsonl, marked with their analysis (field
`analysis`). Then the user can write to it: the same session resumes (`claude -p --resume`), and its labels, if they
change, replace those it had submitted.
On a finished analysis, "Optimize" launches `/optimize` in Optimizer (optimizer/, the skill that, from the analysis, improves
the detectors on its range, its labels as truth, then brings the analyst's scripts into the reading tools, tools/); one
optimization at a time, no analysis or message meanwhile; the summary of the session
(optimizer/sessions/<analysis>_<time>/summary.md, and the <analysis>/tools.md of an older session) shows under the report.
"""
import html
import json
import os
import subprocess
import threading
from datetime import datetime, timezone
import markdown
import station
FOLDER = station.LABELS / "auto" / "analyses"
DETECTORS = station.ROOT / "detectors"
AGENTS = station.ROOT / "agents" # the plugin of the generic agents (~/agents)
OPTIMIZER = station.ROOT / "optimizer" # the Optimizer repository (~/Optimizer), where /optimize runs
SESSIONS = OPTIMIZER / "sessions"
# the request of /optimize (optimizer/README.md), paths relative to the root of Optimizer (its links); Optimizer reads the
# station and the range in the analysis
OPTIMIZE = """/optimize ANALYSIS: labeling/auto/analyses/{name}/. OUT: sessions/{name}_{time}/. Caller's context: \
Diagnostic-app, through Optimizer; read-only on data/ and labeling/; commit only in detectors/ and tools/, on their current \
branch; do not commit OUT."""
REQUEST = """Station: data/{station}/. Range: {start} → {end} (UTC). Hints: {hints}. \
Output folder: {out}/. DETECTORS: no. INSTRUCTIONS: instructions/analyst.md."""
# name -> {"state": "running" | "done" | "error", "lines": [...], "submitted": n}; one analysis at a time per range
analyses, lock = {}, threading.Lock()
# name of the analysis -> {"state": "running" | "done" | "error", "lines": [...]}; one optimization at a time, all analyses
optimizations = {}
BUSY = "an optimization of the detectors is running"
ISOLATED = ["--setting-sources", "project,local", "--strict-mcp-config"] # no plugins, hooks or MCP servers of the workstation
def _env():
"""The environment of claude -p, without an API key: the agents run on the Claude subscription, never on a key."""
return {k: v for k, v in os.environ.items() if k not in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")}
def _optimizing():
return any(o["state"] == "running" for o in optimizations.values())
def launch(body):
st, start, end = body.get("station"), body.get("start"), body.get("end")
station._validate({"description": "-"}, start, end)
station.folder(st)
if not (AGENTS / ".claude-plugin" / "plugin.json").exists():
raise station.Refusal("no agents in agents/ (the link: README.md)")
name = f"{st}_{start[:16].replace(':', '')}_{end[:16].replace(':', '')}"
with lock:
if _optimizing():
raise station.Refusal(BUSY)
if analyses.get(name, {}).get("state") == "running":
raise station.Refusal("this range is already being analyzed")
out = FOLDER / name
if out.exists(): # a new analysis of the same range: the old one stays, next to it
name += datetime.now(timezone.utc).strftime("_%H%M%S")
out = FOLDER / name
hints = (body.get("hints") or "").strip()
request = {"station": st, "start": start, "end": end, "hints": hints}
analyses[name] = {"state": "running", "lines": [], "submitted": 0, "station": st, "request": request}
(out / "work").mkdir(parents=True)
(out / "request.json").write_text(json.dumps(request, ensure_ascii=False, indent=1), encoding="utf-8")
prompt = REQUEST.format(station=st, start=start, end=end, out=out.relative_to(station.ROOT), hints=hints or "none")
threading.Thread(target=work, args=(name, out, prompt), daemon=True).start()
return {"name": name}
def work(name, out, prompt, resume=None):
"""A turn of the analyst: the request, or a message of the user in the same session (resume: its id)."""
a = analyses[name]
try:
p = subprocess.Popen(["claude", "-p", prompt, "--plugin-dir", str(AGENTS), "--agent", "agents:analyst", *ISOLATED,
"--output-format", "stream-json", "--verbose",
"--allowedTools", "Read,Write,Bash", "--permission-mode", "acceptEdits", *station.links()]
+ (["--resume", resume] if resume else []),
cwd=station.ROOT, env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
for line in p.stdout:
log.write(line)
a["lines"] += digest(line)
if '"subtype":"init"' in line: # the session, for the next messages
(out / "work" / "session").write_text(json.loads(line)["session_id"])
if p.wait():
raise RuntimeError(f"claude ended with code {p.returncode}")
a["submitted"] = submit(a["station"], name, out)
a["state"] = "done"
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
a["lines"].append(f"Error: {e}")
a["state"] = "error"
def message(name, body):
"""A message of the user to the analyst, after its report: the same session resumes (claude -p --resume)."""
text, out = (body.get("text") or "").strip(), FOLDER / name
state(name) # the name checked, the state read again after a restart
a, session = analyses[name], out / "work" / "session"
if not text or not session.exists():
raise station.Refusal("empty message, or the analysis has no session to resume")
with lock:
if _optimizing():
raise station.Refusal(BUSY)
if a["state"] == "running":
raise station.Refusal("the analyst is still working")
a["state"] = "running"
line = json.dumps({"type": "page", "text": text}, ensure_ascii=False) + "\n"
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
log.write(line)
a["lines"] += digest(line)
prompt = (f"{text}\n\n(If you change your labels, rewrite {out.relative_to(station.ROOT)}/labels.jsonl in full, the "
"server will submit them again; if the diagnosis changes, update report.md.)")
threading.Thread(target=work, args=(name, out, prompt, session.read_text()), daemon=True).start()
return {}
def digest(line):
"""A line of the log -> what the page shows of it: the user's messages ("› "), the analyst's texts, its tools and their
command, one line each."""
try:
m = json.loads(line)
except json.JSONDecodeError:
return [line.strip()[:200]] if line.strip() else []
if m.get("type") == "page":
return [f"› {m['text']}"]
if m.get("type") != "assistant":
return []
out = []
for c in m["message"].get("content", []):
if c.get("type") == "text" and c["text"].strip():
out.append(c["text"].strip()[:300])
elif c.get("type") == "tool_use":
i = c.get("input", {})
out.append(f"{c['name']}: {i.get('command') or i.get('file_path') or ''}"[:200])
return out
def submit(st, name, out):
"""The labels of the analysis into labeling/auto/<STATION>.jsonl, with their id and the name of the analysis; if the
analyst changed them since the last submission, they replace those it had submitted -> their number."""
labels, copy = out / "labels.jsonl", out / "work" / "submitted.jsonl"
text = labels.read_text(encoding="utf-8") if labels.exists() else ""
new = []
for l in text.split("\n"):
if l.strip():
e = json.loads(l)
station._validate(e, e.get("start"), e.get("end"))
new.append({k: e[k] for k in ("start", "end", "family", "description") if k in e} | {"analysis": name})
if copy.exists() and copy.read_text(encoding="utf-8") == text:
return len(new)
# ponytail: removed then put back, the labels of the analysis lose the edits made in the page; keeping them would
# require matching them one by one
with lock:
f = station.label_file(st, "auto") # the other lines stay as they are
kept = [l for l in (f.read_text(encoding="utf-8").split("\n") if f.exists() else [])
if l.strip() and json.loads(l).get("analysis") != name]
station.assign_ids(st, [json.loads(l) for l in kept] + new)
temp = f.with_name(f.name + ".tmp")
temp.write_text("".join(l + "\n" for l in kept + [json.dumps(e, ensure_ascii=False) for e in new]), encoding="utf-8")
os.replace(temp, f)
copy.write_text(text, encoding="utf-8")
return len(new)
def state(name):
out = FOLDER / name
if "/" in name or name.startswith(".") or not out.is_dir():
raise station.Refusal("unknown analysis")
if name not in analyses: # after a restart of the server, only the folder is left: the thread is read again in the log
log = out / "work" / "log.jsonl"
lines = [r for l in open(log, encoding="utf-8") for r in digest(l)] if log.exists() else []
request = json.loads((out / "request.json").read_text(encoding="utf-8"))
analyses[name] = {"state": "done" if (out / "report.md").exists() else "interrupted", "lines": lines,
"submitted": None, "station": request["station"], "request": request}
a, r = analyses[name], out / "report.md"
return {"name": name, "state": a["state"], "lines": a["lines"], "submitted": a["submitted"],
"report": int(r.stat().st_mtime) if r.exists() else None,
"dialogue": (out / "work" / "session").exists(), "request": a["request"], "optimization": optimization(name)}
def list_analyses():
"""The history of the analyses (the sidebar), the most recent first: those of the server with their state, the others
read on disk (done with a report, otherwise interrupted: an analysis stops with the server)."""
out = []
for d in sorted(FOLDER.glob("*/request.json"), key=lambda f: f.parent.stat().st_mtime, reverse=True):
name = d.parent.name
a = analyses.get(name) or {"state": "done" if (d.parent / "report.md").exists() else "interrupted",
"request": json.loads(d.read_text(encoding="utf-8"))}
out.append({"name": name, "state": a["state"]} | a["request"])
return out
def file(name, path):
"""A file of the folder of the analysis (the report, its figures), never outside."""
f = (FOLDER / name / path).resolve()
if not f.is_file() or not f.is_relative_to(FOLDER.resolve()):
raise station.Refusal("unknown file")
return f
STYLE = """body{font:14px/1.55 Roboto,sans-serif;margin:0;padding:4px 2px;color:#212529;background:transparent}
h1{font-size:20px}h2{font-size:16px;margin-top:22px}img{max-width:100%;border:1px solid #dee2e6;border-radius:4px}
code{font-size:12px;background:#f1f3f5;padding:1px 4px;border-radius:3px}pre{overflow-x:auto}
table{border-collapse:collapse}td,th{border:1px solid #dee2e6;padding:3px 8px;text-align:left}
body.dark{color:#c9c9c9}body.dark code{background:#2e2e2e}body.dark td,body.dark th,body.dark img{border-color:#424242}"""
def _page(text, title, dark):
"""A markdown in HTML, in the style of the page; its relative images resolve in the folder of the address."""
body = markdown.markdown(text, extensions=["tables", "fenced_code"])
return (f"<!doctype html><meta charset=utf-8><title>{html.escape(title)}</title><style>{STYLE}</style>"
f"<body class='{'dark' if dark else ''}'>{body}</body>")
def report(name, dark):
"""report.md in HTML, served in the folder of the analysis: its relative images are found there."""
return _page(file(name, "report.md").read_text(encoding="utf-8"), name, dark)
# --- "Optimize": /optimize in Optimizer on the analysis, by claude -p
def _summary_file(name):
"""The summary of the last session of this analysis, or None. The time pattern rules out the sessions of another
analysis of the same range (<name>_<HHMMSS>_<time>)."""
summaries = sorted(SESSIONS.glob(f"{name}_????????T????Z/summary.md")) # the order of the name is that of time
return summaries[-1] if summaries else None
def optimization(name):
"""-> None (never optimized) or {state, lines, summary: mtime of the last summary}. After a restart, the state is read
again in the stream: its last line {"type": "end"} says done or error; without it, the optimization was cut
(interrupted)."""
stream, s = FOLDER / name / "work" / "optimization.jsonl", _summary_file(name)
o = optimizations.get(name)
if o is None:
if not stream.exists():
return None
lines = stream.read_text(encoding="utf-8").splitlines()
end = json.loads(lines[-1]) if lines and lines[-1].startswith('{"type": "end"') else None
o = {"state": "interrupted" if end is None else end["state"], "lines": [r for l in lines for r in digest(l)]}
optimizations[name] = o
return {"state": o["state"], "lines": o["lines"], "summary": int(s.stat().st_mtime) if s else None}
def optimize(name):
"""The "Optimize" button on a finished analysis: the session in a thread; refused if an optimization or an analysis runs."""
if state(name)["state"] != "done":
raise station.Refusal("only a finished analysis can be optimized")
if not ((OPTIMIZER / ".claude" / "skills" / "optimize" / "SKILL.md").exists()
and all((OPTIMIZER / link).exists() for link in ("data", "labeling", "detectors", "tools"))
and (AGENTS / ".claude-plugin" / "plugin.json").exists()):
raise station.Refusal("no Optimizer in optimizer/ or no agents in agents/ (the links: README.md)")
with lock:
if _optimizing():
raise station.Refusal(BUSY)
if any(a["state"] == "running" for a in analyses.values()):
raise station.Refusal("an analysis is running: the optimization waits for its end")
optimizations[name] = {"state": "running", "lines": []}
threading.Thread(target=optimize_thread, args=(name,), daemon=True).start()
return {}
def optimize_thread(name):
"""The optimization: /optimize in Optimizer, on the analysis (the detectors, then the reading tools)."""
o, out = optimizations[name], FOLDER / name
(out / "work").mkdir(exist_ok=True)
with open(out / "work" / "optimization.jsonl", "a", encoding="utf-8") as stream:
try:
request = OPTIMIZE.format(name=name, time=f"{datetime.now(timezone.utc):%Y%m%dT%H%MZ}")
# ponytail: Optimizer reads data/ and labeling/ through its own links; AIMSIGHT_DATA / AIMSIGHT_LABELS do not follow
p = subprocess.Popen(["claude", "-p", request, *ISOLATED, "--output-format", "stream-json", "--verbose",
"--permission-mode", "acceptEdits", *station.links(), # tools/ among them, where it writes
"--plugin-dir", str(AGENTS.resolve()),
"--add-dir", str(DETECTORS.resolve()), # the target of the link, where the session writes
"--allowedTools", "Read,Write,Edit,Bash,Agent"],
cwd=OPTIMIZER.resolve(), env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True)
for line in p.stdout:
stream.write(line)
o["lines"] += digest(line)
if p.wait():
raise RuntimeError(f"claude ended with code {p.returncode}")
o["state"] = "done"
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
o["lines"].append(f"Error: {e}")
o["state"] = "error"
finally:
stream.write(json.dumps({"type": "end", "state": o["state"]}) + "\n")
station.detectors.cache_clear() # the detectors may have changed: the Data page computes them again
def summary(name, dark):
"""The summary of the session (the detectors and the tools), then the <analysis>/tools.md of an older session."""
s, tools = _summary_file(name), FOLDER / name / "tools.md"
if s is None and not tools.exists():
raise station.Refusal("no summary for this analysis")
text = "\n\n".join(f.read_text(encoding="utf-8") for f in (s, tools) if f and f.exists())
return _page(text, name, dark)