317 lines
18 KiB
Python
317 lines
18 KiB
Python
"""The Analysis tab of the Data page: the analyst of the agents repository (agents/, a plugin: --agent agents:analyst), launched
|
||||
|
|
by `claude -p`, non-interactive, on a range chosen in the page, with the user's optional hints, its instructions
|
|||
|
|
(instructions/analyst.md) and without the detectors (DETECTORS: no: its labels will be used to improve them). It writes in
|
|||
|
|
labeling/auto/analyses/<STATION>_<start>_<end>/ its labels (labels.jsonl) and a diagnostic report (report.md, its figures
|
|||
|
|
next to it); at the end, the server submits its labels to labeling/auto/<STATION>.jsonl, marked with their analysis (field
|
|||
|
|
`analysis`). Then the user can write to it: the same session resumes (`claude -p --resume`), and its labels, if they
|
|||
|
|
change, replace those it had submitted.
|
|||
|
|
On a finished analysis, "Optimize" launches `/improve` in Optimizer (optimizer/, the skill that improves the detectors on the
|
|||
|
|
range of the analysis, its labels as truth), then the tool-builder of the agents repository, which brings the analyst's
|
|||
|
|
scripts into the reading tools (tools/); one optimization at a time, no analysis or message meanwhile; the summary of the
|
|||
|
|
session (optimizer/sessions/<analysis>_<time>/summary.md) and that of the tools (<analysis>/tools.md) show under the report.
|
|||
|
|
"""
|
|||
|
|
import html
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
import subprocess
|
|||
|
|
import threading
|
|||
|
|
from datetime import date, datetime, timedelta, timezone
|
|||
|
|
|
|||
|
|
import markdown
|
|||
|
|
|
|||
|
|
import station
|
|||
|
|
|
|||
|
|
FOLDER = station.LABELS / "auto" / "analyses"
|
|||
|
|
DETECTORS = station.ROOT / "detectors"
|
|||
|
|
AGENTS = station.ROOT / "agents" # the plugin of the generic agents (~/agents)
|
|||
|
|
OPTIMIZER = station.ROOT / "optimizer" # the Optimizer repository (~/Optimizer), where /improve runs
|
|||
|
|
SESSIONS = OPTIMIZER / "sessions"
|
|||
|
|
# the request of /improve (optimizer/README.md), paths relative to the root of Optimizer (its links data, labeling, detectors)
|
|||
|
|
IMPROVE = """/improve DETECTORS: detectors/. STATION: data/{station}. FROM: {start}. TO: {end}. \
|
|||
|
|
LABELS: labeling/auto/{station}.jsonl. REFERENCE: --labels labeling/auto --data data. FAMILIES: labeling/families.md. \
|
|||
|
|
CONTEXT: labeling/auto/analyses/{name}/report.md. OUT: sessions/{name}_{time}/. Caller's context: Diagnostic-app, through \
|
|||
|
|
Optimizer; read-only on data/ and labeling/; commit only in detectors/, on its current branch; do not commit OUT."""
|
|||
|
|
REQUEST = """Station: data/{station}/. Range: {start} → {end} (UTC). Hints: {hints}. \
|
|||
|
|
Output folder: {out}/. DETECTORS: no. INSTRUCTIONS: instructions/analyst.md."""
|
|||
|
|
|
|||
|
|
# name -> {"state": "running" | "done" | "error", "lines": [...], "submitted": n}; one analysis at a time per range
|
|||
|
|
analyses, lock = {}, threading.Lock()
|
|||
|
|
# name of the analysis -> {"state": "running" | "done" | "error", "lines": [...]}; one optimization at a time, all analyses
|
|||
|
|
optimizations = {}
|
|||
|
|
BUSY = "an optimization of the detectors is running"
|
|||
|
|
|
|||
|
|
|
|||
|
|
ISOLATED = ["--setting-sources", "project,local", "--strict-mcp-config"] # no plugins, hooks or MCP servers of the workstation
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _env():
|
|||
|
|
"""The environment of claude -p, without an API key: the agents run on the Claude subscription, never on a key."""
|
|||
|
|
return {k: v for k, v in os.environ.items() if k not in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _optimizing():
|
|||
|
|
return any(o["state"] == "running" for o in optimizations.values())
|
|||
|
|
|
|||
|
|
|
|||
|
|
def launch(body):
|
|||
|
|
st, start, end = body.get("station"), body.get("start"), body.get("end")
|
|||
|
|
station._validate({"description": "-"}, start, end)
|
|||
|
|
station.folder(st)
|
|||
|
|
if not (AGENTS / ".claude-plugin" / "plugin.json").exists():
|
|||
|
|
raise station.Refusal("no agents in agents/ (the link: README.md)")
|
|||
|
|
name = f"{st}_{start[:16].replace(':', '')}_{end[:16].replace(':', '')}"
|
|||
|
|
with lock:
|
|||
|
|
if _optimizing():
|
|||
|
|
raise station.Refusal(BUSY)
|
|||
|
|
if analyses.get(name, {}).get("state") == "running":
|
|||
|
|
raise station.Refusal("this range is already being analyzed")
|
|||
|
|
out = FOLDER / name
|
|||
|
|
if out.exists(): # a new analysis of the same range: the old one stays, next to it
|
|||
|
|
name += datetime.now(timezone.utc).strftime("_%H%M%S")
|
|||
|
|
out = FOLDER / name
|
|||
|
|
hints = (body.get("hints") or "").strip()
|
|||
|
|
request = {"station": st, "start": start, "end": end, "hints": hints}
|
|||
|
|
analyses[name] = {"state": "running", "lines": [], "submitted": 0, "station": st, "request": request}
|
|||
|
|
(out / "work").mkdir(parents=True)
|
|||
|
|
(out / "request.json").write_text(json.dumps(request, ensure_ascii=False, indent=1), encoding="utf-8")
|
|||
|
|
prompt = REQUEST.format(station=st, start=start, end=end, out=out.relative_to(station.ROOT), hints=hints or "none")
|
|||
|
|
threading.Thread(target=work, args=(name, out, prompt), daemon=True).start()
|
|||
|
|
return {"name": name}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def work(name, out, prompt, resume=None):
|
|||
|
|
"""A turn of the analyst: the request, or a message of the user in the same session (resume: its id)."""
|
|||
|
|
a = analyses[name]
|
|||
|
|
try:
|
|||
|
|
p = subprocess.Popen(["claude", "-p", prompt, "--plugin-dir", str(AGENTS), "--agent", "agents:analyst", *ISOLATED,
|
|||
|
|
"--output-format", "stream-json", "--verbose",
|
|||
|
|
"--allowedTools", "Read,Write,Bash", "--permission-mode", "acceptEdits", *station.links()]
|
|||
|
|
+ (["--resume", resume] if resume else []),
|
|||
|
|
cwd=station.ROOT, env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|||
|
|
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
|
|||
|
|
for line in p.stdout:
|
|||
|
|
log.write(line)
|
|||
|
|
a["lines"] += digest(line)
|
|||
|
|
if '"subtype":"init"' in line: # the session, for the next messages
|
|||
|
|
(out / "work" / "session").write_text(json.loads(line)["session_id"])
|
|||
|
|
if p.wait():
|
|||
|
|
raise RuntimeError(f"claude ended with code {p.returncode}")
|
|||
|
|
a["submitted"] = submit(a["station"], name, out)
|
|||
|
|
a["state"] = "done"
|
|||
|
|
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
|
|||
|
|
a["lines"].append(f"Error: {e}")
|
|||
|
|
a["state"] = "error"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def message(name, body):
|
|||
|
|
"""A message of the user to the analyst, after its report: the same session resumes (claude -p --resume)."""
|
|||
|
|
text, out = (body.get("text") or "").strip(), FOLDER / name
|
|||
|
|
state(name) # the name checked, the state read again after a restart
|
|||
|
|
a, session = analyses[name], out / "work" / "session"
|
|||
|
|
if not text or not session.exists():
|
|||
|
|
raise station.Refusal("empty message, or the analysis has no session to resume")
|
|||
|
|
with lock:
|
|||
|
|
if _optimizing():
|
|||
|
|
raise station.Refusal(BUSY)
|
|||
|
|
if a["state"] == "running":
|
|||
|
|
raise station.Refusal("the analyst is still working")
|
|||
|
|
a["state"] = "running"
|
|||
|
|
line = json.dumps({"type": "page", "text": text}, ensure_ascii=False) + "\n"
|
|||
|
|
with open(out / "work" / "log.jsonl", "a", encoding="utf-8") as log:
|
|||
|
|
log.write(line)
|
|||
|
|
a["lines"] += digest(line)
|
|||
|
|
prompt = (f"{text}\n\n(If you change your labels, rewrite {out.relative_to(station.ROOT)}/labels.jsonl in full, the "
|
|||
|
|
"server will submit them again; if the diagnosis changes, update report.md.)")
|
|||
|
|
threading.Thread(target=work, args=(name, out, prompt, session.read_text()), daemon=True).start()
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def digest(line):
|
|||
|
|
"""A line of the log -> what the page shows of it: the user's messages ("› "), the analyst's texts, its tools and their
|
|||
|
|
command, one line each."""
|
|||
|
|
try:
|
|||
|
|
m = json.loads(line)
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
return [line.strip()[:200]] if line.strip() else []
|
|||
|
|
if m.get("type") == "page":
|
|||
|
|
return [f"› {m['text']}"]
|
|||
|
|
if m.get("type") != "assistant":
|
|||
|
|
return []
|
|||
|
|
out = []
|
|||
|
|
for c in m["message"].get("content", []):
|
|||
|
|
if c.get("type") == "text" and c["text"].strip():
|
|||
|
|
out.append(c["text"].strip()[:300])
|
|||
|
|
elif c.get("type") == "tool_use":
|
|||
|
|
i = c.get("input", {})
|
|||
|
|
out.append(f"{c['name']}: {i.get('command') or i.get('file_path') or ''}"[:200])
|
|||
|
|
return out
|
|||
|
|
|
|||
|
|
|
|||
|
|
def submit(st, name, out):
|
|||
|
|
"""The labels of the analysis into labeling/auto/<STATION>.jsonl, with their id and the name of the analysis; if the
|
|||
|
|
analyst changed them since the last submission, they replace those it had submitted -> their number."""
|
|||
|
|
labels, copy = out / "labels.jsonl", out / "work" / "submitted.jsonl"
|
|||
|
|
text = labels.read_text(encoding="utf-8") if labels.exists() else ""
|
|||
|
|
new = []
|
|||
|
|
for l in text.split("\n"):
|
|||
|
|
if l.strip():
|
|||
|
|
e = json.loads(l)
|
|||
|
|
station._validate(e, e.get("start"), e.get("end"))
|
|||
|
|
new.append({k: e[k] for k in ("start", "end", "family", "description") if k in e} | {"analysis": name})
|
|||
|
|
if copy.exists() and copy.read_text(encoding="utf-8") == text:
|
|||
|
|
return len(new)
|
|||
|
|
# ponytail: removed then put back, the labels of the analysis lose the edits made in the page; keeping them would
|
|||
|
|
# require matching them one by one
|
|||
|
|
with lock:
|
|||
|
|
f = station.label_file(st, "auto") # the other lines stay as they are
|
|||
|
|
kept = [l for l in (f.read_text(encoding="utf-8").split("\n") if f.exists() else [])
|
|||
|
|
if l.strip() and json.loads(l).get("analysis") != name]
|
|||
|
|
station.assign_ids(st, [json.loads(l) for l in kept] + new)
|
|||
|
|
temp = f.with_name(f.name + ".tmp")
|
|||
|
|
temp.write_text("".join(l + "\n" for l in kept + [json.dumps(e, ensure_ascii=False) for e in new]), encoding="utf-8")
|
|||
|
|
os.replace(temp, f)
|
|||
|
|
copy.write_text(text, encoding="utf-8")
|
|||
|
|
return len(new)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def state(name):
|
|||
|
|
out = FOLDER / name
|
|||
|
|
if "/" in name or name.startswith(".") or not out.is_dir():
|
|||
|
|
raise station.Refusal("unknown analysis")
|
|||
|
|
if name not in analyses: # after a restart of the server, only the folder is left: the thread is read again in the log
|
|||
|
|
log = out / "work" / "log.jsonl"
|
|||
|
|
lines = [r for l in open(log, encoding="utf-8") for r in digest(l)] if log.exists() else []
|
|||
|
|
request = json.loads((out / "request.json").read_text(encoding="utf-8"))
|
|||
|
|
analyses[name] = {"state": "done" if (out / "report.md").exists() else "interrupted", "lines": lines,
|
|||
|
|
"submitted": None, "station": request["station"], "request": request}
|
|||
|
|
a, r = analyses[name], out / "report.md"
|
|||
|
|
return {"name": name, "state": a["state"], "lines": a["lines"], "submitted": a["submitted"],
|
|||
|
|
"report": int(r.stat().st_mtime) if r.exists() else None,
|
|||
|
|
"dialogue": (out / "work" / "session").exists(), "request": a["request"], "optimization": optimization(name)}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def list_analyses():
|
|||
|
|
"""The history of the analyses (the sidebar), the most recent first: those of the server with their state, the others
|
|||
|
|
read on disk (done with a report, otherwise interrupted: an analysis stops with the server)."""
|
|||
|
|
out = []
|
|||
|
|
for d in sorted(FOLDER.glob("*/request.json"), key=lambda f: f.parent.stat().st_mtime, reverse=True):
|
|||
|
|
name = d.parent.name
|
|||
|
|
a = analyses.get(name) or {"state": "done" if (d.parent / "report.md").exists() else "interrupted",
|
|||
|
|
"request": json.loads(d.read_text(encoding="utf-8"))}
|
|||
|
|
out.append({"name": name, "state": a["state"]} | a["request"])
|
|||
|
|
return out
|
|||
|
|
|
|||
|
|
|
|||
|
|
def file(name, path):
|
|||
|
|
"""A file of the folder of the analysis (the report, its figures), never outside."""
|
|||
|
|
f = (FOLDER / name / path).resolve()
|
|||
|
|
if not f.is_file() or not f.is_relative_to(FOLDER.resolve()):
|
|||
|
|
raise station.Refusal("unknown file")
|
|||
|
|
return f
|
|||
|
|
|
|||
|
|
|
|||
|
|
STYLE = """body{font:14px/1.55 Roboto,sans-serif;margin:0;padding:4px 2px;color:#212529;background:transparent}
|
|||
|
|
h1{font-size:20px}h2{font-size:16px;margin-top:22px}img{max-width:100%;border:1px solid #dee2e6;border-radius:4px}
|
|||
|
|
code{font-size:12px;background:#f1f3f5;padding:1px 4px;border-radius:3px}pre{overflow-x:auto}
|
|||
|
|
table{border-collapse:collapse}td,th{border:1px solid #dee2e6;padding:3px 8px;text-align:left}
|
|||
|
|
body.dark{color:#c9c9c9}body.dark code{background:#2e2e2e}body.dark td,body.dark th,body.dark img{border-color:#424242}"""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _page(text, title, dark):
|
|||
|
|
"""A markdown in HTML, in the style of the page; its relative images resolve in the folder of the address."""
|
|||
|
|
body = markdown.markdown(text, extensions=["tables", "fenced_code"])
|
|||
|
|
return (f"<!doctype html><meta charset=utf-8><title>{html.escape(title)}</title><style>{STYLE}</style>"
|
|||
|
|
f"<body class='{'dark' if dark else ''}'>{body}</body>")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def report(name, dark):
|
|||
|
|
"""report.md in HTML, served in the folder of the analysis: its relative images are found there."""
|
|||
|
|
return _page(file(name, "report.md").read_text(encoding="utf-8"), name, dark)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# --- "Optimize": /improve in Optimizer on the analysis, then the tool-builder, by claude -p
|
|||
|
|
|
|||
|
|
def _summary_file(name):
|
|||
|
|
"""The summary of the last session of this analysis, or None. The time pattern rules out the sessions of another
|
|||
|
|
analysis of the same range (<name>_<HHMMSS>_<time>)."""
|
|||
|
|
summaries = sorted(SESSIONS.glob(f"{name}_????????T????Z/summary.md")) # the order of the name is that of time
|
|||
|
|
return summaries[-1] if summaries else None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def optimization(name):
|
|||
|
|
"""-> None (never optimized) or {state, lines, summary: mtime of the last summary}. After a restart, the state is read
|
|||
|
|
again in the stream: its last line {"type": "end"} says done or error; without it, the optimization was cut
|
|||
|
|
(interrupted)."""
|
|||
|
|
stream, s = FOLDER / name / "work" / "optimization.jsonl", _summary_file(name)
|
|||
|
|
o = optimizations.get(name)
|
|||
|
|
if o is None:
|
|||
|
|
if not stream.exists():
|
|||
|
|
return None
|
|||
|
|
lines = stream.read_text(encoding="utf-8").splitlines()
|
|||
|
|
end = json.loads(lines[-1]) if lines and lines[-1].startswith('{"type": "end"') else None
|
|||
|
|
o = {"state": "interrupted" if end is None else end["state"], "lines": [r for l in lines for r in digest(l)]}
|
|||
|
|
optimizations[name] = o
|
|||
|
|
return {"state": o["state"], "lines": o["lines"], "summary": int(s.stat().st_mtime) if s else None}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def optimize(name):
|
|||
|
|
"""The "Optimize" button on a finished analysis: the session in a thread; refused if an optimization or an analysis runs."""
|
|||
|
|
if state(name)["state"] != "done":
|
|||
|
|
raise station.Refusal("only a finished analysis can be optimized")
|
|||
|
|
if not ((OPTIMIZER / ".claude" / "skills" / "improve" / "SKILL.md").exists()
|
|||
|
|
and all((OPTIMIZER / link).exists() for link in ("data", "labeling", "detectors"))
|
|||
|
|
and (AGENTS / ".claude-plugin" / "plugin.json").exists()):
|
|||
|
|
raise station.Refusal("no Optimizer in optimizer/ or no agents in agents/ (the links: README.md)")
|
|||
|
|
with lock:
|
|||
|
|
if _optimizing():
|
|||
|
|
raise station.Refusal(BUSY)
|
|||
|
|
if any(a["state"] == "running" for a in analyses.values()):
|
|||
|
|
raise station.Refusal("an analysis is running: the optimization waits for its end")
|
|||
|
|
optimizations[name] = {"state": "running", "lines": []}
|
|||
|
|
threading.Thread(target=optimize_thread, args=(name,), daemon=True).start()
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def optimize_thread(name):
|
|||
|
|
"""The two turns of the optimization, one after the other: the detectors (/improve, in Optimizer), then the reading
|
|||
|
|
tools (the tool-builder, even if the detectors did not change)."""
|
|||
|
|
o, out = optimizations[name], FOLDER / name
|
|||
|
|
rel = os.path.relpath(out, station.ROOT)
|
|||
|
|
(out / "work").mkdir(exist_ok=True)
|
|||
|
|
with open(out / "work" / "optimization.jsonl", "a", encoding="utf-8") as stream:
|
|||
|
|
try:
|
|||
|
|
d = json.loads((out / "request.json").read_text(encoding="utf-8"))
|
|||
|
|
request = IMPROVE.format( # TO excluded: the day after the end of the range, as in the page
|
|||
|
|
station=d["station"], start=d["start"][:10], end=date.fromisoformat(d["end"][:10]) + timedelta(days=1),
|
|||
|
|
name=name, time=f"{datetime.now(timezone.utc):%Y%m%dT%H%MZ}")
|
|||
|
|
# ponytail: Optimizer reads data/ and labeling/ through its own links; AIMSIGHT_DATA / AIMSIGHT_LABELS do not follow
|
|||
|
|
turns = [(OPTIMIZER.resolve(), [request, "--plugin-dir", str(AGENTS.resolve()),
|
|||
|
|
"--add-dir", str(DETECTORS.resolve()), # the target of the link, where the session writes
|
|||
|
|
"--allowedTools", "Read,Write,Edit,Bash,Agent"]),
|
|||
|
|
(station.ROOT, [f"Analysis: {rel}/. Tools: tools/. Write {rel}/tools.md.", "--plugin-dir", str(AGENTS),
|
|||
|
|
"--agent", "agents:tool-builder", "--allowedTools", "Read,Write,Edit,Bash"])]
|
|||
|
|
for cwd, (prompt, *options) in turns:
|
|||
|
|
p = subprocess.Popen(["claude", "-p", prompt, *ISOLATED, "--output-format", "stream-json", "--verbose",
|
|||
|
|
"--permission-mode", "acceptEdits", *station.links(), *options],
|
|||
|
|
cwd=cwd, env=_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|||
|
|
for line in p.stdout:
|
|||
|
|
stream.write(line)
|
|||
|
|
o["lines"] += digest(line)
|
|||
|
|
if p.wait():
|
|||
|
|
raise RuntimeError(f"claude ended with code {p.returncode}")
|
|||
|
|
o["state"] = "done"
|
|||
|
|
except Exception as e: # noqa: BLE001 — the error shows in the page, the server goes on
|
|||
|
|
o["lines"].append(f"Error: {e}")
|
|||
|
|
o["state"] = "error"
|
|||
|
|
finally:
|
|||
|
|
stream.write(json.dumps({"type": "end", "state": o["state"]}) + "\n")
|
|||
|
|
station.detectors.cache_clear() # the detectors may have changed: the Data page computes them again
|
|||
|
|
|
|||
|
|
|
|||
|
|
def summary(name, dark):
|
|||
|
|
"""The summary of the detectors' session, then that of the tools (<analysis>/tools.md), in one page."""
|
|||
|
|
s, tools = _summary_file(name), FOLDER / name / "tools.md"
|
|||
|
|
if s is None and not tools.exists():
|
|||
|
|
raise station.Refusal("no summary for this analysis")
|
|||
|
|
text = "\n\n".join(f.read_text(encoding="utf-8") for f in (s, tools) if f and f.exists())
|
|||
|
|
return _page(text, name, dark)
|